summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorben <ben@2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109>2002-06-16 07:12:39 +0000
committerben <ben@2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109>2002-06-16 07:12:39 +0000
commitca4ace407c938d58c7fe33cb872b0705635b39cf (patch)
treefc404794ca9ec272acaaa84fdb83433c79296596
parent7d34f23699cc540bd1986cb3ae62d52952ede596 (diff)
downloadrdiff-backup-ca4ace407c938d58c7fe33cb872b0705635b39cf.tar.gz
Adapted everything to new exploded format
git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@130 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109
-rw-r--r--rdiff-backup/rdiff_backup/FilenameMapping.py94
-rw-r--r--rdiff-backup/rdiff_backup/Globals.py226
-rw-r--r--rdiff-backup/rdiff_backup/Hardlink.py262
-rw-r--r--rdiff-backup/rdiff_backup/Main.py487
-rwxr-xr-xrdiff-backup/rdiff_backup/Make.old (renamed from rdiff-backup/src/Make)0
-rw-r--r--rdiff-backup/rdiff_backup/MiscStats.py72
-rw-r--r--rdiff-backup/rdiff_backup/Rdiff.py181
-rw-r--r--rdiff-backup/rdiff_backup/SetConnections.py219
-rw-r--r--rdiff-backup/rdiff_backup/Time.py199
-rw-r--r--rdiff-backup/rdiff_backup/connection.py40
-rw-r--r--rdiff-backup/rdiff_backup/destructive_stepping.py6
-rw-r--r--rdiff-backup/rdiff_backup/highlevel.py21
-rw-r--r--rdiff-backup/rdiff_backup/increment.py15
-rw-r--r--rdiff-backup/rdiff_backup/iterfile.py2
-rw-r--r--rdiff-backup/rdiff_backup/lazy.py6
-rw-r--r--rdiff-backup/rdiff_backup/log.py6
-rw-r--r--rdiff-backup/rdiff_backup/manage.py5
-rw-r--r--rdiff-backup/rdiff_backup/restore.py9
-rw-r--r--rdiff-backup/rdiff_backup/robust.py14
-rw-r--r--rdiff-backup/rdiff_backup/rorpiter.py9
-rw-r--r--rdiff-backup/rdiff_backup/rpath.py6
-rw-r--r--rdiff-backup/rdiff_backup/selection.py6
-rw-r--r--rdiff-backup/rdiff_backup/static.py2
-rw-r--r--rdiff-backup/rdiff_backup/statistics.py76
-rw-r--r--rdiff-backup/src/FilenameMapping.py94
-rw-r--r--rdiff-backup/src/Globals.py226
-rw-r--r--rdiff-backup/src/Hardlink.py262
-rw-r--r--rdiff-backup/src/Main.py487
-rwxr-xr-xrdiff-backup/src/Make.old38
-rw-r--r--rdiff-backup/src/MiscStats.py72
-rw-r--r--rdiff-backup/src/Rdiff.py181
-rw-r--r--rdiff-backup/src/SetConnections.py219
-rw-r--r--rdiff-backup/src/Time.py199
-rw-r--r--rdiff-backup/src/connection.py40
-rw-r--r--rdiff-backup/src/destructive_stepping.py6
-rw-r--r--rdiff-backup/src/filename_mapping.py93
-rw-r--r--rdiff-backup/src/globals.py237
-rw-r--r--rdiff-backup/src/hardlink.py257
-rw-r--r--rdiff-backup/src/highlevel.py21
-rw-r--r--rdiff-backup/src/increment.py15
-rw-r--r--rdiff-backup/src/iterfile.py2
-rw-r--r--rdiff-backup/src/lazy.py6
-rw-r--r--rdiff-backup/src/log.py6
-rwxr-xr-xrdiff-backup/src/main.py498
-rw-r--r--rdiff-backup/src/manage.py5
-rw-r--r--rdiff-backup/src/rdiff.py192
-rw-r--r--rdiff-backup/src/restore.py9
-rw-r--r--rdiff-backup/src/robust.py14
-rw-r--r--rdiff-backup/src/rorpiter.py9
-rw-r--r--rdiff-backup/src/rpath.py6
-rw-r--r--rdiff-backup/src/selection.py6
-rw-r--r--rdiff-backup/src/setconnections.py219
-rw-r--r--rdiff-backup/src/static.py2
-rw-r--r--rdiff-backup/src/statistics.py76
-rw-r--r--rdiff-backup/src/ttime.py201
-rwxr-xr-xrdiff-backup/testing/chdir-wrapper6
-rwxr-xr-xrdiff-backup/testing/chdir-wrapper216
-rw-r--r--rdiff-backup/testing/commontest.py43
-rw-r--r--rdiff-backup/testing/connectiontest.py6
-rw-r--r--rdiff-backup/testing/destructive_steppingtest.py6
-rw-r--r--rdiff-backup/testing/finaltest.py9
-rw-r--r--rdiff-backup/testing/hardlinktest.py6
-rw-r--r--rdiff-backup/testing/highleveltest.py6
-rw-r--r--rdiff-backup/testing/incrementtest.py8
-rw-r--r--rdiff-backup/testing/iterfiletest.py4
-rw-r--r--rdiff-backup/testing/killtest.py5
-rw-r--r--rdiff-backup/testing/lazytest.py5
-rw-r--r--rdiff-backup/testing/rdifftest.py8
-rw-r--r--rdiff-backup/testing/regressiontest.py21
-rw-r--r--rdiff-backup/testing/restoretest.py6
-rw-r--r--rdiff-backup/testing/robusttest.py5
-rw-r--r--rdiff-backup/testing/roottest.py5
-rw-r--r--rdiff-backup/testing/rorpitertest.py7
-rw-r--r--rdiff-backup/testing/rpathtest.py5
-rw-r--r--rdiff-backup/testing/selectiontest.py7
-rwxr-xr-xrdiff-backup/testing/server.py10
-rw-r--r--rdiff-backup/testing/setconnectionstest.py11
-rw-r--r--rdiff-backup/testing/statictest.py4
-rw-r--r--rdiff-backup/testing/statisticstest.py4
-rw-r--r--rdiff-backup/testing/timetest.py12
80 files changed, 3868 insertions, 2018 deletions
diff --git a/rdiff-backup/rdiff_backup/FilenameMapping.py b/rdiff-backup/rdiff_backup/FilenameMapping.py
new file mode 100644
index 0000000..104519d
--- /dev/null
+++ b/rdiff-backup/rdiff_backup/FilenameMapping.py
@@ -0,0 +1,94 @@
+import re
+from log import *
+import Globals
+
+#######################################################################
+#
+# filename_mapping - used to coordinate related filenames
+#
+# For instance, some source filenames may contain characters not
+# allowed on the mirror end. Also, if a source filename is very long
+# (say 240 characters), the extra characters added to related
+# increments may put them over the usual 255 character limit.
+#
+
+"""Contains class methods which coordinate related filenames"""
+max_filename_length = 255
+
+# If true, enable character quoting, and set characters making
+# regex-style range.
+chars_to_quote = None
+
+# These compiled regular expressions are used in quoting and unquoting
+chars_to_quote_regexp = None
+unquoting_regexp = None
+
+# Use given char to quote. Default is set in Globals.
+quoting_char = None
+
+
+def set_init_quote_vals():
+ """Set quoting value from Globals on all conns"""
+ for conn in Globals.connections:
+ conn.FilenameMapping.set_init_quote_vals_local()
+
+def set_init_quote_vals_local():
+ """Set value on local connection, initialize regexps"""
+ global chars_to_quote
+ chars_to_quote = Globals.chars_to_quote
+ if len(Globals.quoting_char) != 1:
+ Log.FatalError("Expected single character for quoting char,"
+ "got '%s' instead" % (Globals.quoting_char,))
+ quoting_char = Globals.quoting_char
+ init_quoting_regexps()
+
+def init_quoting_regexps():
+ """Compile quoting regular expressions"""
+ global chars_to_quote_regexp, unquoting_regexp
+ try:
+ chars_to_quote_regexp = \
+ re.compile("[%s%s]" % (chars_to_quote, quoting_char), re.S)
+ unquoting_regexp = re.compile("%s[0-9]{3}" % quoting_char, re.S)
+ except re.error:
+ Log.FatalError("Error '%s' when processing char quote list %s" %
+ (re.error, chars_to_quote))
+
+def quote(path):
+ """Return quoted version of given path
+
+ Any characters quoted will be replaced by the quoting char and
+ the ascii number of the character. For instance, "10:11:12"
+ would go to "10;05811;05812" if ":" were quoted and ";" were
+ the quoting character.
+
+ """
+ return chars_to_quote_regexp.sub(quote_single, path)
+
+def quote_single(match):
+ """Return replacement for a single character"""
+ return "%s%03d" % (quoting_char, ord(match.group()))
+
+def unquote(path):
+ """Return original version of quoted filename"""
+ return unquoting_regexp.sub(unquote_single, path)
+
+def unquote_single(match):
+ """Unquote a single quoted character"""
+ assert len(match.group()) == 4
+ return chr(int(match.group()[1:]))
+
+def get_quoted_dir_children(rpath):
+ """For rpath directory, return list of quoted children in dir"""
+ if not rpath.isdir(): return []
+ dir_pairs = [(unquote(filename), filename)
+ for filename in Robust.listrp(rpath)]
+ dir_pairs.sort() # sort by real index, not quoted part
+ child_list = []
+ for unquoted, filename in dir_pairs:
+ childrp = rpath.append(unquoted)
+ childrp.quote_path()
+ child_list.append(childrp)
+ return child_list
+
+
+
diff --git a/rdiff-backup/rdiff_backup/Globals.py b/rdiff-backup/rdiff_backup/Globals.py
new file mode 100644
index 0000000..ca6e8d1
--- /dev/null
+++ b/rdiff-backup/rdiff_backup/Globals.py
@@ -0,0 +1,226 @@
+import re, os
+
+# The current version of rdiff-backup
+version = "0.8.0"
+
+# If this is set, use this value in seconds as the current time
+# instead of reading it from the clock.
+current_time = None
+
+# This determines how many bytes to read at a time when copying
+blocksize = 32768
+
+# This is used by the BufferedRead class to determine how many
+# bytes to request from the underlying file per read(). Larger
+# values may save on connection overhead and latency.
+conn_bufsize = 98304
+
+# True if script is running as a server
+server = None
+
+# uid and gid of the owner of the rdiff-backup process. This can
+# vary depending on the connection.
+process_uid = os.getuid()
+process_gid = os.getgid()
+
+# If true, when copying attributes, also change target's uid/gid
+change_ownership = None
+
+# If true, change the permissions of unwriteable mirror files
+# (such as directories) so that they can be written, and then
+# change them back. This defaults to 1 just in case the process
+# is not running as root (root doesn't need to change
+# permissions).
+change_mirror_perms = (process_uid != 0)
+
+# If true, temporarily change permissions of unreadable files in
+# the source directory to make sure we can read all files.
+change_source_perms = None
+
+# If true, try to reset the atimes of the source partition.
+preserve_atime = None
+
+# This will be set as soon as the LocalConnection class loads
+local_connection = None
+
+# All connections should be added to the following list, so
+# further global changes can be propagated to the remote systems.
+# The first element should be Globals.local_connection. For a
+# server, the second is the connection to the client.
+connections = []
+
+# Each process should have a connection number unique to the
+# session. The client has connection number 0.
+connection_number = 0
+
+# Dictionary pairing connection numbers with connections. Set in
+# SetConnections for all connections.
+connection_dict = {}
+
+# True if the script is the end that reads the source directory
+# for backups. It is true for purely local sessions.
+isbackup_reader = None
+
+# Connection of the real backup reader (for which isbackup_reader
+# is true)
+backup_reader = None
+
+# True if the script is the end that writes to the increment and
+# mirror directories. True for purely local sessions.
+isbackup_writer = None
+
+# Connection of the backup writer
+backup_writer = None
+
+# True if this process is the client invoked by the user
+isclient = None
+
+# Connection of the client
+client_conn = None
+
+# This list is used by the set function below. When a new
+# connection is created with init_connection, its Globals class
+# will match this one for all the variables mentioned in this
+# list.
+changed_settings = []
+
+# rdiff-backup will try to checkpoint its state every
+# checkpoint_interval seconds. Then when resuming, at most this
+# amount of time is lost.
+checkpoint_interval = 20
+
+# The RPath of the rdiff-backup-data directory.
+rbdir = None
+
+# Indicates if a resume or a lack of resume is forced. This
+# should be None for the default. 0 means don't resume, and 1
+# means resume.
+resume = None
+
+# If there has been an aborted backup fewer than this many seconds
+# ago, attempt to resume it where it left off instead of starting
+# a new one.
+resume_window = 7200
+
+# This string is used when recognizing and creating time strings.
+# If the time_separator is ":", then W3 datetime strings like
+# 2001-12-07T04:22:01-07:00 are produced. It can be set to "_" to
+# make filenames that don't contain colons, which aren't allowed
+# under MS windows NT.
+time_separator = ":"
+
+# quoting_enabled is true if we should quote certain characters in
+# filenames on the source side (see FilenameMapping for more
+# info). chars_to_quote is a string whose characters should be
+# quoted, and quoting_char is the character to quote with.
+quoting_enabled = None
+chars_to_quote = ""
+quoting_char = ';'
+
+# If true, emit output intended to be easily readable by a
+# computer. False means output is intended for humans.
+parsable_output = None
+
+# If true, then hardlinks will be preserved to mirror and recorded
+# in the increments directory. There is also a difference here
+# between None and 0. When restoring, None or 1 means to preserve
+# hardlinks iff can find a hardlink dictionary. 0 means ignore
+# hardlink information regardless.
+preserve_hardlinks = 1
+
+# If this is false, then rdiff-backup will not compress any
+# increments. Default is to compress based on regexp below.
+compression = 1
+
+# Increments based on files whose names match this
+# case-insensitive regular expression won't be compressed (applies
+# to .snapshots and .diffs). The second below will be the
+# compiled version of the first.
+no_compression_regexp_string = "(?i).*\\.(gz|z|bz|bz2|tgz|zip|rpm|deb|" \
+ "jpg|gif|png|jp2|mp3|ogg|avi|wmv|mpeg|mpg|rm|mov)$"
+no_compression_regexp = None
+
+# If true, filelists and directory statistics will be split on
+# nulls instead of newlines.
+null_separator = None
+
+# Determines whether or not ssh will be run with the -C switch
+ssh_compression = 1
+
+# If true, print statistics after successful backup
+print_statistics = None
+
+# On the reader and writer connections, the following will be
+# replaced by the source and mirror Select objects respectively.
+select_source, select_mirror = None, None
+
+# On the backup writer connection, holds the main incrementing
+# function. Access is provided to increment error counts.
+ITR = None
+
+def get(name):
+ """Return the value of something in this module"""
+ return globals()[name]
+
+def is_not_None(name):
+ """Returns true if value is not None"""
+ return globals()[name] is not None
+
+def set(name, val):
+ """Set the value of something in this module
+
+ Use this instead of writing the values directly if the setting
+ matters to remote sides. This function updates the
+ changed_settings list, so other connections know to copy the
+ changes.
+
+ """
+ changed_settings.append(name)
+ globals()[name] = val
+
+def set_integer(name, val):
+ """Like set, but make sure val is an integer"""
+ try: intval = int(val)
+ except ValueError:
+ Log.FatalError("Variable %s must be set to an integer -\n"
+ "received %s instead." % (name, val))
+ set(name, intval)
+
+def get_dict_val(name, key):
+ """Return val from dictionary in this class"""
+ return globals()[name][key]
+
+def set_dict_val(name, key, val):
+ """Set value for dictionary in this class"""
+ globals()[name][key] = val
+
+def postset_regexp(name, re_string, flags = None):
+ """Compile re_string on all existing connections, set to name"""
+ for conn in connections:
+ conn.Globals.postset_regexp_local(name, re_string, flags)
+
+def postset_regexp_local(name, re_string, flags):
+ """Set name to compiled re_string locally"""
+ if flags: globals()[name] = re.compile(re_string, flags)
+ else: globals()[name] = re.compile(re_string)
+
+def set_select(dsrpath, tuplelist, quote_mode, *filelists):
+ """Initialize select object using tuplelist
+
+ Note that each list in filelists must each be passed as
+ separate arguments, so each is recognized as a file by the
+ connection. Otherwise we will get an error because a list
+ containing files can't be pickled.
+
+ """
+ global select_source, select_mirror
+ if dsrpath.source:
+ select_source = Select(dsrpath, quote_mode)
+ select_source.ParseArgs(tuplelist, filelists)
+ else:
+ select_mirror = Select(dsrpath, quote_mode)
+ select_mirror.ParseArgs(tuplelist, filelists)
+
+
+from rpath import * # kludge to avoid circularity - not needed in this module
+from selection import *
diff --git a/rdiff-backup/rdiff_backup/Hardlink.py b/rdiff-backup/rdiff_backup/Hardlink.py
new file mode 100644
index 0000000..9389b6f
--- /dev/null
+++ b/rdiff-backup/rdiff_backup/Hardlink.py
@@ -0,0 +1,262 @@
+from __future__ import generators
+import cPickle
+
+#######################################################################
+#
+# hardlink - code for preserving and restoring hardlinks
+#
+# If the preserve_hardlinks option is selected, linked files in the
+# source directory will be linked in the mirror directory. Linked
+# files are treated like any other with respect to incrementing, but a
+# database of all links will be recorded at each session, so linked
+# files can still be restored from the increments.
+#
+
+"""Hardlink class methods and data
+
+All these functions are meant to be executed on the destination
+side. The source side should only transmit inode information.
+
+"""
+
+# In all of these lists of indicies are the values. The keys in
+# _inode_ ones are (inode, devloc) pairs.
+_src_inode_indicies = {}
+_dest_inode_indicies = {}
+
+# The keys for these two are just indicies. They share values
+# with the earlier dictionaries.
+_src_index_indicies = {}
+_dest_index_indicies = {}
+
+# When a linked file is restored, its path is added to this dict,
+# so it can be found when later paths being restored are linked to
+# it.
+_restore_index_path = {}
+
+def get_inode_key(rorp):
+ """Return rorp's key for _inode_ dictionaries"""
+ return (rorp.getinode(), rorp.getdevloc())
+
+def get_indicies(rorp, source):
+ """Return a list of similarly linked indicies, using rorp's index"""
+ if source: dict = _src_index_indicies
+ else: dict = _dest_index_indicies
+ try: return dict[rorp.index]
+ except KeyError: return []
+
+def add_rorp(rorp, source):
+ """Process new rorp and update hard link dictionaries
+
+ First enter it into src_inode_indicies. If we have already
+ seen all the hard links, then we can delete the entry.
+ Everything must stay recorded in src_index_indicies though.
+
+ """
+ if not rorp.isreg() or rorp.getnumlinks() < 2: return
+
+ if source:
+ inode_dict, index_dict = _src_inode_indicies, _src_index_indicies
+ else: inode_dict, index_dict = _dest_inode_indicies, _dest_index_indicies
+
+ rp_inode_key = get_inode_key(rorp)
+ if inode_dict.has_key(rp_inode_key):
+ index_list = inode_dict[rp_inode_key]
+ index_list.append(rorp.index)
+ if len(index_list) == rorp.getnumlinks():
+ del inode_dict[rp_inode_key]
+ else: # make new entry in both src dicts
+ index_list = [rorp.index]
+ inode_dict[rp_inode_key] = index_list
+ index_dict[rorp.index] = index_list
+
+def add_rorp_iter(iter, source):
+ """Return new rorp iterator like iter that add_rorp's first"""
+ for rorp in iter:
+ add_rorp(rorp, source)
+ yield rorp
+
+def rorp_eq(src_rorp, dest_rorp):
+ """Compare hardlinked for equality
+
+ Two files may otherwise seem equal but be hardlinked in
+ different ways. This function considers them equal enough if
+ they have been hardlinked correctly to the previously seen
+ indicies.
+
+ """
+ assert src_rorp.index == dest_rorp.index
+ if (not src_rorp.isreg() or not dest_rorp.isreg() or
+ src_rorp.getnumlinks() == dest_rorp.getnumlinks() == 1):
+ return 1 # Hard links don't apply
+
+ src_index_list = get_indicies(src_rorp, 1)
+ dest_index_list = get_indicies(dest_rorp, None)
+
+ # If a list only has one element, then it is only hardlinked
+ # to itself so far, so that is not a genuine difference yet.
+ if not src_index_list or len(src_index_list) == 1:
+ return not dest_index_list or len(dest_index_list) == 1
+ if not dest_index_list or len(dest_index_list) == 1: return None
+
+ # Both index lists exist and are non-empty
+ return src_index_list == dest_index_list # they are always sorted
+
+def islinked(rorp):
+ """True if rorp's index is already linked to something on src side"""
+ return len(get_indicies(rorp, 1)) >= 2
+
+def restore_link(index, rpath):
+ """Restores a linked file by linking it
+
+ When restoring, all the hardlink data is already present, and
+ we can only link to something already written. In either
+ case, add to the _restore_index_path dict, so we know later
+ that the file is available for hard
+ linking.
+
+ Returns true if succeeded in creating rpath, false if must
+ restore rpath normally.
+
+ """
+ if index not in _src_index_indicies: return None
+ for linked_index in _src_index_indicies[index]:
+ if linked_index in _restore_index_path:
+ srcpath = _restore_index_path[linked_index]
+ Log("Restoring %s by hard linking to %s" %
+ (rpath.path, srcpath), 6)
+ rpath.hardlink(srcpath)
+ return 1
+ _restore_index_path[index] = rpath.path
+ return None
+
+def link_rp(src_rorp, dest_rpath, dest_root = None):
+ """Make dest_rpath into a link analogous to that of src_rorp"""
+ if not dest_root: dest_root = dest_rpath # use base of dest_rpath
+ dest_link_rpath = RPath(dest_root.conn, dest_root.base,
+ get_indicies(src_rorp, 1)[0])
+ dest_rpath.hardlink(dest_link_rpath.path)
+
+def write_linkdict(rpath, dict, compress = None):
+ """Write link data to the rbdata dir
+
+ It is stored as the a big pickled dictionary dated to match
+ the current hardlinks.
+
+ """
+ assert (Globals.isbackup_writer and
+ rpath.conn is Globals.local_connection)
+ tf = TempFileManager.new(rpath)
+ def init():
+ fp = tf.open("wb", compress)
+ cPickle.dump(dict, fp)
+ assert not fp.close()
+ tf.setdata()
+ Robust.make_tf_robustaction(init, (tf,), (rpath,)).execute()
+
+def get_linkrp(data_rpath, time, prefix):
+ """Return RPath of linkdata, or None if cannot find"""
+ for rp in map(data_rpath.append, data_rpath.listdir()):
+ if (rp.isincfile() and rp.getincbase_str() == prefix and
+ (rp.getinctype() == 'snapshot' or rp.getinctype() == 'data')
+ and Time.stringtotime(rp.getinctime()) == time):
+ return rp
+ return None
+
+def get_linkdata(data_rpath, time, prefix = 'hardlink_data'):
+ """Return index dictionary written by write_linkdata at time"""
+ rp = get_linkrp(data_rpath, time, prefix)
+ if not rp: return None
+ fp = rp.open("rb", rp.isinccompressed())
+ index_dict = cPickle.load(fp)
+ assert not fp.close()
+ return index_dict
+
+def final_writedata():
+ """Write final checkpoint data to rbdir after successful backup"""
+ global final_inc
+ if _src_index_indicies:
+ Log("Writing hard link data", 6)
+ if Globals.compression:
+ final_inc = Globals.rbdir.append("hardlink_data.%s.data.gz" %
+ Time.curtimestr)
+ else: final_inc = Globals.rbdir.append("hardlink_data.%s.data" %
+ Time.curtimestr)
+ write_linkdict(final_inc, _src_index_indicies, Globals.compression)
+ else: # no hardlinks, so writing unnecessary
+ final_inc = None
+
+def retrieve_final(time):
+ """Set source index dictionary from hardlink_data file if avail"""
+ global _src_index_indicies
+ hd = get_linkdata(Globals.rbdir, time)
+ if hd is None: return None
+ _src_index_indicies = hd
+ return 1
+
+def final_checkpoint(data_rpath):
+ """Write contents of the four dictionaries to the data dir
+
+ If rdiff-backup receives a fatal error, it may still be able
+ to save the contents of the four hard link dictionaries.
+ Because these dictionaries may be big, they are not saved
+ after every 20 seconds or whatever, but just at the end.
+
+ """
+ Log("Writing intermediate hard link data to disk", 2)
+ src_inode_rp = data_rpath.append("hardlink_source_inode_checkpoint."
+ "%s.data" % Time.curtimestr)
+ src_index_rp = data_rpath.append("hardlink_source_index_checkpoint."
+ "%s.data" % Time.curtimestr)
+ dest_inode_rp = data_rpath.append("hardlink_dest_inode_checkpoint."
+ "%s.data" % Time.curtimestr)
+ dest_index_rp = data_rpath.append("hardlink_dest_index_checkpoint."
+ "%s.data" % Time.curtimestr)
+ for (rp, dict) in ((src_inode_rp, _src_inode_indicies),
+ (src_index_rp, _src_index_indicies),
+ (dest_inode_rp, _dest_inode_indicies),
+ (dest_index_rp, _dest_index_indicies)):
+ write_linkdict(rp, dict)
+
+def retrieve_checkpoint(data_rpath, time):
+ """Retrieve hardlink data from final checkpoint
+
+ Return true if the retrieval worked, false otherwise.
+
+ """
+ global _src_inode_indicies, _src_index_indicies
+ global _dest_inode_indicies, _dest_index_indicies
+ try:
+ src_inode = get_linkdata(data_rpath, time,
+ "hardlink_source_inode_checkpoint")
+ src_index = get_linkdata(data_rpath, time,
+ "hardlink_source_index_checkpoint")
+ dest_inode = get_linkdata(data_rpath, time,
+ "hardlink_dest_inode_checkpoint")
+ dest_index = get_linkdata(data_rpath, time,
+ "hardlink_dest_index_checkpoint")
+ except cPickle.UnpicklingError:
+ Log("Unpickling Error", 2)
+ return None
+ if (src_inode is None or src_index is None or
+ dest_inode is None or dest_index is None): return None
+ _src_inode_indicies, _src_index_indicies = src_inode, src_index
+ _dest_inode_indicies, _dest_index_indicies = dest_inode, dest_index
+ return 1
+
+def remove_all_checkpoints():
+ """Remove all hardlink checkpoint information from directory"""
+ prefix_list = ["hardlink_source_inode_checkpoint",
+ "hardlink_source_index_checkpoint",
+ "hardlink_dest_inode_checkpoint",
+ "hardlink_dest_index_checkpoint"]
+ for rp in map(Globals.rbdir.append, Globals.rbdir.listdir()):
+ if (rp.isincfile() and rp.getincbase_str() in prefix_list and
+ (rp.getinctype() == 'snapshot' or rp.getinctype() == 'data')):
+ rp.delete()
+
+
+from log import *
+from robust import *
+from rpath import *
+import Globals, Time
diff --git a/rdiff-backup/rdiff_backup/Main.py b/rdiff-backup/rdiff_backup/Main.py
new file mode 100644
index 0000000..94ca04a
--- /dev/null
+++ b/rdiff-backup/rdiff_backup/Main.py
@@ -0,0 +1,487 @@
+import getopt, sys, re
+from log import *
+from lazy import *
+from connection import *
+from rpath import *
+from destructive_stepping import *
+from robust import *
+from restore import *
+from highlevel import *
+from manage import *
+import Globals, Time, SetConnections
+
+#######################################################################
+#
+# main - Start here: Read arguments, set global settings, etc.
+#
+action = None
+remote_cmd, remote_schema = None, None
+force = None
+select_opts, select_mirror_opts = [], []
+select_files = []
+
+def parse_cmdlineoptions(arglist):
+ """Parse argument list and set global preferences"""
+ global args, action, force, restore_timestr, remote_cmd, remote_schema
+ global remove_older_than_string
+ def sel_fl(filename):
+ """Helper function for including/excluding filelists below"""
+ try: return open(filename, "r")
+ except IOError: Log.FatalError("Error opening file %s" % filename)
+
+ try: optlist, args = getopt.getopt(arglist, "blmr:sv:V",
+ ["backup-mode", "calculate-average",
+ "change-source-perms", "chars-to-quote=",
+ "checkpoint-interval=", "current-time=", "exclude=",
+ "exclude-device-files", "exclude-filelist=",
+ "exclude-filelist-stdin", "exclude-mirror=",
+ "exclude-regexp=", "force", "include=",
+ "include-filelist=", "include-filelist-stdin",
+ "include-regexp=", "list-increments", "mirror-only",
+ "no-compression", "no-compression-regexp=",
+ "no-hard-links", "no-resume", "null-separator",
+ "parsable-output", "print-statistics", "quoting-char=",
+ "remote-cmd=", "remote-schema=", "remove-older-than=",
+ "restore-as-of=", "resume", "resume-window=", "server",
+ "ssh-no-compression", "terminal-verbosity=",
+ "test-server", "verbosity", "version", "windows-mode",
+ "windows-time-format"])
+ except getopt.error, e:
+ commandline_error("Bad commandline options: %s" % str(e))
+
+ for opt, arg in optlist:
+ if opt == "-b" or opt == "--backup-mode": action = "backup"
+ elif opt == "--calculate-average": action = "calculate-average"
+ elif opt == "--change-source-perms":
+ Globals.set('change_source_perms', 1)
+ elif opt == "--chars-to-quote":
+ Globals.set('chars_to_quote', arg)
+ Globals.set('quoting_enabled', 1)
+ elif opt == "--checkpoint-interval":
+ Globals.set_integer('checkpoint_interval', arg)
+ elif opt == "--current-time":
+ Globals.set_integer('current_time', arg)
+ elif opt == "--exclude": select_opts.append((opt, arg))
+ elif opt == "--exclude-device-files": select_opts.append((opt, arg))
+ elif opt == "--exclude-filelist":
+ select_opts.append((opt, arg))
+ select_files.append(sel_fl(arg))
+ elif opt == "--exclude-filelist-stdin":
+ select_opts.append(("--exclude-filelist", "standard input"))
+ select_files.append(sys.stdin)
+ elif opt == "--exclude-mirror":
+ select_mirror_opts.append(("--exclude", arg))
+ elif opt == "--exclude-regexp": select_opts.append((opt, arg))
+ elif opt == "--force": force = 1
+ elif opt == "--include": select_opts.append((opt, arg))
+ elif opt == "--include-filelist":
+ select_opts.append((opt, arg))
+ select_files.append(sel_fl(arg))
+ elif opt == "--include-filelist-stdin":
+ select_opts.append(("--include-filelist", "standard input"))
+ select_files.append(sys.stdin)
+ elif opt == "--include-regexp": select_opts.append((opt, arg))
+ elif opt == "-l" or opt == "--list-increments":
+ action = "list-increments"
+ elif opt == "-m" or opt == "--mirror-only": action = "mirror"
+ elif opt == "--no-compression": Globals.set("compression", None)
+ elif opt == "--no-compression-regexp":
+ Globals.set("no_compression_regexp_string", arg)
+ elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0)
+ elif opt == '--no-resume': Globals.resume = 0
+ elif opt == "--null-separator": Globals.set("null_separator", 1)
+ elif opt == "-r" or opt == "--restore-as-of":
+ restore_timestr, action = arg, "restore-as-of"
+ elif opt == "--parsable-output": Globals.set('parsable_output', 1)
+ elif opt == "--print-statistics":
+ Globals.set('print_statistics', 1)
+ elif opt == "--quoting-char":
+ Globals.set('quoting_char', arg)
+ Globals.set('quoting_enabled', 1)
+ elif opt == "--remote-cmd": remote_cmd = arg
+ elif opt == "--remote-schema": remote_schema = arg
+ elif opt == "--remove-older-than":
+ remove_older_than_string = arg
+ action = "remove-older-than"
+ elif opt == '--resume': Globals.resume = 1
+ elif opt == '--resume-window':
+ Globals.set_integer('resume_window', arg)
+ elif opt == "-s" or opt == "--server": action = "server"
+ elif opt == "--ssh-no-compression":
+ Globals.set('ssh_compression', None)
+ elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg)
+ elif opt == "--test-server": action = "test-server"
+ elif opt == "-V" or opt == "--version":
+ print "rdiff-backup " + Globals.version
+ sys.exit(0)
+ elif opt == "-v" or opt == "--verbosity": Log.setverbosity(arg)
+ elif opt == "--windows-mode":
+ Globals.set('time_separator', "_")
+ Globals.set('chars_to_quote', ":")
+ Globals.set('quoting_enabled', 1)
+ elif opt == '--windows-time-format':
+ Globals.set('time_separator', "_")
+ else: Log.FatalError("Unknown option %s" % opt)
+
+def set_action():
+ """Check arguments and try to set action"""
+ global action
+ l = len(args)
+ if not action:
+ if l == 0: commandline_error("No arguments given")
+ elif l == 1: action = "restore"
+ elif l == 2:
+ if RPath(Globals.local_connection, args[0]).isincfile():
+ action = "restore"
+ else: action = "backup"
+ else: commandline_error("Too many arguments given")
+
+ if l == 0 and action != "server" and action != "test-server":
+ commandline_error("No arguments given")
+ if l > 0 and action == "server":
+ commandline_error("Too many arguments given")
+ if l < 2 and (action == "backup" or action == "mirror" or
+ action == "restore-as-of"):
+ commandline_error("Two arguments are required (source, destination).")
+ if l == 2 and (action == "list-increments" or
+ action == "remove-older-than"):
+ commandline_error("Only use one argument, "
+ "the root of the backup directory")
+ if l > 2 and action != "calculate-average":
+ commandline_error("Too many arguments given")
+
+def commandline_error(message):
+ sys.stderr.write("Error: %s\n" % message)
+ sys.stderr.write("See the rdiff-backup manual page for instructions\n")
+ sys.exit(1)
+
+def misc_setup(rps):
+ """Set default change ownership flag, umask, relay regexps"""
+ if ((len(rps) == 2 and rps[1].conn.os.getuid() == 0) or
+ (len(rps) < 2 and os.getuid() == 0)):
+ # Allow change_ownership if destination connection is root
+ for conn in Globals.connections:
+ conn.Globals.set('change_ownership', 1)
+ for rp in rps: rp.setdata() # Update with userinfo
+
+ os.umask(077)
+ Time.setcurtime(Globals.current_time)
+ FilenameMapping.set_init_quote_vals()
+ Globals.set("isclient", 1)
+ SetConnections.UpdateGlobal("client_conn", Globals.local_connection)
+
+ # This is because I originally didn't think compiled regexps
+ # could be pickled, and so must be compiled on remote side.
+ Globals.postset_regexp('no_compression_regexp',
+ Globals.no_compression_regexp_string)
+
+ for conn in Globals.connections: Robust.install_signal_handlers()
+
+def take_action(rps):
+ """Do whatever action says"""
+ if action == "server": PipeConnection(sys.stdin, sys.stdout).Server()
+ elif action == "backup": Backup(rps[0], rps[1])
+ elif action == "restore": restore(*rps)
+ elif action == "restore-as-of": RestoreAsOf(rps[0], rps[1])
+ elif action == "mirror": Mirror(rps[0], rps[1])
+ elif action == "test-server": SetConnections.TestConnections()
+ elif action == "list-increments": ListIncrements(rps[0])
+ elif action == "remove-older-than": RemoveOlderThan(rps[0])
+ elif action == "calculate-average": CalculateAverage(rps)
+ else: raise AssertionError("Unknown action " + action)
+
+def cleanup():
+ """Do any last minute cleaning before exiting"""
+ Log("Cleaning up", 6)
+ Log.close_logfile()
+ if not Globals.server: SetConnections.CloseConnections()
+
+def Main(arglist):
+ """Start everything up!"""
+ parse_cmdlineoptions(arglist)
+ set_action()
+ rps = SetConnections.InitRPs(args, remote_schema, remote_cmd)
+ misc_setup(rps)
+ take_action(rps)
+ cleanup()
+
+
+def Mirror(src_rp, dest_rp):
+ """Turn dest_path into a copy of src_path"""
+ Log("Mirroring %s to %s" % (src_rp.path, dest_rp.path), 5)
+ mirror_check_paths(src_rp, dest_rp)
+ # Since no "rdiff-backup-data" dir, use root of destination.
+ SetConnections.UpdateGlobal('rbdir', dest_rp)
+ SetConnections.BackupInitConnections(src_rp.conn, dest_rp.conn)
+ HighLevel.Mirror(src_rp, dest_rp)
+
+def mirror_check_paths(rpin, rpout):
+ """Check paths and return rpin, rpout"""
+ if not rpin.lstat():
+ Log.FatalError("Source directory %s does not exist" % rpin.path)
+ if rpout.lstat() and not force: Log.FatalError(
+"""Destination %s exists so continuing could mess it up. Run
+rdiff-backup with the --force option if you want to mirror anyway.""" %
+ rpout.path)
+
+
+def Backup(rpin, rpout):
+ """Backup, possibly incrementally, src_path to dest_path."""
+ SetConnections.BackupInitConnections(rpin.conn, rpout.conn)
+ backup_init_select(rpin, rpout)
+ backup_init_dirs(rpin, rpout)
+ RSI = Globals.backup_writer.Resume.ResumeCheck()
+ SaveState.init_filenames()
+ if prevtime:
+ Time.setprevtime(prevtime)
+ HighLevel.Mirror_and_increment(rpin, rpout, incdir, RSI)
+ else: HighLevel.Mirror(rpin, rpout, incdir, RSI)
+ backup_touch_curmirror(rpin, rpout)
+
+def backup_init_select(rpin, rpout):
+ """Create Select objects on source and dest connections"""
+ rpin.conn.Globals.set_select(DSRPath(1, rpin), select_opts,
+ None, *select_files)
+ rpout.conn.Globals.set_select(DSRPath(None, rpout), select_mirror_opts, 1)
+
+def backup_init_dirs(rpin, rpout):
+ """Make sure rpin and rpout are valid, init data dir and logging"""
+ global datadir, incdir, prevtime
+ if rpout.lstat() and not rpout.isdir():
+ if not force: Log.FatalError("Destination %s exists and is not a "
+ "directory" % rpout.path)
+ else:
+ Log("Deleting %s" % rpout.path, 3)
+ rpout.delete()
+
+ if not rpin.lstat():
+ Log.FatalError("Source directory %s does not exist" % rpin.path)
+ elif not rpin.isdir():
+ Log.FatalError("Source %s is not a directory" % rpin.path)
+
+ datadir = rpout.append("rdiff-backup-data")
+ SetConnections.UpdateGlobal('rbdir', datadir)
+ incdir = RPath(rpout.conn, os.path.join(datadir.path, "increments"))
+ prevtime = backup_get_mirrortime()
+
+ if rpout.lstat():
+ if rpout.isdir() and not rpout.listdir(): # rpout is empty dir
+ rpout.chmod(0700) # just make sure permissions aren't too lax
+ elif not datadir.lstat() and not force: Log.FatalError(
+"""Destination directory %s exists, but does not look like a
+rdiff-backup directory. Running rdiff-backup like this could mess up
+what is currently in it. If you want to overwrite it, run
+rdiff-backup with the --force option.""" % rpout.path)
+
+ if not rpout.lstat():
+ try: rpout.mkdir()
+ except os.error:
+ Log.FatalError("Unable to create directory %s" % rpout.path)
+ if not datadir.lstat(): datadir.mkdir()
+ if Log.verbosity > 0:
+ Log.open_logfile(datadir.append("backup.log"))
+ backup_warn_if_infinite_regress(rpin, rpout)
+
+def backup_warn_if_infinite_regress(rpin, rpout):
+ """Warn user if destination area contained in source area"""
+ if rpout.conn is rpin.conn: # it's meaningful to compare paths
+ if ((len(rpout.path) > len(rpin.path)+1 and
+ rpout.path[:len(rpin.path)] == rpin.path and
+ rpout.path[len(rpin.path)] == '/') or
+ (rpin.path == "." and rpout.path[0] != '/' and
+ rpout.path[:2] != '..')):
+ # Just a few heuristics, we don't have to get every case
+ if Globals.backup_reader.Globals.select_source.Select(rpout): Log(
+"""Warning: The destination directory '%s' may be contained in the
+source directory '%s'. This could cause an infinite regress. You
+may need to use the --exclude option.""" % (rpout.path, rpin.path), 2)
+
+def backup_get_mirrorrps():
+ """Return list of current_mirror rps"""
+ if not datadir.isdir(): return []
+ mirrorrps = [datadir.append(fn) for fn in datadir.listdir()
+ if fn.startswith("current_mirror.")]
+ return filter(lambda rp: rp.isincfile(), mirrorrps)
+
+def backup_get_mirrortime():
+ """Return time in seconds of previous mirror, or None if cannot"""
+ mirrorrps = backup_get_mirrorrps()
+ if not mirrorrps: return None
+ if len(mirrorrps) > 1:
+ Log(
+"""Warning: duplicate current_mirror files found. Perhaps something
+went wrong during your last backup? Using """ + mirrorrps[-1].path, 2)
+
+ timestr = mirrorrps[-1].getinctime()
+ return Time.stringtotime(timestr)
+
+def backup_touch_curmirror(rpin, rpout):
+ """Make a file like current_mirror.time.data to record time
+
+ Also updates rpout so mod times don't get messed up.
+
+ """
+ map(RPath.delete, backup_get_mirrorrps())
+ mirrorrp = datadir.append("current_mirror.%s.%s" % (Time.curtimestr,
+ "data"))
+ Log("Touching mirror marker %s" % mirrorrp.path, 6)
+ mirrorrp.touch()
+ RPath.copy_attribs(rpin, rpout)
+
+
+def restore(src_rp, dest_rp = None):
+ """Main restoring function
+
+ Here src_rp should be an increment file, and if dest_rp is
+ missing it defaults to the base of the increment.
+
+ """
+ rpin, rpout = restore_check_paths(src_rp, dest_rp)
+ time = Time.stringtotime(rpin.getinctime())
+ restore_common(rpin, rpout, time)
+
+def RestoreAsOf(rpin, target):
+ """Secondary syntax for restore operation
+
+ rpin - RPath of mirror file to restore (not nec. with correct index)
+ target - RPath of place to put restored file
+
+ """
+ restore_check_paths(rpin, target, 1)
+ try: time = Time.genstrtotime(restore_timestr)
+ except Time.TimeException, exc: Log.FatalError(str(exc))
+ restore_common(rpin, target, time)
+
+def restore_common(rpin, target, time):
+ """Restore operation common to Restore and RestoreAsOf"""
+ Log("Starting Restore", 5)
+ mirror_root, index = restore_get_root(rpin)
+ mirror = mirror_root.new_index(index)
+ inc_rpath = datadir.append_path('increments', index)
+ restore_init_select(mirror_root, target)
+ Log.open_logfile(datadir.append("restore.log"))
+ Restore.Restore(inc_rpath, mirror, target, time)
+
+def restore_check_paths(rpin, rpout, restoreasof = None):
+ """Check paths and return pair of corresponding rps"""
+ if not restoreasof:
+ if not rpin.lstat():
+ Log.FatalError("Source file %s does not exist" % rpin.path)
+ elif not rpin.isincfile():
+ Log.FatalError("""File %s does not look like an increment file.
+
+Try restoring from an increment file (the filenames look like
+"foobar.2001-09-01T04:49:04-07:00.diff").""" % rpin.path)
+
+ if not rpout: rpout = RPath(Globals.local_connection,
+ rpin.getincbase_str())
+ if rpout.lstat():
+ Log.FatalError("Restore target %s already exists, "
+ "and will not be overwritten." % rpout.path)
+ return rpin, rpout
+
+def restore_init_select(rpin, rpout):
+ """Initialize Select
+
+ Unlike the backup selections, here they are on the local
+ connection, because the backup operation is pipelined in a way
+ the restore operation isn't.
+
+ """
+ Globals.set_select(DSRPath(1, rpin), select_mirror_opts, None)
+ Globals.set_select(DSRPath(None, rpout), select_opts, None, *select_files)
+
+def restore_get_root(rpin):
+ """Return (mirror root, index) and set the data dir
+
+ The idea here is to keep backing up on the path until we find
+ a directory that contains "rdiff-backup-data". That is the
+ mirror root. If the path from there starts
+ "rdiff-backup-data/increments*", then the index is the
+ remainder minus that. Otherwise the index is just the path
+ minus the root.
+
+ All this could fail if the increment file is pointed to in a
+ funny way, using symlinks or somesuch.
+
+ """
+ global datadir
+ if rpin.isincfile(): relpath = rpin.getincbase().path
+ else: relpath = rpin.path
+ pathcomps = os.path.join(rpin.conn.os.getcwd(), relpath).split("/")
+ assert len(pathcomps) >= 2 # path should be relative to /
+
+ i = len(pathcomps)
+ while i >= 2:
+ parent_dir = RPath(rpin.conn, "/".join(pathcomps[:i]))
+ if (parent_dir.isdir() and
+ "rdiff-backup-data" in parent_dir.listdir()): break
+ i = i-1
+ else: Log.FatalError("Unable to find rdiff-backup-data directory")
+
+ rootrp = parent_dir
+ Log("Using mirror root directory %s" % rootrp.path, 6)
+
+ datadir = rootrp.append_path("rdiff-backup-data")
+ SetConnections.UpdateGlobal('rbdir', datadir)
+ if not datadir.isdir():
+ Log.FatalError("Unable to read rdiff-backup-data directory %s" %
+ datadir.path)
+
+ from_datadir = tuple(pathcomps[i:])
+ if not from_datadir or from_datadir[0] != "rdiff-backup-data":
+ return (rootrp, from_datadir) # in mirror, not increments
+ assert from_datadir[1] == "increments"
+ return (rootrp, from_datadir[2:])
+
+
+def ListIncrements(rp):
+ """Print out a summary of the increments and their times"""
+ mirror_root, index = restore_get_root(rp)
+ Globals.rbdir = datadir = \
+ mirror_root.append_path("rdiff-backup-data")
+ mirrorrp = mirror_root.new_index(index)
+ inc_rpath = datadir.append_path('increments', index)
+ incs = Restore.get_inclist(inc_rpath)
+ mirror_time = Restore.get_mirror_time()
+ if Globals.parsable_output:
+ print Manage.describe_incs_parsable(incs, mirror_time, mirrorrp)
+ else: print Manage.describe_incs_human(incs, mirror_time, mirrorrp)
+
+
+def CalculateAverage(rps):
+ """Print out the average of the given statistics files"""
+ statobjs = map(lambda rp: StatsObj().read_stats_from_rp(rp), rps)
+ average_stats = StatsObj().set_to_average(statobjs)
+ print average_stats.get_stats_logstring(
+ "Average of %d stat files" % len(rps))
+
+
+def RemoveOlderThan(rootrp):
+ """Remove all increment files older than a certain time"""
+ datadir = rootrp.append("rdiff-backup-data")
+ if not datadir.lstat() or not datadir.isdir():
+ Log.FatalError("Unable to open rdiff-backup-data dir %s" %
+ (datadir.path,))
+
+ try: time = Time.genstrtotime(remove_older_than_string)
+ except TimeError, exc: Log.FatalError(str(exc))
+ timep = Time.timetopretty(time)
+ Log("Deleting increment(s) before %s" % timep, 4)
+
+ itimes = [Time.stringtopretty(inc.getinctime())
+ for inc in Restore.get_inclist(datadir.append("increments"))
+ if Time.stringtotime(inc.getinctime()) < time]
+
+ if not itimes:
+ Log.FatalError("No increments older than %s found" % timep)
+ inc_pretty_time = "\n".join(itimes)
+ if len(itimes) > 1 and not force:
+ Log.FatalError("Found %d relevant increments, dated:\n%s"
+ "\nIf you want to delete multiple increments in this way, "
+ "use the --force." % (len(itimes), inc_pretty_time))
+
+ Log("Deleting increment%sat times:\n%s" %
+ (len(itimes) == 1 and " " or "s ", inc_pretty_time), 3)
+ Manage.delete_earlier_than(datadir, time)
+
diff --git a/rdiff-backup/src/Make b/rdiff-backup/rdiff_backup/Make.old
index 2b79ffe..2b79ffe 100755
--- a/rdiff-backup/src/Make
+++ b/rdiff-backup/rdiff_backup/Make.old
diff --git a/rdiff-backup/rdiff_backup/MiscStats.py b/rdiff-backup/rdiff_backup/MiscStats.py
new file mode 100644
index 0000000..cd62dd6
--- /dev/null
+++ b/rdiff-backup/rdiff_backup/MiscStats.py
@@ -0,0 +1,72 @@
+from statistics import *
+
+"""Misc statistics methods, pertaining to dir and session stat files"""
+# This is the RPath of the directory statistics file, and the
+# associated open file. It will hold a line of statistics for
+# each directory that is backed up.
+_dir_stats_rp = None
+_dir_stats_fp = None
+
+# This goes at the beginning of the directory statistics file and
+# explains the format.
+_dir_stats_header = """# rdiff-backup directory statistics file
+#
+# Each line is in the following format:
+# RelativeDirName %s
+""" % " ".join(StatsObj.stat_file_attrs)
+
+def open_dir_stats_file():
+ """Open directory statistics file, write header"""
+ global _dir_stats_fp, _dir_stats_rp
+ assert not _dir_stats_fp, "Directory file already open"
+
+ if Globals.compression: suffix = "data.gz"
+ else: suffix = "data"
+ _dir_stats_rp = Inc.get_inc(Globals.rbdir.append("directory_statistics"),
+ Time.curtime, suffix)
+
+ if _dir_stats_rp.lstat():
+ Log("Warning, statistics file %s already exists, appending" %
+ _dir_stats_rp.path, 2)
+ _dir_stats_fp = _dir_stats_rp.open("ab", Globals.compression)
+ else: _dir_stats_fp = _dir_stats_rp.open("wb", Globals.compression)
+ _dir_stats_fp.write(_dir_stats_header)
+
+def write_dir_stats_line(statobj, index):
+ """Write info from statobj about rpath to statistics file"""
+ if Globals.null_separator:
+ _dir_stats_fp.write(statobj.get_stats_line(index, None) + "\0")
+ else: _dir_stats_fp.write(statobj.get_stats_line(index) + "\n")
+
+def close_dir_stats_file():
+ """Close directory statistics file if its open"""
+ global _dir_stats_fp
+ if _dir_stats_fp:
+ _dir_stats_fp.close()
+ _dir_stats_fp = None
+
+def write_session_statistics(statobj):
+ """Write session statistics into file, log"""
+ stat_inc = Inc.get_inc(Globals.rbdir.append("session_statistics"),
+ Time.curtime, "data")
+ statobj.StartTime = Time.curtime
+ statobj.EndTime = time.time()
+
+ # include hardlink data and dir stats in size of increments
+ if Globals.preserve_hardlinks and Hardlink.final_inc:
+ # include hardlink data in size of increments
+ statobj.IncrementFiles += 1
+ statobj.IncrementFileSize += Hardlink.final_inc.getsize()
+ if _dir_stats_rp and _dir_stats_rp.lstat():
+ statobj.IncrementFiles += 1
+ statobj.IncrementFileSize += _dir_stats_rp.getsize()
+
+ statobj.write_stats_to_rp(stat_inc)
+ if Globals.print_statistics:
+ message = statobj.get_stats_logstring("Session statistics")
+ Log.log_to_file(message)
+ Globals.client_conn.sys.stdout.write(message)
+
+
+from increment import *
+import Hardlink
diff --git a/rdiff-backup/rdiff_backup/Rdiff.py b/rdiff-backup/rdiff_backup/Rdiff.py
new file mode 100644
index 0000000..c9895cb
--- /dev/null
+++ b/rdiff-backup/rdiff_backup/Rdiff.py
@@ -0,0 +1,181 @@
+import os, popen2
+
+#######################################################################
+#
+# rdiff - Invoke rdiff utility to make signatures, deltas, or patch
+#
+# All these operations should be done in a relatively safe manner
+# using RobustAction and the like.
+
+class RdiffException(Exception): pass
+
+def get_signature(rp):
+ """Take signature of rpin file and return in file object"""
+ Log("Getting signature of %s" % rp.path, 7)
+ return rp.conn.Rdiff.Popen(['rdiff', 'signature', rp.path])
+
+def get_delta_sigfileobj(sig_fileobj, rp_new):
+ """Like get_delta but signature is in a file object"""
+ sig_tf = TempFileManager.new(rp_new, None)
+ sig_tf.write_from_fileobj(sig_fileobj)
+ rdiff_popen_obj = get_delta_sigrp(sig_tf, rp_new)
+ rdiff_popen_obj.set_thunk(sig_tf.delete)
+ return rdiff_popen_obj
+
+def get_delta_sigrp(rp_signature, rp_new):
+ """Take signature rp and new rp, return delta file object"""
+ assert rp_signature.conn is rp_new.conn
+ Log("Getting delta of %s with signature %s" %
+ (rp_new.path, rp_signature.path), 7)
+ return rp_new.conn.Rdiff.Popen(['rdiff', 'delta',
+ rp_signature.path, rp_new.path])
+
+def write_delta_action(basis, new, delta, compress = None):
+ """Return action writing delta which brings basis to new
+
+ If compress is true, the output of rdiff will be gzipped
+ before written to delta.
+
+ """
+ sig_tf = TempFileManager.new(new, None)
+ delta_tf = TempFileManager.new(delta)
+ def init(): write_delta(basis, new, delta_tf, compress, sig_tf)
+ return Robust.make_tf_robustaction(init, (sig_tf, delta_tf),
+ (None, delta))
+
+def write_delta(basis, new, delta, compress = None, sig_tf = None):
+ """Write rdiff delta which brings basis to new"""
+ Log("Writing delta %s from %s -> %s" %
+ (basis.path, new.path, delta.path), 7)
+ if not sig_tf: sig_tf = TempFileManager.new(new, None)
+ sig_tf.write_from_fileobj(get_signature(basis))
+ delta.write_from_fileobj(get_delta_sigrp(sig_tf, new), compress)
+ sig_tf.delete()
+
+def patch_action(rp_basis, rp_delta, rp_out = None,
+ out_tf = None, delta_compressed = None):
+ """Return RobustAction which patches rp_basis with rp_delta
+
+ If rp_out is None, put output in rp_basis. Will use TempFile
+ out_tf it is specified. If delta_compressed is true, the
+ delta file will be decompressed before processing with rdiff.
+
+ """
+ if not rp_out: rp_out = rp_basis
+ else: assert rp_out.conn is rp_basis.conn
+ if (delta_compressed or
+ not (isinstance(rp_delta, RPath) and isinstance(rp_basis, RPath)
+ and rp_basis.conn is rp_delta.conn)):
+ if delta_compressed:
+ assert isinstance(rp_delta, RPath)
+ return patch_fileobj_action(rp_basis, rp_delta.open('rb', 1),
+ rp_out, out_tf)
+ else: return patch_fileobj_action(rp_basis, rp_delta.open('rb'),
+ rp_out, out_tf)
+
+ # Files are uncompressed on same connection, run rdiff
+ if out_tf is None: out_tf = TempFileManager.new(rp_out)
+ def init():
+ Log("Patching %s using %s to %s via %s" %
+ (rp_basis.path, rp_delta.path, rp_out.path, out_tf.path), 7)
+ cmdlist = ["rdiff", "patch", rp_basis.path,
+ rp_delta.path, out_tf.path]
+ return_val = rp_basis.conn.os.spawnvp(os.P_WAIT, 'rdiff', cmdlist)
+ out_tf.setdata()
+ if return_val != 0 or not out_tf.lstat():
+ RdiffException("Error running %s" % cmdlist)
+ return Robust.make_tf_robustaction(init, (out_tf,), (rp_out,))
+
+def patch_fileobj_action(rp_basis, delta_fileobj, rp_out = None,
+ out_tf = None, delta_compressed = None):
+ """Like patch_action but diff is given in fileobj form
+
+ Nest a writing of a tempfile with the actual patching to
+ create a new action. We have to nest so that the tempfile
+ will be around until the patching finishes.
+
+ """
+ if not rp_out: rp_out = rp_basis
+ delta_tf = TempFileManager.new(rp_out, None)
+ def init(): delta_tf.write_from_fileobj(delta_fileobj)
+ def final(init_val): delta_tf.delete()
+ def error(exc, ran_init, init_val): delta_tf.delete()
+ write_delta_action = RobustAction(init, final, error)
+ return Robust.chain(write_delta_action, patch_action(rp_basis, delta_tf,
+ rp_out, out_tf))
+
+def patch_with_attribs_action(rp_basis, rp_delta, rp_out = None):
+ """Like patch_action, but also transfers attributs from rp_delta"""
+ if not rp_out: rp_out = rp_basis
+ tf = TempFileManager.new(rp_out)
+ return Robust.chain_nested(patch_action(rp_basis, rp_delta, rp_out, tf),
+ Robust.copy_attribs_action(rp_delta, tf))
+
+def copy_action(rpin, rpout):
+ """Use rdiff to copy rpin to rpout, conserving bandwidth"""
+ if not rpin.isreg() or not rpout.isreg() or rpin.conn is rpout.conn:
+ # rdiff not applicable, fallback to regular copying
+ return Robust.copy_action(rpin, rpout)
+
+ Log("Rdiff copying %s to %s" % (rpin.path, rpout.path), 6)
+ delta_tf = TempFileManager.new(rpout, None)
+ return Robust.chain(write_delta_action(rpout, rpin, delta_tf),
+ patch_action(rpout, delta_tf),
+ RobustAction(lambda: None, delta_tf.delete,
+ lambda exc: delta_tf.delete))
+
+
+class Popen:
+ """Spawn process and treat stdout as file object
+
+ Instead of using popen, which evaluates arguments with the shell
+ and thus may lead to security holes (thanks to Jamie Heilman for
+ this point), use the popen2 class and discard stdin.
+
+ When closed, this object checks to make sure the process exited
+ cleanly, and executes closing_thunk.
+
+ """
+ def __init__(self, cmdlist, closing_thunk = None):
+ """RdiffFilehook initializer
+
+ fileobj is the file we are emulating
+ thunk is called with no parameters right after the file is closed
+
+ """
+ assert type(cmdlist) is types.ListType
+ self.p3obj = popen2.Popen3(cmdlist)
+ self.fileobj = self.p3obj.fromchild
+ self.closing_thunk = closing_thunk
+ self.cmdlist = cmdlist
+
+ def set_thunk(self, closing_thunk):
+ """Set closing_thunk if not already"""
+ assert not self.closing_thunk
+ self.closing_thunk = closing_thunk
+
+ def read(self, length = -1): return self.fileobj.read(length)
+
+ def close(self):
+ closeval = self.fileobj.close()
+ if self.closing_thunk: self.closing_thunk()
+ exitval = self.p3obj.poll()
+ if exitval == 0: return closeval
+ elif exitval == 256:
+ Log("Failure probably because %s couldn't be found in PATH."
+ % self.cmdlist[0], 2)
+ assert 0, "rdiff not found"
+ elif exitval == -1:
+ # There may a race condition where a process closes
+ # but doesn't provide its exitval fast enough.
+ Log("Waiting for process to close", 8)
+ time.sleep(0.2)
+ exitval = self.p3obj.poll()
+ if exitval == 0: return closeval
+ raise RdiffException("%s exited with non-zero value %d" %
+ (self.cmdlist, exitval))
+
+
+from log import *
+from robust import *
+
diff --git a/rdiff-backup/rdiff_backup/SetConnections.py b/rdiff-backup/rdiff_backup/SetConnections.py
new file mode 100644
index 0000000..be3fdfd
--- /dev/null
+++ b/rdiff-backup/rdiff_backup/SetConnections.py
@@ -0,0 +1,219 @@
+#######################################################################
+#
+# setconnections - Parse initial arguments and establish connections
+#
+
+"""Parse args and setup connections
+
+The methods in this class are used once by Main to parse file
+descriptions like bescoto@folly.stanford.edu:/usr/bin/ls and to
+set up the related connections.
+
+"""
+
+class SetConnectionsException(Exception): pass
+
+
+# This is the schema that determines how rdiff-backup will open a
+# pipe to the remote system. If the file is given as A::B, %s will
+# be substituted with A in the schema.
+__cmd_schema = 'ssh -C %s rdiff-backup --server'
+__cmd_schema_no_compress = 'ssh %s rdiff-backup --server'
+
+# This is a list of remote commands used to start the connections.
+# The first is None because it is the local connection.
+__conn_remote_cmds = [None]
+
+def InitRPs(arglist, remote_schema = None, remote_cmd = None):
+ """Map the given file descriptions into rpaths and return list"""
+ global __cmd_schema
+ if remote_schema: __cmd_schema = remote_schema
+ elif not Globals.ssh_compression: __cmd_schema = __cmd_schema_no_compress
+
+ if not arglist: return []
+ desc_pairs = map(parse_file_desc, arglist)
+
+ if filter(lambda x: x[0], desc_pairs): # True if any host_info found
+ if remote_cmd:
+ Log.FatalError("The --remote-cmd flag is not compatible "
+ "with remote file descriptions.")
+ elif remote_schema:
+ Log("Remote schema option ignored - no remote file "
+ "descriptions.", 2)
+
+ cmd_pairs = map(desc2cmd_pairs, desc_pairs)
+ if remote_cmd: # last file description gets remote_cmd
+ cmd_pairs[-1] = (remote_cmd, cmd_pairs[-1][1])
+ return map(cmdpair2rp, cmd_pairs)
+
+def cmdpair2rp(cmd_pair):
+ """Return RPath from cmd_pair (remote_cmd, filename)"""
+ cmd, filename = cmd_pair
+ if cmd: conn = init_connection(cmd)
+ else: conn = Globals.local_connection
+ return RPath(conn, filename)
+
+def desc2cmd_pairs(desc_pair):
+ """Return pair (remote_cmd, filename) from desc_pair"""
+ host_info, filename = desc_pair
+ if not host_info: return (None, filename)
+ else: return (fill_schema(host_info), filename)
+
+def parse_file_desc(file_desc):
+ """Parse file description returning pair (host_info, filename)
+
+ In other words, bescoto@folly.stanford.edu::/usr/bin/ls =>
+ ("bescoto@folly.stanford.edu", "/usr/bin/ls"). The
+ complication is to allow for quoting of : by a \. If the
+ string is not separated by :, then the host_info is None.
+
+ """
+ def check_len(i):
+ if i >= len(file_desc):
+ raise SetConnectionsException(
+ "Unexpected end to file description %s" % file_desc)
+
+ host_info_list, i, last_was_quoted = [], 0, None
+ while 1:
+ if i == len(file_desc):
+ return (None, file_desc)
+
+ if file_desc[i] == '\\':
+ i = i+1
+ check_len(i)
+ last_was_quoted = 1
+ elif (file_desc[i] == ":" and i > 0 and file_desc[i-1] == ":"
+ and not last_was_quoted):
+ host_info_list.pop() # Remove last colon from name
+ break
+ else: last_was_quoted = None
+ host_info_list.append(file_desc[i])
+ i = i+1
+
+ check_len(i+1)
+ return ("".join(host_info_list), file_desc[i+1:])
+
+def fill_schema(host_info):
+ """Fills host_info into the schema and returns remote command"""
+ return __cmd_schema % host_info
+
+def init_connection(remote_cmd):
+ """Run remote_cmd, register connection, and then return it
+
+ If remote_cmd is None, then the local connection will be
+ returned. This also updates some settings on the remote side,
+ like global settings, its connection number, and verbosity.
+
+ """
+ if not remote_cmd: return Globals.local_connection
+
+ Log("Executing " + remote_cmd, 4)
+ stdin, stdout = os.popen2(remote_cmd)
+ conn_number = len(Globals.connections)
+ conn = PipeConnection(stdout, stdin, conn_number)
+
+ check_connection_version(conn, remote_cmd)
+ Log("Registering connection %d" % conn_number, 7)
+ init_connection_routing(conn, conn_number, remote_cmd)
+ init_connection_settings(conn)
+ return conn
+
+def check_connection_version(conn, remote_cmd):
+ """Log warning if connection has different version"""
+ try: remote_version = conn.Globals.get('version')
+ except ConnectionReadError, exception:
+ Log.FatalError("""%s
+
+Couldn't start up the remote connection by executing
+
+ %s
+
+Remember that, under the default settings, rdiff-backup must be
+installed in the PATH on the remote system. See the man page for more
+information.""" % (exception, remote_cmd))
+
+ if remote_version != Globals.version:
+ Log("Warning: Local version %s does not match remote version %s."
+ % (Globals.version, remote_version), 2)
+
+def init_connection_routing(conn, conn_number, remote_cmd):
+ """Called by init_connection, establish routing, conn dict"""
+ Globals.connection_dict[conn_number] = conn
+
+ conn.SetConnections.init_connection_remote(conn_number)
+ for other_remote_conn in Globals.connections[1:]:
+ conn.SetConnections.add_redirected_conn(
+ other_remote_conn.conn_number)
+ other_remote_conn.SetConnections.add_redirected_conn(conn_number)
+
+ Globals.connections.append(conn)
+ __conn_remote_cmds.append(remote_cmd)
+
+def init_connection_settings(conn):
+ """Tell new conn about log settings and updated globals"""
+ conn.Log.setverbosity(Log.verbosity)
+ conn.Log.setterm_verbosity(Log.term_verbosity)
+ for setting_name in Globals.changed_settings:
+ conn.Globals.set(setting_name, Globals.get(setting_name))
+
+def init_connection_remote(conn_number):
+ """Run on server side to tell self that have given conn_number"""
+ Globals.connection_number = conn_number
+ Globals.local_connection.conn_number = conn_number
+ Globals.connection_dict[0] = Globals.connections[1]
+ Globals.connection_dict[conn_number] = Globals.local_connection
+
+def add_redirected_conn(conn_number):
+ """Run on server side - tell about redirected connection"""
+ Globals.connection_dict[conn_number] = \
+ RedirectedConnection(conn_number)
+
+def UpdateGlobal(setting_name, val):
+ """Update value of global variable across all connections"""
+ for conn in Globals.connections:
+ conn.Globals.set(setting_name, val)
+
+def BackupInitConnections(reading_conn, writing_conn):
+ """Backup specific connection initialization"""
+ reading_conn.Globals.set("isbackup_reader", 1)
+ writing_conn.Globals.set("isbackup_writer", 1)
+ UpdateGlobal("backup_reader", reading_conn)
+ UpdateGlobal("backup_writer", writing_conn)
+
+def CloseConnections():
+ """Close all connections. Run by client"""
+ assert not Globals.server
+ for conn in Globals.connections: conn.quit()
+ del Globals.connections[1:] # Only leave local connection
+ Globals.connection_dict = {0: Globals.local_connection}
+ Globals.backup_reader = Globals.isbackup_reader = \
+ Globals.backup_writer = Globals.isbackup_writer = None
+
+def TestConnections():
+ """Test connections, printing results"""
+ if len(Globals.connections) == 1: print "No remote connections specified"
+ else:
+ for i in range(1, len(Globals.connections)): test_connection(i)
+
+def test_connection(conn_number):
+ """Test connection. conn_number 0 is the local connection"""
+ print "Testing server started by: ", __conn_remote_cmds[conn_number]
+ conn = Globals.connections[conn_number]
+ try:
+ assert conn.pow(2,3) == 8
+ assert conn.os.path.join("a", "b") == "a/b"
+ version = conn.reval("lambda: Globals.version")
+ except:
+ sys.stderr.write("Server tests failed\n")
+ raise
+ if not version == Globals.version:
+ print """Server may work, but there is a version mismatch:
+Local version: %s
+Remote version: %s""" % (Globals.version, version)
+ else: print "Server OK"
+
+
+from log import *
+from rpath import *
+from connection import *
+import Globals
diff --git a/rdiff-backup/rdiff_backup/Time.py b/rdiff-backup/rdiff_backup/Time.py
new file mode 100644
index 0000000..4eb2107
--- /dev/null
+++ b/rdiff-backup/rdiff_backup/Time.py
@@ -0,0 +1,199 @@
+import time, types, re
+import Globals
+
+#######################################################################
+#
+# ttime - Provide Time class, which contains time related functions.
+#
+
+class TimeException(Exception): pass
+
+_interval_conv_dict = {"s": 1, "m": 60, "h": 3600, "D": 86400,
+ "W": 7*86400, "M": 30*86400, "Y": 365*86400}
+_integer_regexp = re.compile("^[0-9]+$")
+_interval_regexp = re.compile("^([0-9]+)([smhDWMY])")
+_genstr_date_regexp1 = re.compile("^(?P<year>[0-9]{4})[-/]"
+ "(?P<month>[0-9]{1,2})[-/](?P<day>[0-9]{1,2})$")
+_genstr_date_regexp2 = re.compile("^(?P<month>[0-9]{1,2})[-/]"
+ "(?P<day>[0-9]{1,2})[-/](?P<year>[0-9]{4})$")
+curtime = curtimestr = None
+
+def setcurtime(curtime = None):
+ """Sets the current time in curtime and curtimestr on all systems"""
+ t = curtime or time.time()
+ for conn in Globals.connections:
+ conn.Time.setcurtime_local(t, timetostring(t))
+
+def setcurtime_local(timeinseconds, timestr):
+ """Only set the current time locally"""
+ global curtime, curtimestr
+ curtime, curtimestr = timeinseconds, timestr
+
+def setprevtime(timeinseconds):
+ """Sets the previous inc time in prevtime and prevtimestr"""
+ assert timeinseconds > 0, timeinseconds
+ timestr = timetostring(timeinseconds)
+ for conn in Globals.connections:
+ conn.Time.setprevtime_local(timeinseconds, timestr)
+
+def setprevtime_local(timeinseconds, timestr):
+ """Like setprevtime but only set the local version"""
+ global prevtime, prevtimestr
+ prevtime, prevtimestr = timeinseconds, timestr
+
+def timetostring(timeinseconds):
+ """Return w3 datetime compliant listing of timeinseconds"""
+ return time.strftime("%Y-%m-%dT%H" + Globals.time_separator +
+ "%M" + Globals.time_separator + "%S",
+ time.localtime(timeinseconds)) + gettzd()
+
+def stringtotime(timestring):
+ """Return time in seconds from w3 timestring
+
+ If there is an error parsing the string, or it doesn't look
+ like a w3 datetime string, return None.
+
+ """
+ try:
+ date, daytime = timestring[:19].split("T")
+ year, month, day = map(int, date.split("-"))
+ hour, minute, second = map(int,
+ daytime.split(Globals.time_separator))
+ assert 1900 < year < 2100, year
+ assert 1 <= month <= 12
+ assert 1 <= day <= 31
+ assert 0 <= hour <= 23
+ assert 0 <= minute <= 59
+ assert 0 <= second <= 61 # leap seconds
+ timetuple = (year, month, day, hour, minute, second, -1, -1, -1)
+ if time.daylight:
+ utc_in_secs = time.mktime(timetuple) - time.altzone
+ else: utc_in_secs = time.mktime(timetuple) - time.timezone
+
+ return long(utc_in_secs) + tzdtoseconds(timestring[19:])
+ except (TypeError, ValueError, AssertionError): return None
+
+def timetopretty(timeinseconds):
+ """Return pretty version of time"""
+ return time.asctime(time.localtime(timeinseconds))
+
+def stringtopretty(timestring):
+ """Return pretty version of time given w3 time string"""
+ return timetopretty(stringtotime(timestring))
+
+def inttopretty(seconds):
+ """Convert num of seconds to readable string like "2 hours"."""
+ partlist = []
+ hours, seconds = divmod(seconds, 3600)
+ if hours > 1: partlist.append("%d hours" % hours)
+ elif hours == 1: partlist.append("1 hour")
+
+ minutes, seconds = divmod(seconds, 60)
+ if minutes > 1: partlist.append("%d minutes" % minutes)
+ elif minutes == 1: partlist.append("1 minute")
+
+ if seconds == 1: partlist.append("1 second")
+ elif not partlist or seconds > 1:
+ if isinstance(seconds, int) or isinstance(seconds, long):
+ partlist.append("%s seconds" % seconds)
+ else: partlist.append("%.2f seconds" % seconds)
+ return " ".join(partlist)
+
+def intstringtoseconds(interval_string):
+ """Convert a string expressing an interval (e.g. "4D2s") to seconds"""
+ def error():
+ raise TimeException("""Bad interval string "%s"
+
+Intervals are specified like 2Y (2 years) or 2h30m (2.5 hours). The
+allowed special characters are s, m, h, D, W, M, and Y. See the man
+page for more information.
+""" % interval_string)
+ if len(interval_string) < 2: error()
+
+ total = 0
+ while interval_string:
+ match = _interval_regexp.match(interval_string)
+ if not match: error()
+ num, ext = int(match.group(1)), match.group(2)
+ if not ext in _interval_conv_dict or num < 0: error()
+ total += num*_interval_conv_dict[ext]
+ interval_string = interval_string[match.end(0):]
+ return total
+
+def gettzd():
+ """Return w3's timezone identification string.
+
+ Expresed as [+/-]hh:mm. For instance, PST is -08:00. Zone is
+ coincides with what localtime(), etc., use.
+
+ """
+ if time.daylight: offset = -1 * time.altzone/60
+ else: offset = -1 * time.timezone/60
+ if offset > 0: prefix = "+"
+ elif offset < 0: prefix = "-"
+ else: return "Z" # time is already in UTC
+
+ hours, minutes = map(abs, divmod(offset, 60))
+ assert 0 <= hours <= 23
+ assert 0 <= minutes <= 59
+ return "%s%02d%s%02d" % (prefix, hours,
+ Globals.time_separator, minutes)
+
+def tzdtoseconds(tzd):
+ """Given w3 compliant TZD, return how far ahead UTC is"""
+ if tzd == "Z": return 0
+ assert len(tzd) == 6 # only accept forms like +08:00 for now
+ assert (tzd[0] == "-" or tzd[0] == "+") and \
+ tzd[3] == Globals.time_separator
+ return -60 * (60 * int(tzd[:3]) + int(tzd[4:]))
+
+def cmp(time1, time2):
+ """Compare time1 and time2 and return -1, 0, or 1"""
+ if type(time1) is types.StringType:
+ time1 = stringtotime(time1)
+ assert time1 is not None
+ if type(time2) is types.StringType:
+ time2 = stringtotime(time2)
+ assert time2 is not None
+
+ if time1 < time2: return -1
+ elif time1 == time2: return 0
+ else: return 1
+
+def genstrtotime(timestr, curtime = None):
+ """Convert a generic time string to a time in seconds"""
+ if curtime is None: curtime = globals()['curtime']
+ if timestr == "now": return curtime
+
+ def error():
+ raise TimeException("""Bad time string "%s"
+
+The acceptible time strings are intervals (like "3D64s"), w3-datetime
+strings, like "2002-04-26T04:22:01-07:00" (strings like
+"2002-04-26T04:22:01" are also acceptable - rdiff-backup will use the
+current time zone), or ordinary dates like 2/4/1997 or 2001-04-23
+(various combinations are acceptable, but the month always precedes
+the day).""" % timestr)
+
+ # Test for straight integer
+ if _integer_regexp.search(timestr): return int(timestr)
+
+ # Test for w3-datetime format, possibly missing tzd
+ t = stringtotime(timestr) or stringtotime(timestr+gettzd())
+ if t: return t
+
+ try: # test for an interval, like "2 days ago"
+ return curtime - intstringtoseconds(timestr)
+ except TimeException: pass
+
+ # Now check for dates like 2001/3/23
+ match = _genstr_date_regexp1.search(timestr) or \
+ _genstr_date_regexp2.search(timestr)
+ if not match: error()
+ timestr = "%s-%02d-%02dT00:00:00%s" % (match.group('year'),
+ int(match.group('month')), int(match.group('day')), gettzd())
+ t = stringtotime(timestr)
+ if t: return t
+ else: error()
+
+
diff --git a/rdiff-backup/rdiff_backup/connection.py b/rdiff-backup/rdiff_backup/connection.py
index deff577..74d413d 100644
--- a/rdiff-backup/rdiff_backup/connection.py
+++ b/rdiff-backup/rdiff_backup/connection.py
@@ -1,5 +1,4 @@
from __future__ import generators
-execfile("rdiff.py")
import types, os, tempfile, cPickle, shutil, traceback
#######################################################################
@@ -38,10 +37,9 @@ class LocalConnection(Connection):
self.conn_number = 0 # changed by SetConnections for server
def __getattr__(self, name):
- try: return globals()[name]
- except KeyError:
- try: return __builtins__.__dict__[name]
- except KeyError: raise NameError, name
+ if name in globals(): return globals()[name]
+ elif isinstance(__builtins__, dict): return __builtins__[name]
+ else: return __builtins__.__dict__[name]
def __setattr__(self, name, value):
globals()[name] = value
@@ -56,11 +54,6 @@ class LocalConnection(Connection):
def quit(self): pass
-Globals.local_connection = LocalConnection()
-Globals.connections.append(Globals.local_connection)
-# Following changed by server in SetConnections
-Globals.connection_dict[0] = Globals.local_connection
-
class ConnectionRequest:
"""Simple wrapper around a PipeConnection request"""
@@ -493,3 +486,30 @@ class VirtualFile:
line = self.readline()
if not line: break
yield line
+
+
+# everything has to be available here for remote connection's use, but
+# put at bottom to reduce circularities.
+import Globals, Time, Rdiff, Hardlink, FilenameMapping
+from static import *
+from lazy import *
+from log import *
+from iterfile import *
+from connection import *
+from rpath import *
+from robust import *
+from rorpiter import *
+from destructive_stepping import *
+from selection import *
+from statistics import *
+from increment import *
+from restore import *
+from manage import *
+from highlevel import *
+
+
+Globals.local_connection = LocalConnection()
+Globals.connections.append(Globals.local_connection)
+# Following changed by server in SetConnections
+Globals.connection_dict[0] = Globals.local_connection
+
diff --git a/rdiff-backup/rdiff_backup/destructive_stepping.py b/rdiff-backup/rdiff_backup/destructive_stepping.py
index 7dfde11..a64ecbc 100644
--- a/rdiff-backup/rdiff_backup/destructive_stepping.py
+++ b/rdiff-backup/rdiff_backup/destructive_stepping.py
@@ -1,6 +1,7 @@
from __future__ import generators
import types
-execfile("rorpiter.py")
+from rpath import *
+from lazy import *
#######################################################################
#
@@ -206,3 +207,6 @@ class DestructiveSteppingFinalizer(ErrorITR):
if self.dsrpath: self.dsrpath.write_changes()
+from log import *
+from robust import *
+import Globals
diff --git a/rdiff-backup/rdiff_backup/highlevel.py b/rdiff-backup/rdiff_backup/highlevel.py
index 8c95a1f..0b477d8 100644
--- a/rdiff-backup/rdiff_backup/highlevel.py
+++ b/rdiff-backup/rdiff_backup/highlevel.py
@@ -1,5 +1,12 @@
from __future__ import generators
-execfile("manage.py")
+from static import *
+from log import *
+from rpath import *
+from robust import *
+from increment import *
+from destructive_stepping import *
+from rorpiter import *
+import Globals, Hardlink, MiscStats
#######################################################################
#
@@ -248,7 +255,7 @@ class HLDestinationStruct:
"""Apply diffs and finalize, with checkpointing and statistics"""
collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath)
- Stats.open_dir_stats_file()
+ MiscStats.open_dir_stats_file()
dsrp, finished_dsrp = None, None
try:
@@ -266,15 +273,15 @@ class HLDestinationStruct:
except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
if Globals.preserve_hardlinks: Hardlink.final_writedata()
- Stats.close_dir_stats_file()
- Stats.write_session_statistics(ITR)
+ MiscStats.close_dir_stats_file()
+ MiscStats.write_session_statistics(ITR)
SaveState.checkpoint_remove()
def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath):
"""Apply diffs, write increment if necessary, and finalize"""
collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath)
- Stats.open_dir_stats_file()
+ MiscStats.open_dir_stats_file()
dsrp, finished_dsrp = None, None
try:
@@ -293,8 +300,8 @@ class HLDestinationStruct:
except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
if Globals.preserve_hardlinks: Hardlink.final_writedata()
- Stats.close_dir_stats_file()
- Stats.write_session_statistics(ITR)
+ MiscStats.close_dir_stats_file()
+ MiscStats.write_session_statistics(ITR)
SaveState.checkpoint_remove()
def handle_last_error(cls, dsrp, finalizer, ITR):
diff --git a/rdiff-backup/rdiff_backup/increment.py b/rdiff-backup/rdiff_backup/increment.py
index b03b464..d5543a0 100644
--- a/rdiff-backup/rdiff_backup/increment.py
+++ b/rdiff-backup/rdiff_backup/increment.py
@@ -1,5 +1,7 @@
import traceback
-execfile("statistics.py")
+from static import *
+from statistics import *
+from lazy import *
#######################################################################
#
@@ -256,7 +258,7 @@ class IncrementITR(ErrorITR, StatsITR):
self.end_stats(diff_rorp, dsrp, self.incrp)
if self.mirror_isdirectory or dsrp.isdir():
- Stats.write_dir_stats_line(self, dsrp.index)
+ MiscStats.write_dir_stats_line(self, dsrp.index)
def branch_process(self, subinstance):
"""Update statistics, and the has_changed flag if change in branch"""
@@ -286,8 +288,15 @@ class MirrorITR(ErrorITR, StatsITR):
"""Update statistics when leaving"""
self.end_stats(self.diff_rorp, self.mirror_dsrp)
if self.mirror_dsrp.isdir():
- Stats.write_dir_stats_line(self, self.mirror_dsrp.index)
+ MiscStats.write_dir_stats_line(self, self.mirror_dsrp.index)
def branch_process(self, subinstance):
"""Update statistics with subdirectory results"""
self.add_file_stats(subinstance)
+
+
+from log import *
+from rpath import *
+from robust import *
+from rorpiter import *
+import Globals, Time, MiscStats
diff --git a/rdiff-backup/rdiff_backup/iterfile.py b/rdiff-backup/rdiff_backup/iterfile.py
index 21629b2..26cc952 100644
--- a/rdiff-backup/rdiff_backup/iterfile.py
+++ b/rdiff-backup/rdiff_backup/iterfile.py
@@ -1,5 +1,5 @@
-execfile("ttime.py")
import cPickle
+import Globals
#######################################################################
#
diff --git a/rdiff-backup/rdiff_backup/lazy.py b/rdiff-backup/rdiff_backup/lazy.py
index 425a9c4..1eb0211 100644
--- a/rdiff-backup/rdiff_backup/lazy.py
+++ b/rdiff-backup/rdiff_backup/lazy.py
@@ -1,6 +1,6 @@
from __future__ import generators
-execfile("static.py")
import os, stat, types
+from static import *
#######################################################################
#
@@ -324,3 +324,7 @@ class ErrorITR(IterTreeReducer):
Log("Error '%s' processing %s" % (exc, filename), 2)
+# Put at bottom to prevent (viciously) circular module dependencies
+from robust import *
+from log import *
+
diff --git a/rdiff-backup/rdiff_backup/log.py b/rdiff-backup/rdiff_backup/log.py
index 6b4ba06..aade607 100644
--- a/rdiff-backup/rdiff_backup/log.py
+++ b/rdiff-backup/rdiff_backup/log.py
@@ -1,5 +1,4 @@
-import time, sys, traceback
-execfile("lazy.py")
+import time, sys, traceback, types
#######################################################################
#
@@ -132,7 +131,7 @@ class Logger:
def FatalError(self, message):
self("Fatal Error: " + message, 1)
- Globals.Main.cleanup()
+ Main.cleanup()
sys.exit(1)
def exception_to_string(self):
@@ -158,3 +157,4 @@ class Logger:
logging_func(self.exception_to_string(), verbosity)
Log = Logger()
+import Globals, Main
diff --git a/rdiff-backup/rdiff_backup/manage.py b/rdiff-backup/rdiff_backup/manage.py
index 0c08872..4dad8b1 100644
--- a/rdiff-backup/rdiff_backup/manage.py
+++ b/rdiff-backup/rdiff_backup/manage.py
@@ -1,4 +1,7 @@
-execfile("restore.py")
+from __future__ import generators
+from static import *
+from log import *
+import Globals, Time
#######################################################################
#
diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py
index 30820b8..5202854 100644
--- a/rdiff-backup/rdiff_backup/restore.py
+++ b/rdiff-backup/rdiff_backup/restore.py
@@ -1,6 +1,6 @@
from __future__ import generators
-execfile("increment.py")
import tempfile
+from static import *
#######################################################################
#
@@ -362,3 +362,10 @@ class RestoreCombinedData:
else: RPath.copy(inc, target)
else: raise RestoreError("Unknown inctype %s" % inctype)
RPath.copy_attribs(inc, target)
+
+
+from log import *
+from destructive_stepping import *
+from rpath import *
+from rorpiter import *
+import Globals, Time, Rdiff, Hardlink, FilenameMapping, SetConnections
diff --git a/rdiff-backup/rdiff_backup/robust.py b/rdiff-backup/rdiff_backup/robust.py
index e539827..3c9851c 100644
--- a/rdiff-backup/rdiff_backup/robust.py
+++ b/rdiff-backup/rdiff_backup/robust.py
@@ -1,5 +1,5 @@
-import tempfile, errno, signal
-execfile("hardlink.py")
+import tempfile, errno, signal, cPickle
+from static import *
#######################################################################
#
@@ -243,7 +243,7 @@ class Robust:
"""
try: return function(*args)
except (EnvironmentError, SkipFileException, DSRPPermError,
- RPathException, RdiffException), exc:
+ RPathException, Rdiff.RdiffException), exc:
TracebackArchive.add()
if (not isinstance(exc, EnvironmentError) or
(errno.errorcode[exc[0]] in
@@ -356,6 +356,8 @@ class TempFileManager:
MakeClass(TempFileManager)
+from rpath import *
+
class TempFile(RPath):
"""Like an RPath, but keep track of which ones are still here"""
def rename(self, rp_dest):
@@ -642,3 +644,9 @@ class ResumeSessionInfo:
self.last_index = last_index
self.last_definitive = last_definitive
self.ITR, self.finalizer, = ITR, finalizer
+
+
+from log import *
+from destructive_stepping import *
+import Time, Rdiff
+from highlevel import *
diff --git a/rdiff-backup/rdiff_backup/rorpiter.py b/rdiff-backup/rdiff_backup/rorpiter.py
index efa0303..03705aa 100644
--- a/rdiff-backup/rdiff_backup/rorpiter.py
+++ b/rdiff-backup/rdiff_backup/rorpiter.py
@@ -1,6 +1,11 @@
-execfile("robust.py")
from __future__ import generators
-import tempfile, UserList
+import tempfile, UserList, types
+from static import *
+from log import *
+from rpath import *
+from robust import *
+from iterfile import *
+import Globals, Rdiff, Hardlink
#######################################################################
#
diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py
index c6bcca6..73910be 100644
--- a/rdiff-backup/rdiff_backup/rpath.py
+++ b/rdiff-backup/rdiff_backup/rpath.py
@@ -1,5 +1,5 @@
-execfile("connection.py")
import os, stat, re, sys, shutil, gzip
+from static import *
#######################################################################
#
@@ -778,6 +778,10 @@ class RPathFileHook:
self.closing_thunk()
return result
+# Import these late to avoid circular dependencies
+from lazy import *
+from selection import *
+from destructive_stepping import *
class RpathDeleter(IterTreeReducer):
"""Delete a directory. Called by RPath.delete()"""
diff --git a/rdiff-backup/rdiff_backup/selection.py b/rdiff-backup/rdiff_backup/selection.py
index 3d1f0e2..4fee9ee 100644
--- a/rdiff-backup/rdiff_backup/selection.py
+++ b/rdiff-backup/rdiff_backup/selection.py
@@ -1,6 +1,9 @@
from __future__ import generators
-execfile("destructive_stepping.py")
import re
+from log import *
+from robust import *
+from destructive_stepping import *
+
#######################################################################
#
@@ -521,3 +524,4 @@ probably isn't what you meant.""" %
else: res = res + re.escape(c)
return res
+
diff --git a/rdiff-backup/rdiff_backup/static.py b/rdiff-backup/rdiff_backup/static.py
index 2e97cd0..0355f44 100644
--- a/rdiff-backup/rdiff_backup/static.py
+++ b/rdiff-backup/rdiff_backup/static.py
@@ -1,5 +1,3 @@
-execfile("globals.py")
-
#######################################################################
#
# static - MakeStatic and MakeClass
diff --git a/rdiff-backup/rdiff_backup/statistics.py b/rdiff-backup/rdiff_backup/statistics.py
index a91a681..16dd881 100644
--- a/rdiff-backup/rdiff_backup/statistics.py
+++ b/rdiff-backup/rdiff_backup/statistics.py
@@ -1,4 +1,4 @@
-execfile("filename_mapping.py")
+from lazy import *
#######################################################################
#
@@ -277,73 +277,7 @@ class StatsITR(IterTreeReducer, StatsObj):
self.__dict__[attr] += subinstance.__dict__[attr]
-class Stats:
- """Misc statistics methods, pertaining to dir and session stat files"""
- # This is the RPath of the directory statistics file, and the
- # associated open file. It will hold a line of statistics for
- # each directory that is backed up.
- _dir_stats_rp = None
- _dir_stats_fp = None
-
- # This goes at the beginning of the directory statistics file and
- # explains the format.
- _dir_stats_header = """# rdiff-backup directory statistics file
-#
-# Each line is in the following format:
-# RelativeDirName %s
-""" % " ".join(StatsObj.stat_file_attrs)
-
- def open_dir_stats_file(cls):
- """Open directory statistics file, write header"""
- assert not cls._dir_stats_fp, "Directory file already open"
-
- if Globals.compression: suffix = "data.gz"
- else: suffix = "data"
- cls._dir_stats_rp = Inc.get_inc(Globals.rbdir.append(
- "directory_statistics"), Time.curtime, suffix)
-
- if cls._dir_stats_rp.lstat():
- Log("Warning, statistics file %s already exists, appending" %
- cls._dir_stats_rp.path, 2)
- cls._dir_stats_fp = cls._dir_stats_rp.open("ab",
- Globals.compression)
- else: cls._dir_stats_fp = \
- cls._dir_stats_rp.open("wb", Globals.compression)
- cls._dir_stats_fp.write(cls._dir_stats_header)
-
- def write_dir_stats_line(cls, statobj, index):
- """Write info from statobj about rpath to statistics file"""
- if Globals.null_separator:
- cls._dir_stats_fp.write(statobj.get_stats_line(index, None) + "\0")
- else: cls._dir_stats_fp.write(statobj.get_stats_line(index) + "\n")
-
- def close_dir_stats_file(cls):
- """Close directory statistics file if its open"""
- if cls._dir_stats_fp:
- cls._dir_stats_fp.close()
- cls._dir_stats_fp = None
-
- def write_session_statistics(cls, statobj):
- """Write session statistics into file, log"""
- stat_inc = Inc.get_inc(Globals.rbdir.append("session_statistics"),
- Time.curtime, "data")
- statobj.StartTime = Time.curtime
- statobj.EndTime = time.time()
-
- # include hardlink data and dir stats in size of increments
- if Globals.preserve_hardlinks and Hardlink.final_inc:
- # include hardlink data in size of increments
- statobj.IncrementFiles += 1
- statobj.IncrementFileSize += Hardlink.final_inc.getsize()
- if cls._dir_stats_rp and cls._dir_stats_rp.lstat():
- statobj.IncrementFiles += 1
- statobj.IncrementFileSize += cls._dir_stats_rp.getsize()
-
- statobj.write_stats_to_rp(stat_inc)
- if Globals.print_statistics:
- message = statobj.get_stats_logstring("Session statistics")
- Log.log_to_file(message)
- Globals.client_conn.sys.stdout.write(message)
-
-MakeClass(Stats)
-
+from log import *
+from increment import *
+from robust import *
+import Globals
diff --git a/rdiff-backup/src/FilenameMapping.py b/rdiff-backup/src/FilenameMapping.py
new file mode 100644
index 0000000..104519d
--- /dev/null
+++ b/rdiff-backup/src/FilenameMapping.py
@@ -0,0 +1,94 @@
+import re
+from log import *
+import Globals
+
+#######################################################################
+#
+# filename_mapping - used to coordinate related filenames
+#
+# For instance, some source filenames may contain characters not
+# allowed on the mirror end. Also, if a source filename is very long
+# (say 240 characters), the extra characters added to related
+# increments may put them over the usual 255 character limit.
+#
+
+"""Contains class methods which coordinate related filenames"""
+max_filename_length = 255
+
+# If true, enable character quoting, and set characters making
+# regex-style range.
+chars_to_quote = None
+
+# These compiled regular expressions are used in quoting and unquoting
+chars_to_quote_regexp = None
+unquoting_regexp = None
+
+# Use given char to quote. Default is set in Globals.
+quoting_char = None
+
+
+def set_init_quote_vals():
+ """Set quoting value from Globals on all conns"""
+ for conn in Globals.connections:
+ conn.FilenameMapping.set_init_quote_vals_local()
+
+def set_init_quote_vals_local():
+ """Set value on local connection, initialize regexps"""
+ global chars_to_quote
+ chars_to_quote = Globals.chars_to_quote
+ if len(Globals.quoting_char) != 1:
+ Log.FatalError("Expected single character for quoting char,"
+ "got '%s' instead" % (Globals.quoting_char,))
+ quoting_char = Globals.quoting_char
+ init_quoting_regexps()
+
+def init_quoting_regexps():
+ """Compile quoting regular expressions"""
+ global chars_to_quote_regexp, unquoting_regexp
+ try:
+ chars_to_quote_regexp = \
+ re.compile("[%s%s]" % (chars_to_quote, quoting_char), re.S)
+ unquoting_regexp = re.compile("%s[0-9]{3}" % quoting_char, re.S)
+ except re.error:
+ Log.FatalError("Error '%s' when processing char quote list %s" %
+ (re.error, chars_to_quote))
+
+def quote(path):
+ """Return quoted version of given path
+
+ Any characters quoted will be replaced by the quoting char and
+ the ascii number of the character. For instance, "10:11:12"
+ would go to "10;05811;05812" if ":" were quoted and ";" were
+ the quoting character.
+
+ """
+ return chars_to_quote_regexp.sub(quote_single, path)
+
+def quote_single(match):
+ """Return replacement for a single character"""
+ return "%s%03d" % (quoting_char, ord(match.group()))
+
+def unquote(path):
+ """Return original version of quoted filename"""
+ return unquoting_regexp.sub(unquote_single, path)
+
+def unquote_single(match):
+ """Unquote a single quoted character"""
+ assert len(match.group()) == 4
+ return chr(int(match.group()[1:]))
+
+def get_quoted_dir_children(rpath):
+ """For rpath directory, return list of quoted children in dir"""
+ if not rpath.isdir(): return []
+ dir_pairs = [(unquote(filename), filename)
+ for filename in Robust.listrp(rpath)]
+ dir_pairs.sort() # sort by real index, not quoted part
+ child_list = []
+ for unquoted, filename in dir_pairs:
+ childrp = rpath.append(unquoted)
+ childrp.quote_path()
+ child_list.append(childrp)
+ return child_list
+
+
+
diff --git a/rdiff-backup/src/Globals.py b/rdiff-backup/src/Globals.py
new file mode 100644
index 0000000..ca6e8d1
--- /dev/null
+++ b/rdiff-backup/src/Globals.py
@@ -0,0 +1,226 @@
+import re, os
+
+# The current version of rdiff-backup
+version = "0.8.0"
+
+# If this is set, use this value in seconds as the current time
+# instead of reading it from the clock.
+current_time = None
+
+# This determines how many bytes to read at a time when copying
+blocksize = 32768
+
+# This is used by the BufferedRead class to determine how many
+# bytes to request from the underlying file per read(). Larger
+# values may save on connection overhead and latency.
+conn_bufsize = 98304
+
+# True if script is running as a server
+server = None
+
+# uid and gid of the owner of the rdiff-backup process. This can
+# vary depending on the connection.
+process_uid = os.getuid()
+process_gid = os.getgid()
+
+# If true, when copying attributes, also change target's uid/gid
+change_ownership = None
+
+# If true, change the permissions of unwriteable mirror files
+# (such as directories) so that they can be written, and then
+# change them back. This defaults to 1 just in case the process
+# is not running as root (root doesn't need to change
+# permissions).
+change_mirror_perms = (process_uid != 0)
+
+# If true, temporarily change permissions of unreadable files in
+# the source directory to make sure we can read all files.
+change_source_perms = None
+
+# If true, try to reset the atimes of the source partition.
+preserve_atime = None
+
+# This will be set as soon as the LocalConnection class loads
+local_connection = None
+
+# All connections should be added to the following list, so
+# further global changes can be propagated to the remote systems.
+# The first element should be Globals.local_connection. For a
+# server, the second is the connection to the client.
+connections = []
+
+# Each process should have a connection number unique to the
+# session. The client has connection number 0.
+connection_number = 0
+
+# Dictionary pairing connection numbers with connections. Set in
+# SetConnections for all connections.
+connection_dict = {}
+
+# True if the script is the end that reads the source directory
+# for backups. It is true for purely local sessions.
+isbackup_reader = None
+
+# Connection of the real backup reader (for which isbackup_reader
+# is true)
+backup_reader = None
+
+# True if the script is the end that writes to the increment and
+# mirror directories. True for purely local sessions.
+isbackup_writer = None
+
+# Connection of the backup writer
+backup_writer = None
+
+# True if this process is the client invoked by the user
+isclient = None
+
+# Connection of the client
+client_conn = None
+
+# This list is used by the set function below. When a new
+# connection is created with init_connection, its Globals class
+# will match this one for all the variables mentioned in this
+# list.
+changed_settings = []
+
+# rdiff-backup will try to checkpoint its state every
+# checkpoint_interval seconds. Then when resuming, at most this
+# amount of time is lost.
+checkpoint_interval = 20
+
+# The RPath of the rdiff-backup-data directory.
+rbdir = None
+
+# Indicates if a resume or a lack of resume is forced. This
+# should be None for the default. 0 means don't resume, and 1
+# means resume.
+resume = None
+
+# If there has been an aborted backup fewer than this many seconds
+# ago, attempt to resume it where it left off instead of starting
+# a new one.
+resume_window = 7200
+
+# This string is used when recognizing and creating time strings.
+# If the time_separator is ":", then W3 datetime strings like
+# 2001-12-07T04:22:01-07:00 are produced. It can be set to "_" to
+# make filenames that don't contain colons, which aren't allowed
+# under MS windows NT.
+time_separator = ":"
+
+# quoting_enabled is true if we should quote certain characters in
+# filenames on the source side (see FilenameMapping for more
+# info). chars_to_quote is a string whose characters should be
+# quoted, and quoting_char is the character to quote with.
+quoting_enabled = None
+chars_to_quote = ""
+quoting_char = ';'
+
+# If true, emit output intended to be easily readable by a
+# computer. False means output is intended for humans.
+parsable_output = None
+
+# If true, then hardlinks will be preserved to mirror and recorded
+# in the increments directory. There is also a difference here
+# between None and 0. When restoring, None or 1 means to preserve
+# hardlinks iff can find a hardlink dictionary. 0 means ignore
+# hardlink information regardless.
+preserve_hardlinks = 1
+
+# If this is false, then rdiff-backup will not compress any
+# increments. Default is to compress based on regexp below.
+compression = 1
+
+# Increments based on files whose names match this
+# case-insensitive regular expression won't be compressed (applies
+# to .snapshots and .diffs). The second below will be the
+# compiled version of the first.
+no_compression_regexp_string = "(?i).*\\.(gz|z|bz|bz2|tgz|zip|rpm|deb|" \
+ "jpg|gif|png|jp2|mp3|ogg|avi|wmv|mpeg|mpg|rm|mov)$"
+no_compression_regexp = None
+
+# If true, filelists and directory statistics will be split on
+# nulls instead of newlines.
+null_separator = None
+
+# Determines whether or not ssh will be run with the -C switch
+ssh_compression = 1
+
+# If true, print statistics after successful backup
+print_statistics = None
+
+# On the reader and writer connections, the following will be
+# replaced by the source and mirror Select objects respectively.
+select_source, select_mirror = None, None
+
+# On the backup writer connection, holds the main incrementing
+# function. Access is provided to increment error counts.
+ITR = None
+
+def get(name):
+ """Return the value of something in this module"""
+ return globals()[name]
+
+def is_not_None(name):
+ """Returns true if value is not None"""
+ return globals()[name] is not None
+
+def set(name, val):
+ """Set the value of something in this module
+
+ Use this instead of writing the values directly if the setting
+ matters to remote sides. This function updates the
+ changed_settings list, so other connections know to copy the
+ changes.
+
+ """
+ changed_settings.append(name)
+ globals()[name] = val
+
+def set_integer(name, val):
+ """Like set, but make sure val is an integer"""
+ try: intval = int(val)
+ except ValueError:
+ Log.FatalError("Variable %s must be set to an integer -\n"
+ "received %s instead." % (name, val))
+ set(name, intval)
+
+def get_dict_val(name, key):
+ """Return val from dictionary in this class"""
+ return globals()[name][key]
+
+def set_dict_val(name, key, val):
+ """Set value for dictionary in this class"""
+ globals()[name][key] = val
+
+def postset_regexp(name, re_string, flags = None):
+ """Compile re_string on all existing connections, set to name"""
+ for conn in connections:
+ conn.Globals.postset_regexp_local(name, re_string, flags)
+
+def postset_regexp_local(name, re_string, flags):
+ """Set name to compiled re_string locally"""
+ if flags: globals()[name] = re.compile(re_string, flags)
+ else: globals()[name] = re.compile(re_string)
+
+def set_select(dsrpath, tuplelist, quote_mode, *filelists):
+ """Initialize select object using tuplelist
+
+ Note that each list in filelists must each be passed as
+ separate arguments, so each is recognized as a file by the
+ connection. Otherwise we will get an error because a list
+ containing files can't be pickled.
+
+ """
+ global select_source, select_mirror
+ if dsrpath.source:
+ select_source = Select(dsrpath, quote_mode)
+ select_source.ParseArgs(tuplelist, filelists)
+ else:
+ select_mirror = Select(dsrpath, quote_mode)
+ select_mirror.ParseArgs(tuplelist, filelists)
+
+
+from rpath import * # kludge to avoid circularity - not needed in this module
+from selection import *
diff --git a/rdiff-backup/src/Hardlink.py b/rdiff-backup/src/Hardlink.py
new file mode 100644
index 0000000..9389b6f
--- /dev/null
+++ b/rdiff-backup/src/Hardlink.py
@@ -0,0 +1,262 @@
+from __future__ import generators
+import cPickle
+
+#######################################################################
+#
+# hardlink - code for preserving and restoring hardlinks
+#
+# If the preserve_hardlinks option is selected, linked files in the
+# source directory will be linked in the mirror directory. Linked
+# files are treated like any other with respect to incrementing, but a
+# database of all links will be recorded at each session, so linked
+# files can still be restored from the increments.
+#
+
+"""Hardlink class methods and data
+
+All these functions are meant to be executed on the destination
+side. The source side should only transmit inode information.
+
+"""
+
+# In all of these lists of indicies are the values. The keys in
+# _inode_ ones are (inode, devloc) pairs.
+_src_inode_indicies = {}
+_dest_inode_indicies = {}
+
+# The keys for these two are just indicies. They share values
+# with the earlier dictionaries.
+_src_index_indicies = {}
+_dest_index_indicies = {}
+
+# When a linked file is restored, its path is added to this dict,
+# so it can be found when later paths being restored are linked to
+# it.
+_restore_index_path = {}
+
+def get_inode_key(rorp):
+ """Return rorp's key for _inode_ dictionaries"""
+ return (rorp.getinode(), rorp.getdevloc())
+
+def get_indicies(rorp, source):
+ """Return a list of similarly linked indicies, using rorp's index"""
+ if source: dict = _src_index_indicies
+ else: dict = _dest_index_indicies
+ try: return dict[rorp.index]
+ except KeyError: return []
+
+def add_rorp(rorp, source):
+ """Process new rorp and update hard link dictionaries
+
+ First enter it into src_inode_indicies. If we have already
+ seen all the hard links, then we can delete the entry.
+ Everything must stay recorded in src_index_indicies though.
+
+ """
+ if not rorp.isreg() or rorp.getnumlinks() < 2: return
+
+ if source:
+ inode_dict, index_dict = _src_inode_indicies, _src_index_indicies
+ else: inode_dict, index_dict = _dest_inode_indicies, _dest_index_indicies
+
+ rp_inode_key = get_inode_key(rorp)
+ if inode_dict.has_key(rp_inode_key):
+ index_list = inode_dict[rp_inode_key]
+ index_list.append(rorp.index)
+ if len(index_list) == rorp.getnumlinks():
+ del inode_dict[rp_inode_key]
+ else: # make new entry in both src dicts
+ index_list = [rorp.index]
+ inode_dict[rp_inode_key] = index_list
+ index_dict[rorp.index] = index_list
+
+def add_rorp_iter(iter, source):
+ """Return new rorp iterator like iter that add_rorp's first"""
+ for rorp in iter:
+ add_rorp(rorp, source)
+ yield rorp
+
+def rorp_eq(src_rorp, dest_rorp):
+ """Compare hardlinked for equality
+
+ Two files may otherwise seem equal but be hardlinked in
+ different ways. This function considers them equal enough if
+ they have been hardlinked correctly to the previously seen
+ indicies.
+
+ """
+ assert src_rorp.index == dest_rorp.index
+ if (not src_rorp.isreg() or not dest_rorp.isreg() or
+ src_rorp.getnumlinks() == dest_rorp.getnumlinks() == 1):
+ return 1 # Hard links don't apply
+
+ src_index_list = get_indicies(src_rorp, 1)
+ dest_index_list = get_indicies(dest_rorp, None)
+
+ # If a list only has one element, then it is only hardlinked
+ # to itself so far, so that is not a genuine difference yet.
+ if not src_index_list or len(src_index_list) == 1:
+ return not dest_index_list or len(dest_index_list) == 1
+ if not dest_index_list or len(dest_index_list) == 1: return None
+
+ # Both index lists exist and are non-empty
+ return src_index_list == dest_index_list # they are always sorted
+
+def islinked(rorp):
+ """True if rorp's index is already linked to something on src side"""
+ return len(get_indicies(rorp, 1)) >= 2
+
+def restore_link(index, rpath):
+ """Restores a linked file by linking it
+
+ When restoring, all the hardlink data is already present, and
+ we can only link to something already written. In either
+ case, add to the _restore_index_path dict, so we know later
+ that the file is available for hard
+ linking.
+
+ Returns true if succeeded in creating rpath, false if must
+ restore rpath normally.
+
+ """
+ if index not in _src_index_indicies: return None
+ for linked_index in _src_index_indicies[index]:
+ if linked_index in _restore_index_path:
+ srcpath = _restore_index_path[linked_index]
+ Log("Restoring %s by hard linking to %s" %
+ (rpath.path, srcpath), 6)
+ rpath.hardlink(srcpath)
+ return 1
+ _restore_index_path[index] = rpath.path
+ return None
+
+def link_rp(src_rorp, dest_rpath, dest_root = None):
+ """Make dest_rpath into a link analogous to that of src_rorp"""
+ if not dest_root: dest_root = dest_rpath # use base of dest_rpath
+ dest_link_rpath = RPath(dest_root.conn, dest_root.base,
+ get_indicies(src_rorp, 1)[0])
+ dest_rpath.hardlink(dest_link_rpath.path)
+
+def write_linkdict(rpath, dict, compress = None):
+ """Write link data to the rbdata dir
+
+ It is stored as the a big pickled dictionary dated to match
+ the current hardlinks.
+
+ """
+ assert (Globals.isbackup_writer and
+ rpath.conn is Globals.local_connection)
+ tf = TempFileManager.new(rpath)
+ def init():
+ fp = tf.open("wb", compress)
+ cPickle.dump(dict, fp)
+ assert not fp.close()
+ tf.setdata()
+ Robust.make_tf_robustaction(init, (tf,), (rpath,)).execute()
+
+def get_linkrp(data_rpath, time, prefix):
+ """Return RPath of linkdata, or None if cannot find"""
+ for rp in map(data_rpath.append, data_rpath.listdir()):
+ if (rp.isincfile() and rp.getincbase_str() == prefix and
+ (rp.getinctype() == 'snapshot' or rp.getinctype() == 'data')
+ and Time.stringtotime(rp.getinctime()) == time):
+ return rp
+ return None
+
+def get_linkdata(data_rpath, time, prefix = 'hardlink_data'):
+ """Return index dictionary written by write_linkdata at time"""
+ rp = get_linkrp(data_rpath, time, prefix)
+ if not rp: return None
+ fp = rp.open("rb", rp.isinccompressed())
+ index_dict = cPickle.load(fp)
+ assert not fp.close()
+ return index_dict
+
+def final_writedata():
+ """Write final checkpoint data to rbdir after successful backup"""
+ global final_inc
+ if _src_index_indicies:
+ Log("Writing hard link data", 6)
+ if Globals.compression:
+ final_inc = Globals.rbdir.append("hardlink_data.%s.data.gz" %
+ Time.curtimestr)
+ else: final_inc = Globals.rbdir.append("hardlink_data.%s.data" %
+ Time.curtimestr)
+ write_linkdict(final_inc, _src_index_indicies, Globals.compression)
+ else: # no hardlinks, so writing unnecessary
+ final_inc = None
+
+def retrieve_final(time):
+ """Set source index dictionary from hardlink_data file if avail"""
+ global _src_index_indicies
+ hd = get_linkdata(Globals.rbdir, time)
+ if hd is None: return None
+ _src_index_indicies = hd
+ return 1
+
+def final_checkpoint(data_rpath):
+ """Write contents of the four dictionaries to the data dir
+
+ If rdiff-backup receives a fatal error, it may still be able
+ to save the contents of the four hard link dictionaries.
+ Because these dictionaries may be big, they are not saved
+ after every 20 seconds or whatever, but just at the end.
+
+ """
+ Log("Writing intermediate hard link data to disk", 2)
+ src_inode_rp = data_rpath.append("hardlink_source_inode_checkpoint."
+ "%s.data" % Time.curtimestr)
+ src_index_rp = data_rpath.append("hardlink_source_index_checkpoint."
+ "%s.data" % Time.curtimestr)
+ dest_inode_rp = data_rpath.append("hardlink_dest_inode_checkpoint."
+ "%s.data" % Time.curtimestr)
+ dest_index_rp = data_rpath.append("hardlink_dest_index_checkpoint."
+ "%s.data" % Time.curtimestr)
+ for (rp, dict) in ((src_inode_rp, _src_inode_indicies),
+ (src_index_rp, _src_index_indicies),
+ (dest_inode_rp, _dest_inode_indicies),
+ (dest_index_rp, _dest_index_indicies)):
+ write_linkdict(rp, dict)
+
+def retrieve_checkpoint(data_rpath, time):
+ """Retrieve hardlink data from final checkpoint
+
+ Return true if the retrieval worked, false otherwise.
+
+ """
+ global _src_inode_indicies, _src_index_indicies
+ global _dest_inode_indicies, _dest_index_indicies
+ try:
+ src_inode = get_linkdata(data_rpath, time,
+ "hardlink_source_inode_checkpoint")
+ src_index = get_linkdata(data_rpath, time,
+ "hardlink_source_index_checkpoint")
+ dest_inode = get_linkdata(data_rpath, time,
+ "hardlink_dest_inode_checkpoint")
+ dest_index = get_linkdata(data_rpath, time,
+ "hardlink_dest_index_checkpoint")
+ except cPickle.UnpicklingError:
+ Log("Unpickling Error", 2)
+ return None
+ if (src_inode is None or src_index is None or
+ dest_inode is None or dest_index is None): return None
+ _src_inode_indicies, _src_index_indicies = src_inode, src_index
+ _dest_inode_indicies, _dest_index_indicies = dest_inode, dest_index
+ return 1
+
+def remove_all_checkpoints():
+ """Remove all hardlink checkpoint information from directory"""
+ prefix_list = ["hardlink_source_inode_checkpoint",
+ "hardlink_source_index_checkpoint",
+ "hardlink_dest_inode_checkpoint",
+ "hardlink_dest_index_checkpoint"]
+ for rp in map(Globals.rbdir.append, Globals.rbdir.listdir()):
+ if (rp.isincfile() and rp.getincbase_str() in prefix_list and
+ (rp.getinctype() == 'snapshot' or rp.getinctype() == 'data')):
+ rp.delete()
+
+
+from log import *
+from robust import *
+from rpath import *
+import Globals, Time
diff --git a/rdiff-backup/src/Main.py b/rdiff-backup/src/Main.py
new file mode 100644
index 0000000..94ca04a
--- /dev/null
+++ b/rdiff-backup/src/Main.py
@@ -0,0 +1,487 @@
+import getopt, sys, re
+from log import *
+from lazy import *
+from connection import *
+from rpath import *
+from destructive_stepping import *
+from robust import *
+from restore import *
+from highlevel import *
+from manage import *
+import Globals, Time, SetConnections
+
+#######################################################################
+#
+# main - Start here: Read arguments, set global settings, etc.
+#
+action = None
+remote_cmd, remote_schema = None, None
+force = None
+select_opts, select_mirror_opts = [], []
+select_files = []
+
+def parse_cmdlineoptions(arglist):
+ """Parse argument list and set global preferences"""
+ global args, action, force, restore_timestr, remote_cmd, remote_schema
+ global remove_older_than_string
+ def sel_fl(filename):
+ """Helper function for including/excluding filelists below"""
+ try: return open(filename, "r")
+ except IOError: Log.FatalError("Error opening file %s" % filename)
+
+ try: optlist, args = getopt.getopt(arglist, "blmr:sv:V",
+ ["backup-mode", "calculate-average",
+ "change-source-perms", "chars-to-quote=",
+ "checkpoint-interval=", "current-time=", "exclude=",
+ "exclude-device-files", "exclude-filelist=",
+ "exclude-filelist-stdin", "exclude-mirror=",
+ "exclude-regexp=", "force", "include=",
+ "include-filelist=", "include-filelist-stdin",
+ "include-regexp=", "list-increments", "mirror-only",
+ "no-compression", "no-compression-regexp=",
+ "no-hard-links", "no-resume", "null-separator",
+ "parsable-output", "print-statistics", "quoting-char=",
+ "remote-cmd=", "remote-schema=", "remove-older-than=",
+ "restore-as-of=", "resume", "resume-window=", "server",
+ "ssh-no-compression", "terminal-verbosity=",
+ "test-server", "verbosity", "version", "windows-mode",
+ "windows-time-format"])
+ except getopt.error, e:
+ commandline_error("Bad commandline options: %s" % str(e))
+
+ for opt, arg in optlist:
+ if opt == "-b" or opt == "--backup-mode": action = "backup"
+ elif opt == "--calculate-average": action = "calculate-average"
+ elif opt == "--change-source-perms":
+ Globals.set('change_source_perms', 1)
+ elif opt == "--chars-to-quote":
+ Globals.set('chars_to_quote', arg)
+ Globals.set('quoting_enabled', 1)
+ elif opt == "--checkpoint-interval":
+ Globals.set_integer('checkpoint_interval', arg)
+ elif opt == "--current-time":
+ Globals.set_integer('current_time', arg)
+ elif opt == "--exclude": select_opts.append((opt, arg))
+ elif opt == "--exclude-device-files": select_opts.append((opt, arg))
+ elif opt == "--exclude-filelist":
+ select_opts.append((opt, arg))
+ select_files.append(sel_fl(arg))
+ elif opt == "--exclude-filelist-stdin":
+ select_opts.append(("--exclude-filelist", "standard input"))
+ select_files.append(sys.stdin)
+ elif opt == "--exclude-mirror":
+ select_mirror_opts.append(("--exclude", arg))
+ elif opt == "--exclude-regexp": select_opts.append((opt, arg))
+ elif opt == "--force": force = 1
+ elif opt == "--include": select_opts.append((opt, arg))
+ elif opt == "--include-filelist":
+ select_opts.append((opt, arg))
+ select_files.append(sel_fl(arg))
+ elif opt == "--include-filelist-stdin":
+ select_opts.append(("--include-filelist", "standard input"))
+ select_files.append(sys.stdin)
+ elif opt == "--include-regexp": select_opts.append((opt, arg))
+ elif opt == "-l" or opt == "--list-increments":
+ action = "list-increments"
+ elif opt == "-m" or opt == "--mirror-only": action = "mirror"
+ elif opt == "--no-compression": Globals.set("compression", None)
+ elif opt == "--no-compression-regexp":
+ Globals.set("no_compression_regexp_string", arg)
+ elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0)
+ elif opt == '--no-resume': Globals.resume = 0
+ elif opt == "--null-separator": Globals.set("null_separator", 1)
+ elif opt == "-r" or opt == "--restore-as-of":
+ restore_timestr, action = arg, "restore-as-of"
+ elif opt == "--parsable-output": Globals.set('parsable_output', 1)
+ elif opt == "--print-statistics":
+ Globals.set('print_statistics', 1)
+ elif opt == "--quoting-char":
+ Globals.set('quoting_char', arg)
+ Globals.set('quoting_enabled', 1)
+ elif opt == "--remote-cmd": remote_cmd = arg
+ elif opt == "--remote-schema": remote_schema = arg
+ elif opt == "--remove-older-than":
+ remove_older_than_string = arg
+ action = "remove-older-than"
+ elif opt == '--resume': Globals.resume = 1
+ elif opt == '--resume-window':
+ Globals.set_integer('resume_window', arg)
+ elif opt == "-s" or opt == "--server": action = "server"
+ elif opt == "--ssh-no-compression":
+ Globals.set('ssh_compression', None)
+ elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg)
+ elif opt == "--test-server": action = "test-server"
+ elif opt == "-V" or opt == "--version":
+ print "rdiff-backup " + Globals.version
+ sys.exit(0)
+ elif opt == "-v" or opt == "--verbosity": Log.setverbosity(arg)
+ elif opt == "--windows-mode":
+ Globals.set('time_separator', "_")
+ Globals.set('chars_to_quote', ":")
+ Globals.set('quoting_enabled', 1)
+ elif opt == '--windows-time-format':
+ Globals.set('time_separator', "_")
+ else: Log.FatalError("Unknown option %s" % opt)
+
+def set_action():
+ """Check arguments and try to set action"""
+ global action
+ l = len(args)
+ if not action:
+ if l == 0: commandline_error("No arguments given")
+ elif l == 1: action = "restore"
+ elif l == 2:
+ if RPath(Globals.local_connection, args[0]).isincfile():
+ action = "restore"
+ else: action = "backup"
+ else: commandline_error("Too many arguments given")
+
+ if l == 0 and action != "server" and action != "test-server":
+ commandline_error("No arguments given")
+ if l > 0 and action == "server":
+ commandline_error("Too many arguments given")
+ if l < 2 and (action == "backup" or action == "mirror" or
+ action == "restore-as-of"):
+ commandline_error("Two arguments are required (source, destination).")
+ if l == 2 and (action == "list-increments" or
+ action == "remove-older-than"):
+ commandline_error("Only use one argument, "
+ "the root of the backup directory")
+ if l > 2 and action != "calculate-average":
+ commandline_error("Too many arguments given")
+
+def commandline_error(message):
+ sys.stderr.write("Error: %s\n" % message)
+ sys.stderr.write("See the rdiff-backup manual page for instructions\n")
+ sys.exit(1)
+
+def misc_setup(rps):
+ """Set default change ownership flag, umask, relay regexps"""
+ if ((len(rps) == 2 and rps[1].conn.os.getuid() == 0) or
+ (len(rps) < 2 and os.getuid() == 0)):
+ # Allow change_ownership if destination connection is root
+ for conn in Globals.connections:
+ conn.Globals.set('change_ownership', 1)
+ for rp in rps: rp.setdata() # Update with userinfo
+
+ os.umask(077)
+ Time.setcurtime(Globals.current_time)
+ FilenameMapping.set_init_quote_vals()
+ Globals.set("isclient", 1)
+ SetConnections.UpdateGlobal("client_conn", Globals.local_connection)
+
+ # This is because I originally didn't think compiled regexps
+ # could be pickled, and so must be compiled on remote side.
+ Globals.postset_regexp('no_compression_regexp',
+ Globals.no_compression_regexp_string)
+
+ for conn in Globals.connections: Robust.install_signal_handlers()
+
+def take_action(rps):
+ """Do whatever action says"""
+ if action == "server": PipeConnection(sys.stdin, sys.stdout).Server()
+ elif action == "backup": Backup(rps[0], rps[1])
+ elif action == "restore": restore(*rps)
+ elif action == "restore-as-of": RestoreAsOf(rps[0], rps[1])
+ elif action == "mirror": Mirror(rps[0], rps[1])
+ elif action == "test-server": SetConnections.TestConnections()
+ elif action == "list-increments": ListIncrements(rps[0])
+ elif action == "remove-older-than": RemoveOlderThan(rps[0])
+ elif action == "calculate-average": CalculateAverage(rps)
+ else: raise AssertionError("Unknown action " + action)
+
+def cleanup():
+ """Do any last minute cleaning before exiting"""
+ Log("Cleaning up", 6)
+ Log.close_logfile()
+ if not Globals.server: SetConnections.CloseConnections()
+
+def Main(arglist):
+ """Start everything up!"""
+ parse_cmdlineoptions(arglist)
+ set_action()
+ rps = SetConnections.InitRPs(args, remote_schema, remote_cmd)
+ misc_setup(rps)
+ take_action(rps)
+ cleanup()
+
+
+def Mirror(src_rp, dest_rp):
+ """Turn dest_path into a copy of src_path"""
+ Log("Mirroring %s to %s" % (src_rp.path, dest_rp.path), 5)
+ mirror_check_paths(src_rp, dest_rp)
+ # Since no "rdiff-backup-data" dir, use root of destination.
+ SetConnections.UpdateGlobal('rbdir', dest_rp)
+ SetConnections.BackupInitConnections(src_rp.conn, dest_rp.conn)
+ HighLevel.Mirror(src_rp, dest_rp)
+
+def mirror_check_paths(rpin, rpout):
+ """Check paths and return rpin, rpout"""
+ if not rpin.lstat():
+ Log.FatalError("Source directory %s does not exist" % rpin.path)
+ if rpout.lstat() and not force: Log.FatalError(
+"""Destination %s exists so continuing could mess it up. Run
+rdiff-backup with the --force option if you want to mirror anyway.""" %
+ rpout.path)
+
+
+def Backup(rpin, rpout):
+ """Backup, possibly incrementally, src_path to dest_path."""
+ SetConnections.BackupInitConnections(rpin.conn, rpout.conn)
+ backup_init_select(rpin, rpout)
+ backup_init_dirs(rpin, rpout)
+ RSI = Globals.backup_writer.Resume.ResumeCheck()
+ SaveState.init_filenames()
+ if prevtime:
+ Time.setprevtime(prevtime)
+ HighLevel.Mirror_and_increment(rpin, rpout, incdir, RSI)
+ else: HighLevel.Mirror(rpin, rpout, incdir, RSI)
+ backup_touch_curmirror(rpin, rpout)
+
+def backup_init_select(rpin, rpout):
+ """Create Select objects on source and dest connections"""
+ rpin.conn.Globals.set_select(DSRPath(1, rpin), select_opts,
+ None, *select_files)
+ rpout.conn.Globals.set_select(DSRPath(None, rpout), select_mirror_opts, 1)
+
+def backup_init_dirs(rpin, rpout):
+ """Make sure rpin and rpout are valid, init data dir and logging"""
+ global datadir, incdir, prevtime
+ if rpout.lstat() and not rpout.isdir():
+ if not force: Log.FatalError("Destination %s exists and is not a "
+ "directory" % rpout.path)
+ else:
+ Log("Deleting %s" % rpout.path, 3)
+ rpout.delete()
+
+ if not rpin.lstat():
+ Log.FatalError("Source directory %s does not exist" % rpin.path)
+ elif not rpin.isdir():
+ Log.FatalError("Source %s is not a directory" % rpin.path)
+
+ datadir = rpout.append("rdiff-backup-data")
+ SetConnections.UpdateGlobal('rbdir', datadir)
+ incdir = RPath(rpout.conn, os.path.join(datadir.path, "increments"))
+ prevtime = backup_get_mirrortime()
+
+ if rpout.lstat():
+ if rpout.isdir() and not rpout.listdir(): # rpout is empty dir
+ rpout.chmod(0700) # just make sure permissions aren't too lax
+ elif not datadir.lstat() and not force: Log.FatalError(
+"""Destination directory %s exists, but does not look like a
+rdiff-backup directory. Running rdiff-backup like this could mess up
+what is currently in it. If you want to overwrite it, run
+rdiff-backup with the --force option.""" % rpout.path)
+
+ if not rpout.lstat():
+ try: rpout.mkdir()
+ except os.error:
+ Log.FatalError("Unable to create directory %s" % rpout.path)
+ if not datadir.lstat(): datadir.mkdir()
+ if Log.verbosity > 0:
+ Log.open_logfile(datadir.append("backup.log"))
+ backup_warn_if_infinite_regress(rpin, rpout)
+
+def backup_warn_if_infinite_regress(rpin, rpout):
+ """Warn user if destination area contained in source area"""
+ if rpout.conn is rpin.conn: # it's meaningful to compare paths
+ if ((len(rpout.path) > len(rpin.path)+1 and
+ rpout.path[:len(rpin.path)] == rpin.path and
+ rpout.path[len(rpin.path)] == '/') or
+ (rpin.path == "." and rpout.path[0] != '/' and
+ rpout.path[:2] != '..')):
+ # Just a few heuristics, we don't have to get every case
+ if Globals.backup_reader.Globals.select_source.Select(rpout): Log(
+"""Warning: The destination directory '%s' may be contained in the
+source directory '%s'. This could cause an infinite regress. You
+may need to use the --exclude option.""" % (rpout.path, rpin.path), 2)
+
+def backup_get_mirrorrps():
+ """Return list of current_mirror rps"""
+ if not datadir.isdir(): return []
+ mirrorrps = [datadir.append(fn) for fn in datadir.listdir()
+ if fn.startswith("current_mirror.")]
+ return filter(lambda rp: rp.isincfile(), mirrorrps)
+
+def backup_get_mirrortime():
+ """Return time in seconds of previous mirror, or None if cannot"""
+ mirrorrps = backup_get_mirrorrps()
+ if not mirrorrps: return None
+ if len(mirrorrps) > 1:
+ Log(
+"""Warning: duplicate current_mirror files found. Perhaps something
+went wrong during your last backup? Using """ + mirrorrps[-1].path, 2)
+
+ timestr = mirrorrps[-1].getinctime()
+ return Time.stringtotime(timestr)
+
+def backup_touch_curmirror(rpin, rpout):
+ """Make a file like current_mirror.time.data to record time
+
+ Also updates rpout so mod times don't get messed up.
+
+ """
+ map(RPath.delete, backup_get_mirrorrps())
+ mirrorrp = datadir.append("current_mirror.%s.%s" % (Time.curtimestr,
+ "data"))
+ Log("Touching mirror marker %s" % mirrorrp.path, 6)
+ mirrorrp.touch()
+ RPath.copy_attribs(rpin, rpout)
+
+
+def restore(src_rp, dest_rp = None):
+ """Main restoring function
+
+ Here src_rp should be an increment file, and if dest_rp is
+ missing it defaults to the base of the increment.
+
+ """
+ rpin, rpout = restore_check_paths(src_rp, dest_rp)
+ time = Time.stringtotime(rpin.getinctime())
+ restore_common(rpin, rpout, time)
+
+def RestoreAsOf(rpin, target):
+ """Secondary syntax for restore operation
+
+ rpin - RPath of mirror file to restore (not nec. with correct index)
+ target - RPath of place to put restored file
+
+ """
+ restore_check_paths(rpin, target, 1)
+ try: time = Time.genstrtotime(restore_timestr)
+ except Time.TimeException, exc: Log.FatalError(str(exc))
+ restore_common(rpin, target, time)
+
+def restore_common(rpin, target, time):
+ """Restore operation common to Restore and RestoreAsOf"""
+ Log("Starting Restore", 5)
+ mirror_root, index = restore_get_root(rpin)
+ mirror = mirror_root.new_index(index)
+ inc_rpath = datadir.append_path('increments', index)
+ restore_init_select(mirror_root, target)
+ Log.open_logfile(datadir.append("restore.log"))
+ Restore.Restore(inc_rpath, mirror, target, time)
+
+def restore_check_paths(rpin, rpout, restoreasof = None):
+ """Check paths and return pair of corresponding rps"""
+ if not restoreasof:
+ if not rpin.lstat():
+ Log.FatalError("Source file %s does not exist" % rpin.path)
+ elif not rpin.isincfile():
+ Log.FatalError("""File %s does not look like an increment file.
+
+Try restoring from an increment file (the filenames look like
+"foobar.2001-09-01T04:49:04-07:00.diff").""" % rpin.path)
+
+ if not rpout: rpout = RPath(Globals.local_connection,
+ rpin.getincbase_str())
+ if rpout.lstat():
+ Log.FatalError("Restore target %s already exists, "
+ "and will not be overwritten." % rpout.path)
+ return rpin, rpout
+
+def restore_init_select(rpin, rpout):
+ """Initialize Select
+
+ Unlike the backup selections, here they are on the local
+ connection, because the backup operation is pipelined in a way
+ the restore operation isn't.
+
+ """
+ Globals.set_select(DSRPath(1, rpin), select_mirror_opts, None)
+ Globals.set_select(DSRPath(None, rpout), select_opts, None, *select_files)
+
+def restore_get_root(rpin):
+ """Return (mirror root, index) and set the data dir
+
+ The idea here is to keep backing up on the path until we find
+ a directory that contains "rdiff-backup-data". That is the
+ mirror root. If the path from there starts
+ "rdiff-backup-data/increments*", then the index is the
+ remainder minus that. Otherwise the index is just the path
+ minus the root.
+
+ All this could fail if the increment file is pointed to in a
+ funny way, using symlinks or somesuch.
+
+ """
+ global datadir
+ if rpin.isincfile(): relpath = rpin.getincbase().path
+ else: relpath = rpin.path
+ pathcomps = os.path.join(rpin.conn.os.getcwd(), relpath).split("/")
+ assert len(pathcomps) >= 2 # path should be relative to /
+
+ i = len(pathcomps)
+ while i >= 2:
+ parent_dir = RPath(rpin.conn, "/".join(pathcomps[:i]))
+ if (parent_dir.isdir() and
+ "rdiff-backup-data" in parent_dir.listdir()): break
+ i = i-1
+ else: Log.FatalError("Unable to find rdiff-backup-data directory")
+
+ rootrp = parent_dir
+ Log("Using mirror root directory %s" % rootrp.path, 6)
+
+ datadir = rootrp.append_path("rdiff-backup-data")
+ SetConnections.UpdateGlobal('rbdir', datadir)
+ if not datadir.isdir():
+ Log.FatalError("Unable to read rdiff-backup-data directory %s" %
+ datadir.path)
+
+ from_datadir = tuple(pathcomps[i:])
+ if not from_datadir or from_datadir[0] != "rdiff-backup-data":
+ return (rootrp, from_datadir) # in mirror, not increments
+ assert from_datadir[1] == "increments"
+ return (rootrp, from_datadir[2:])
+
+
+def ListIncrements(rp):
+ """Print out a summary of the increments and their times"""
+ mirror_root, index = restore_get_root(rp)
+ Globals.rbdir = datadir = \
+ mirror_root.append_path("rdiff-backup-data")
+ mirrorrp = mirror_root.new_index(index)
+ inc_rpath = datadir.append_path('increments', index)
+ incs = Restore.get_inclist(inc_rpath)
+ mirror_time = Restore.get_mirror_time()
+ if Globals.parsable_output:
+ print Manage.describe_incs_parsable(incs, mirror_time, mirrorrp)
+ else: print Manage.describe_incs_human(incs, mirror_time, mirrorrp)
+
+
+def CalculateAverage(rps):
+ """Print out the average of the given statistics files"""
+ statobjs = map(lambda rp: StatsObj().read_stats_from_rp(rp), rps)
+ average_stats = StatsObj().set_to_average(statobjs)
+ print average_stats.get_stats_logstring(
+ "Average of %d stat files" % len(rps))
+
+
+def RemoveOlderThan(rootrp):
+ """Remove all increment files older than a certain time"""
+ datadir = rootrp.append("rdiff-backup-data")
+ if not datadir.lstat() or not datadir.isdir():
+ Log.FatalError("Unable to open rdiff-backup-data dir %s" %
+ (datadir.path,))
+
+ try: time = Time.genstrtotime(remove_older_than_string)
+ except TimeError, exc: Log.FatalError(str(exc))
+ timep = Time.timetopretty(time)
+ Log("Deleting increment(s) before %s" % timep, 4)
+
+ itimes = [Time.stringtopretty(inc.getinctime())
+ for inc in Restore.get_inclist(datadir.append("increments"))
+ if Time.stringtotime(inc.getinctime()) < time]
+
+ if not itimes:
+ Log.FatalError("No increments older than %s found" % timep)
+ inc_pretty_time = "\n".join(itimes)
+ if len(itimes) > 1 and not force:
+ Log.FatalError("Found %d relevant increments, dated:\n%s"
+ "\nIf you want to delete multiple increments in this way, "
+ "use the --force." % (len(itimes), inc_pretty_time))
+
+ Log("Deleting increment%sat times:\n%s" %
+ (len(itimes) == 1 and " " or "s ", inc_pretty_time), 3)
+ Manage.delete_earlier_than(datadir, time)
+
diff --git a/rdiff-backup/src/Make.old b/rdiff-backup/src/Make.old
new file mode 100755
index 0000000..2b79ffe
--- /dev/null
+++ b/rdiff-backup/src/Make.old
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+"""Read component files of rdiff-backup, and glue them together after
+removing unnecessary bits."""
+
+import os
+
+def mystrip(filename):
+ """Open filename, read input, strip appropriately, and return contents"""
+ fp = open(filename, "r")
+ lines = fp.readlines()
+ fp.close()
+
+ i = 0
+ while(lines[i][:60] !=
+ "############################################################"):
+ i = i+1
+
+ return "".join(lines[i:]).strip() + "\n\n\n"
+
+
+
+files = ["globals.py", "static.py", "lazy.py", "log.py", "ttime.py",
+ "iterfile.py", "rdiff.py", "connection.py", "rpath.py",
+ "hardlink.py", "robust.py", "rorpiter.py",
+ "destructive_stepping.py", "selection.py",
+ "filename_mapping.py", "statistics.py", "increment.py",
+ "restore.py", "manage.py", "highlevel.py",
+ "setconnections.py", "main.py"]
+
+os.system("cp header.py rdiff-backup")
+
+outfp = open("rdiff-backup", "a")
+for file in files:
+ outfp.write(mystrip(file))
+outfp.close()
+
+os.system("chmod 755 rdiff-backup")
diff --git a/rdiff-backup/src/MiscStats.py b/rdiff-backup/src/MiscStats.py
new file mode 100644
index 0000000..cd62dd6
--- /dev/null
+++ b/rdiff-backup/src/MiscStats.py
@@ -0,0 +1,72 @@
+from statistics import *
+
+"""Misc statistics methods, pertaining to dir and session stat files"""
+# This is the RPath of the directory statistics file, and the
+# associated open file. It will hold a line of statistics for
+# each directory that is backed up.
+_dir_stats_rp = None
+_dir_stats_fp = None
+
+# This goes at the beginning of the directory statistics file and
+# explains the format.
+_dir_stats_header = """# rdiff-backup directory statistics file
+#
+# Each line is in the following format:
+# RelativeDirName %s
+""" % " ".join(StatsObj.stat_file_attrs)
+
+def open_dir_stats_file():
+ """Open directory statistics file, write header"""
+ global _dir_stats_fp, _dir_stats_rp
+ assert not _dir_stats_fp, "Directory file already open"
+
+ if Globals.compression: suffix = "data.gz"
+ else: suffix = "data"
+ _dir_stats_rp = Inc.get_inc(Globals.rbdir.append("directory_statistics"),
+ Time.curtime, suffix)
+
+ if _dir_stats_rp.lstat():
+ Log("Warning, statistics file %s already exists, appending" %
+ _dir_stats_rp.path, 2)
+ _dir_stats_fp = _dir_stats_rp.open("ab", Globals.compression)
+ else: _dir_stats_fp = _dir_stats_rp.open("wb", Globals.compression)
+ _dir_stats_fp.write(_dir_stats_header)
+
+def write_dir_stats_line(statobj, index):
+ """Write info from statobj about rpath to statistics file"""
+ if Globals.null_separator:
+ _dir_stats_fp.write(statobj.get_stats_line(index, None) + "\0")
+ else: _dir_stats_fp.write(statobj.get_stats_line(index) + "\n")
+
+def close_dir_stats_file():
+ """Close directory statistics file if its open"""
+ global _dir_stats_fp
+ if _dir_stats_fp:
+ _dir_stats_fp.close()
+ _dir_stats_fp = None
+
+def write_session_statistics(statobj):
+ """Write session statistics into file, log"""
+ stat_inc = Inc.get_inc(Globals.rbdir.append("session_statistics"),
+ Time.curtime, "data")
+ statobj.StartTime = Time.curtime
+ statobj.EndTime = time.time()
+
+ # include hardlink data and dir stats in size of increments
+ if Globals.preserve_hardlinks and Hardlink.final_inc:
+ # include hardlink data in size of increments
+ statobj.IncrementFiles += 1
+ statobj.IncrementFileSize += Hardlink.final_inc.getsize()
+ if _dir_stats_rp and _dir_stats_rp.lstat():
+ statobj.IncrementFiles += 1
+ statobj.IncrementFileSize += _dir_stats_rp.getsize()
+
+ statobj.write_stats_to_rp(stat_inc)
+ if Globals.print_statistics:
+ message = statobj.get_stats_logstring("Session statistics")
+ Log.log_to_file(message)
+ Globals.client_conn.sys.stdout.write(message)
+
+
+from increment import *
+import Hardlink
diff --git a/rdiff-backup/src/Rdiff.py b/rdiff-backup/src/Rdiff.py
new file mode 100644
index 0000000..c9895cb
--- /dev/null
+++ b/rdiff-backup/src/Rdiff.py
@@ -0,0 +1,181 @@
+import os, popen2
+
+#######################################################################
+#
+# rdiff - Invoke rdiff utility to make signatures, deltas, or patch
+#
+# All these operations should be done in a relatively safe manner
+# using RobustAction and the like.
+
+class RdiffException(Exception): pass
+
+def get_signature(rp):
+ """Take signature of rpin file and return in file object"""
+ Log("Getting signature of %s" % rp.path, 7)
+ return rp.conn.Rdiff.Popen(['rdiff', 'signature', rp.path])
+
+def get_delta_sigfileobj(sig_fileobj, rp_new):
+ """Like get_delta but signature is in a file object"""
+ sig_tf = TempFileManager.new(rp_new, None)
+ sig_tf.write_from_fileobj(sig_fileobj)
+ rdiff_popen_obj = get_delta_sigrp(sig_tf, rp_new)
+ rdiff_popen_obj.set_thunk(sig_tf.delete)
+ return rdiff_popen_obj
+
+def get_delta_sigrp(rp_signature, rp_new):
+ """Take signature rp and new rp, return delta file object"""
+ assert rp_signature.conn is rp_new.conn
+ Log("Getting delta of %s with signature %s" %
+ (rp_new.path, rp_signature.path), 7)
+ return rp_new.conn.Rdiff.Popen(['rdiff', 'delta',
+ rp_signature.path, rp_new.path])
+
+def write_delta_action(basis, new, delta, compress = None):
+ """Return action writing delta which brings basis to new
+
+ If compress is true, the output of rdiff will be gzipped
+ before written to delta.
+
+ """
+ sig_tf = TempFileManager.new(new, None)
+ delta_tf = TempFileManager.new(delta)
+ def init(): write_delta(basis, new, delta_tf, compress, sig_tf)
+ return Robust.make_tf_robustaction(init, (sig_tf, delta_tf),
+ (None, delta))
+
+def write_delta(basis, new, delta, compress = None, sig_tf = None):
+ """Write rdiff delta which brings basis to new"""
+ Log("Writing delta %s from %s -> %s" %
+ (basis.path, new.path, delta.path), 7)
+ if not sig_tf: sig_tf = TempFileManager.new(new, None)
+ sig_tf.write_from_fileobj(get_signature(basis))
+ delta.write_from_fileobj(get_delta_sigrp(sig_tf, new), compress)
+ sig_tf.delete()
+
+def patch_action(rp_basis, rp_delta, rp_out = None,
+ out_tf = None, delta_compressed = None):
+ """Return RobustAction which patches rp_basis with rp_delta
+
+ If rp_out is None, put output in rp_basis. Will use TempFile
+ out_tf it is specified. If delta_compressed is true, the
+ delta file will be decompressed before processing with rdiff.
+
+ """
+ if not rp_out: rp_out = rp_basis
+ else: assert rp_out.conn is rp_basis.conn
+ if (delta_compressed or
+ not (isinstance(rp_delta, RPath) and isinstance(rp_basis, RPath)
+ and rp_basis.conn is rp_delta.conn)):
+ if delta_compressed:
+ assert isinstance(rp_delta, RPath)
+ return patch_fileobj_action(rp_basis, rp_delta.open('rb', 1),
+ rp_out, out_tf)
+ else: return patch_fileobj_action(rp_basis, rp_delta.open('rb'),
+ rp_out, out_tf)
+
+ # Files are uncompressed on same connection, run rdiff
+ if out_tf is None: out_tf = TempFileManager.new(rp_out)
+ def init():
+ Log("Patching %s using %s to %s via %s" %
+ (rp_basis.path, rp_delta.path, rp_out.path, out_tf.path), 7)
+ cmdlist = ["rdiff", "patch", rp_basis.path,
+ rp_delta.path, out_tf.path]
+ return_val = rp_basis.conn.os.spawnvp(os.P_WAIT, 'rdiff', cmdlist)
+ out_tf.setdata()
+ if return_val != 0 or not out_tf.lstat():
+ RdiffException("Error running %s" % cmdlist)
+ return Robust.make_tf_robustaction(init, (out_tf,), (rp_out,))
+
+def patch_fileobj_action(rp_basis, delta_fileobj, rp_out = None,
+ out_tf = None, delta_compressed = None):
+ """Like patch_action but diff is given in fileobj form
+
+ Nest a writing of a tempfile with the actual patching to
+ create a new action. We have to nest so that the tempfile
+ will be around until the patching finishes.
+
+ """
+ if not rp_out: rp_out = rp_basis
+ delta_tf = TempFileManager.new(rp_out, None)
+ def init(): delta_tf.write_from_fileobj(delta_fileobj)
+ def final(init_val): delta_tf.delete()
+ def error(exc, ran_init, init_val): delta_tf.delete()
+ write_delta_action = RobustAction(init, final, error)
+ return Robust.chain(write_delta_action, patch_action(rp_basis, delta_tf,
+ rp_out, out_tf))
+
+def patch_with_attribs_action(rp_basis, rp_delta, rp_out = None):
+ """Like patch_action, but also transfers attributs from rp_delta"""
+ if not rp_out: rp_out = rp_basis
+ tf = TempFileManager.new(rp_out)
+ return Robust.chain_nested(patch_action(rp_basis, rp_delta, rp_out, tf),
+ Robust.copy_attribs_action(rp_delta, tf))
+
+def copy_action(rpin, rpout):
+ """Use rdiff to copy rpin to rpout, conserving bandwidth"""
+ if not rpin.isreg() or not rpout.isreg() or rpin.conn is rpout.conn:
+ # rdiff not applicable, fallback to regular copying
+ return Robust.copy_action(rpin, rpout)
+
+ Log("Rdiff copying %s to %s" % (rpin.path, rpout.path), 6)
+ delta_tf = TempFileManager.new(rpout, None)
+ return Robust.chain(write_delta_action(rpout, rpin, delta_tf),
+ patch_action(rpout, delta_tf),
+ RobustAction(lambda: None, delta_tf.delete,
+ lambda exc: delta_tf.delete))
+
+
+class Popen:
+ """Spawn process and treat stdout as file object
+
+ Instead of using popen, which evaluates arguments with the shell
+ and thus may lead to security holes (thanks to Jamie Heilman for
+ this point), use the popen2 class and discard stdin.
+
+ When closed, this object checks to make sure the process exited
+ cleanly, and executes closing_thunk.
+
+ """
+ def __init__(self, cmdlist, closing_thunk = None):
+ """RdiffFilehook initializer
+
+ fileobj is the file we are emulating
+ thunk is called with no parameters right after the file is closed
+
+ """
+ assert type(cmdlist) is types.ListType
+ self.p3obj = popen2.Popen3(cmdlist)
+ self.fileobj = self.p3obj.fromchild
+ self.closing_thunk = closing_thunk
+ self.cmdlist = cmdlist
+
+ def set_thunk(self, closing_thunk):
+ """Set closing_thunk if not already"""
+ assert not self.closing_thunk
+ self.closing_thunk = closing_thunk
+
+ def read(self, length = -1): return self.fileobj.read(length)
+
+ def close(self):
+ closeval = self.fileobj.close()
+ if self.closing_thunk: self.closing_thunk()
+ exitval = self.p3obj.poll()
+ if exitval == 0: return closeval
+ elif exitval == 256:
+ Log("Failure probably because %s couldn't be found in PATH."
+ % self.cmdlist[0], 2)
+ assert 0, "rdiff not found"
+ elif exitval == -1:
+ # There may a race condition where a process closes
+ # but doesn't provide its exitval fast enough.
+ Log("Waiting for process to close", 8)
+ time.sleep(0.2)
+ exitval = self.p3obj.poll()
+ if exitval == 0: return closeval
+ raise RdiffException("%s exited with non-zero value %d" %
+ (self.cmdlist, exitval))
+
+
+from log import *
+from robust import *
+
diff --git a/rdiff-backup/src/SetConnections.py b/rdiff-backup/src/SetConnections.py
new file mode 100644
index 0000000..be3fdfd
--- /dev/null
+++ b/rdiff-backup/src/SetConnections.py
@@ -0,0 +1,219 @@
+#######################################################################
+#
+# setconnections - Parse initial arguments and establish connections
+#
+
+"""Parse args and setup connections
+
+The methods in this class are used once by Main to parse file
+descriptions like bescoto@folly.stanford.edu:/usr/bin/ls and to
+set up the related connections.
+
+"""
+
+class SetConnectionsException(Exception): pass
+
+
+# This is the schema that determines how rdiff-backup will open a
+# pipe to the remote system. If the file is given as A::B, %s will
+# be substituted with A in the schema.
+__cmd_schema = 'ssh -C %s rdiff-backup --server'
+__cmd_schema_no_compress = 'ssh %s rdiff-backup --server'
+
+# This is a list of remote commands used to start the connections.
+# The first is None because it is the local connection.
+__conn_remote_cmds = [None]
+
+def InitRPs(arglist, remote_schema = None, remote_cmd = None):
+ """Map the given file descriptions into rpaths and return list"""
+ global __cmd_schema
+ if remote_schema: __cmd_schema = remote_schema
+ elif not Globals.ssh_compression: __cmd_schema = __cmd_schema_no_compress
+
+ if not arglist: return []
+ desc_pairs = map(parse_file_desc, arglist)
+
+ if filter(lambda x: x[0], desc_pairs): # True if any host_info found
+ if remote_cmd:
+ Log.FatalError("The --remote-cmd flag is not compatible "
+ "with remote file descriptions.")
+ elif remote_schema:
+ Log("Remote schema option ignored - no remote file "
+ "descriptions.", 2)
+
+ cmd_pairs = map(desc2cmd_pairs, desc_pairs)
+ if remote_cmd: # last file description gets remote_cmd
+ cmd_pairs[-1] = (remote_cmd, cmd_pairs[-1][1])
+ return map(cmdpair2rp, cmd_pairs)
+
+def cmdpair2rp(cmd_pair):
+ """Return RPath from cmd_pair (remote_cmd, filename)"""
+ cmd, filename = cmd_pair
+ if cmd: conn = init_connection(cmd)
+ else: conn = Globals.local_connection
+ return RPath(conn, filename)
+
+def desc2cmd_pairs(desc_pair):
+ """Return pair (remote_cmd, filename) from desc_pair"""
+ host_info, filename = desc_pair
+ if not host_info: return (None, filename)
+ else: return (fill_schema(host_info), filename)
+
+def parse_file_desc(file_desc):
+ """Parse file description returning pair (host_info, filename)
+
+ In other words, bescoto@folly.stanford.edu::/usr/bin/ls =>
+ ("bescoto@folly.stanford.edu", "/usr/bin/ls"). The
+ complication is to allow for quoting of : by a \. If the
+ string is not separated by :, then the host_info is None.
+
+ """
+ def check_len(i):
+ if i >= len(file_desc):
+ raise SetConnectionsException(
+ "Unexpected end to file description %s" % file_desc)
+
+ host_info_list, i, last_was_quoted = [], 0, None
+ while 1:
+ if i == len(file_desc):
+ return (None, file_desc)
+
+ if file_desc[i] == '\\':
+ i = i+1
+ check_len(i)
+ last_was_quoted = 1
+ elif (file_desc[i] == ":" and i > 0 and file_desc[i-1] == ":"
+ and not last_was_quoted):
+ host_info_list.pop() # Remove last colon from name
+ break
+ else: last_was_quoted = None
+ host_info_list.append(file_desc[i])
+ i = i+1
+
+ check_len(i+1)
+ return ("".join(host_info_list), file_desc[i+1:])
+
+def fill_schema(host_info):
+ """Fills host_info into the schema and returns remote command"""
+ return __cmd_schema % host_info
+
+def init_connection(remote_cmd):
+ """Run remote_cmd, register connection, and then return it
+
+ If remote_cmd is None, then the local connection will be
+ returned. This also updates some settings on the remote side,
+ like global settings, its connection number, and verbosity.
+
+ """
+ if not remote_cmd: return Globals.local_connection
+
+ Log("Executing " + remote_cmd, 4)
+ stdin, stdout = os.popen2(remote_cmd)
+ conn_number = len(Globals.connections)
+ conn = PipeConnection(stdout, stdin, conn_number)
+
+ check_connection_version(conn, remote_cmd)
+ Log("Registering connection %d" % conn_number, 7)
+ init_connection_routing(conn, conn_number, remote_cmd)
+ init_connection_settings(conn)
+ return conn
+
+def check_connection_version(conn, remote_cmd):
+ """Log warning if connection has different version"""
+ try: remote_version = conn.Globals.get('version')
+ except ConnectionReadError, exception:
+ Log.FatalError("""%s
+
+Couldn't start up the remote connection by executing
+
+ %s
+
+Remember that, under the default settings, rdiff-backup must be
+installed in the PATH on the remote system. See the man page for more
+information.""" % (exception, remote_cmd))
+
+ if remote_version != Globals.version:
+ Log("Warning: Local version %s does not match remote version %s."
+ % (Globals.version, remote_version), 2)
+
+def init_connection_routing(conn, conn_number, remote_cmd):
+ """Called by init_connection, establish routing, conn dict"""
+ Globals.connection_dict[conn_number] = conn
+
+ conn.SetConnections.init_connection_remote(conn_number)
+ for other_remote_conn in Globals.connections[1:]:
+ conn.SetConnections.add_redirected_conn(
+ other_remote_conn.conn_number)
+ other_remote_conn.SetConnections.add_redirected_conn(conn_number)
+
+ Globals.connections.append(conn)
+ __conn_remote_cmds.append(remote_cmd)
+
+def init_connection_settings(conn):
+ """Tell new conn about log settings and updated globals"""
+ conn.Log.setverbosity(Log.verbosity)
+ conn.Log.setterm_verbosity(Log.term_verbosity)
+ for setting_name in Globals.changed_settings:
+ conn.Globals.set(setting_name, Globals.get(setting_name))
+
+def init_connection_remote(conn_number):
+ """Run on server side to tell self that have given conn_number"""
+ Globals.connection_number = conn_number
+ Globals.local_connection.conn_number = conn_number
+ Globals.connection_dict[0] = Globals.connections[1]
+ Globals.connection_dict[conn_number] = Globals.local_connection
+
+def add_redirected_conn(conn_number):
+ """Run on server side - tell about redirected connection"""
+ Globals.connection_dict[conn_number] = \
+ RedirectedConnection(conn_number)
+
+def UpdateGlobal(setting_name, val):
+ """Update value of global variable across all connections"""
+ for conn in Globals.connections:
+ conn.Globals.set(setting_name, val)
+
+def BackupInitConnections(reading_conn, writing_conn):
+ """Backup specific connection initialization"""
+ reading_conn.Globals.set("isbackup_reader", 1)
+ writing_conn.Globals.set("isbackup_writer", 1)
+ UpdateGlobal("backup_reader", reading_conn)
+ UpdateGlobal("backup_writer", writing_conn)
+
+def CloseConnections():
+ """Close all connections. Run by client"""
+ assert not Globals.server
+ for conn in Globals.connections: conn.quit()
+ del Globals.connections[1:] # Only leave local connection
+ Globals.connection_dict = {0: Globals.local_connection}
+ Globals.backup_reader = Globals.isbackup_reader = \
+ Globals.backup_writer = Globals.isbackup_writer = None
+
+def TestConnections():
+ """Test connections, printing results"""
+ if len(Globals.connections) == 1: print "No remote connections specified"
+ else:
+ for i in range(1, len(Globals.connections)): test_connection(i)
+
+def test_connection(conn_number):
+ """Test connection. conn_number 0 is the local connection"""
+ print "Testing server started by: ", __conn_remote_cmds[conn_number]
+ conn = Globals.connections[conn_number]
+ try:
+ assert conn.pow(2,3) == 8
+ assert conn.os.path.join("a", "b") == "a/b"
+ version = conn.reval("lambda: Globals.version")
+ except:
+ sys.stderr.write("Server tests failed\n")
+ raise
+ if not version == Globals.version:
+ print """Server may work, but there is a version mismatch:
+Local version: %s
+Remote version: %s""" % (Globals.version, version)
+ else: print "Server OK"
+
+
+from log import *
+from rpath import *
+from connection import *
+import Globals
diff --git a/rdiff-backup/src/Time.py b/rdiff-backup/src/Time.py
new file mode 100644
index 0000000..4eb2107
--- /dev/null
+++ b/rdiff-backup/src/Time.py
@@ -0,0 +1,199 @@
+import time, types, re
+import Globals
+
+#######################################################################
+#
+# ttime - Provide Time class, which contains time related functions.
+#
+
+class TimeException(Exception): pass
+
+_interval_conv_dict = {"s": 1, "m": 60, "h": 3600, "D": 86400,
+ "W": 7*86400, "M": 30*86400, "Y": 365*86400}
+_integer_regexp = re.compile("^[0-9]+$")
+_interval_regexp = re.compile("^([0-9]+)([smhDWMY])")
+_genstr_date_regexp1 = re.compile("^(?P<year>[0-9]{4})[-/]"
+ "(?P<month>[0-9]{1,2})[-/](?P<day>[0-9]{1,2})$")
+_genstr_date_regexp2 = re.compile("^(?P<month>[0-9]{1,2})[-/]"
+ "(?P<day>[0-9]{1,2})[-/](?P<year>[0-9]{4})$")
+curtime = curtimestr = None
+
+def setcurtime(curtime = None):
+ """Sets the current time in curtime and curtimestr on all systems"""
+ t = curtime or time.time()
+ for conn in Globals.connections:
+ conn.Time.setcurtime_local(t, timetostring(t))
+
+def setcurtime_local(timeinseconds, timestr):
+ """Only set the current time locally"""
+ global curtime, curtimestr
+ curtime, curtimestr = timeinseconds, timestr
+
+def setprevtime(timeinseconds):
+ """Sets the previous inc time in prevtime and prevtimestr"""
+ assert timeinseconds > 0, timeinseconds
+ timestr = timetostring(timeinseconds)
+ for conn in Globals.connections:
+ conn.Time.setprevtime_local(timeinseconds, timestr)
+
+def setprevtime_local(timeinseconds, timestr):
+ """Like setprevtime but only set the local version"""
+ global prevtime, prevtimestr
+ prevtime, prevtimestr = timeinseconds, timestr
+
+def timetostring(timeinseconds):
+ """Return w3 datetime compliant listing of timeinseconds"""
+ return time.strftime("%Y-%m-%dT%H" + Globals.time_separator +
+ "%M" + Globals.time_separator + "%S",
+ time.localtime(timeinseconds)) + gettzd()
+
+def stringtotime(timestring):
+ """Return time in seconds from w3 timestring
+
+ If there is an error parsing the string, or it doesn't look
+ like a w3 datetime string, return None.
+
+ """
+ try:
+ date, daytime = timestring[:19].split("T")
+ year, month, day = map(int, date.split("-"))
+ hour, minute, second = map(int,
+ daytime.split(Globals.time_separator))
+ assert 1900 < year < 2100, year
+ assert 1 <= month <= 12
+ assert 1 <= day <= 31
+ assert 0 <= hour <= 23
+ assert 0 <= minute <= 59
+ assert 0 <= second <= 61 # leap seconds
+ timetuple = (year, month, day, hour, minute, second, -1, -1, -1)
+ if time.daylight:
+ utc_in_secs = time.mktime(timetuple) - time.altzone
+ else: utc_in_secs = time.mktime(timetuple) - time.timezone
+
+ return long(utc_in_secs) + tzdtoseconds(timestring[19:])
+ except (TypeError, ValueError, AssertionError): return None
+
+def timetopretty(timeinseconds):
+ """Return pretty version of time"""
+ return time.asctime(time.localtime(timeinseconds))
+
+def stringtopretty(timestring):
+ """Return pretty version of time given w3 time string"""
+ return timetopretty(stringtotime(timestring))
+
+def inttopretty(seconds):
+ """Convert num of seconds to readable string like "2 hours"."""
+ partlist = []
+ hours, seconds = divmod(seconds, 3600)
+ if hours > 1: partlist.append("%d hours" % hours)
+ elif hours == 1: partlist.append("1 hour")
+
+ minutes, seconds = divmod(seconds, 60)
+ if minutes > 1: partlist.append("%d minutes" % minutes)
+ elif minutes == 1: partlist.append("1 minute")
+
+ if seconds == 1: partlist.append("1 second")
+ elif not partlist or seconds > 1:
+ if isinstance(seconds, int) or isinstance(seconds, long):
+ partlist.append("%s seconds" % seconds)
+ else: partlist.append("%.2f seconds" % seconds)
+ return " ".join(partlist)
+
+def intstringtoseconds(interval_string):
+ """Convert a string expressing an interval (e.g. "4D2s") to seconds"""
+ def error():
+ raise TimeException("""Bad interval string "%s"
+
+Intervals are specified like 2Y (2 years) or 2h30m (2.5 hours). The
+allowed special characters are s, m, h, D, W, M, and Y. See the man
+page for more information.
+""" % interval_string)
+ if len(interval_string) < 2: error()
+
+ total = 0
+ while interval_string:
+ match = _interval_regexp.match(interval_string)
+ if not match: error()
+ num, ext = int(match.group(1)), match.group(2)
+ if not ext in _interval_conv_dict or num < 0: error()
+ total += num*_interval_conv_dict[ext]
+ interval_string = interval_string[match.end(0):]
+ return total
+
+def gettzd():
+ """Return w3's timezone identification string.
+
+ Expresed as [+/-]hh:mm. For instance, PST is -08:00. Zone is
+ coincides with what localtime(), etc., use.
+
+ """
+ if time.daylight: offset = -1 * time.altzone/60
+ else: offset = -1 * time.timezone/60
+ if offset > 0: prefix = "+"
+ elif offset < 0: prefix = "-"
+ else: return "Z" # time is already in UTC
+
+ hours, minutes = map(abs, divmod(offset, 60))
+ assert 0 <= hours <= 23
+ assert 0 <= minutes <= 59
+ return "%s%02d%s%02d" % (prefix, hours,
+ Globals.time_separator, minutes)
+
+def tzdtoseconds(tzd):
+ """Given w3 compliant TZD, return how far ahead UTC is"""
+ if tzd == "Z": return 0
+ assert len(tzd) == 6 # only accept forms like +08:00 for now
+ assert (tzd[0] == "-" or tzd[0] == "+") and \
+ tzd[3] == Globals.time_separator
+ return -60 * (60 * int(tzd[:3]) + int(tzd[4:]))
+
+def cmp(time1, time2):
+ """Compare time1 and time2 and return -1, 0, or 1"""
+ if type(time1) is types.StringType:
+ time1 = stringtotime(time1)
+ assert time1 is not None
+ if type(time2) is types.StringType:
+ time2 = stringtotime(time2)
+ assert time2 is not None
+
+ if time1 < time2: return -1
+ elif time1 == time2: return 0
+ else: return 1
+
+def genstrtotime(timestr, curtime = None):
+ """Convert a generic time string to a time in seconds"""
+ if curtime is None: curtime = globals()['curtime']
+ if timestr == "now": return curtime
+
+ def error():
+ raise TimeException("""Bad time string "%s"
+
+The acceptible time strings are intervals (like "3D64s"), w3-datetime
+strings, like "2002-04-26T04:22:01-07:00" (strings like
+"2002-04-26T04:22:01" are also acceptable - rdiff-backup will use the
+current time zone), or ordinary dates like 2/4/1997 or 2001-04-23
+(various combinations are acceptable, but the month always precedes
+the day).""" % timestr)
+
+ # Test for straight integer
+ if _integer_regexp.search(timestr): return int(timestr)
+
+ # Test for w3-datetime format, possibly missing tzd
+ t = stringtotime(timestr) or stringtotime(timestr+gettzd())
+ if t: return t
+
+ try: # test for an interval, like "2 days ago"
+ return curtime - intstringtoseconds(timestr)
+ except TimeException: pass
+
+ # Now check for dates like 2001/3/23
+ match = _genstr_date_regexp1.search(timestr) or \
+ _genstr_date_regexp2.search(timestr)
+ if not match: error()
+ timestr = "%s-%02d-%02dT00:00:00%s" % (match.group('year'),
+ int(match.group('month')), int(match.group('day')), gettzd())
+ t = stringtotime(timestr)
+ if t: return t
+ else: error()
+
+
diff --git a/rdiff-backup/src/connection.py b/rdiff-backup/src/connection.py
index deff577..74d413d 100644
--- a/rdiff-backup/src/connection.py
+++ b/rdiff-backup/src/connection.py
@@ -1,5 +1,4 @@
from __future__ import generators
-execfile("rdiff.py")
import types, os, tempfile, cPickle, shutil, traceback
#######################################################################
@@ -38,10 +37,9 @@ class LocalConnection(Connection):
self.conn_number = 0 # changed by SetConnections for server
def __getattr__(self, name):
- try: return globals()[name]
- except KeyError:
- try: return __builtins__.__dict__[name]
- except KeyError: raise NameError, name
+ if name in globals(): return globals()[name]
+ elif isinstance(__builtins__, dict): return __builtins__[name]
+ else: return __builtins__.__dict__[name]
def __setattr__(self, name, value):
globals()[name] = value
@@ -56,11 +54,6 @@ class LocalConnection(Connection):
def quit(self): pass
-Globals.local_connection = LocalConnection()
-Globals.connections.append(Globals.local_connection)
-# Following changed by server in SetConnections
-Globals.connection_dict[0] = Globals.local_connection
-
class ConnectionRequest:
"""Simple wrapper around a PipeConnection request"""
@@ -493,3 +486,30 @@ class VirtualFile:
line = self.readline()
if not line: break
yield line
+
+
+# everything has to be available here for remote connection's use, but
+# put at bottom to reduce circularities.
+import Globals, Time, Rdiff, Hardlink, FilenameMapping
+from static import *
+from lazy import *
+from log import *
+from iterfile import *
+from connection import *
+from rpath import *
+from robust import *
+from rorpiter import *
+from destructive_stepping import *
+from selection import *
+from statistics import *
+from increment import *
+from restore import *
+from manage import *
+from highlevel import *
+
+
+Globals.local_connection = LocalConnection()
+Globals.connections.append(Globals.local_connection)
+# Following changed by server in SetConnections
+Globals.connection_dict[0] = Globals.local_connection
+
diff --git a/rdiff-backup/src/destructive_stepping.py b/rdiff-backup/src/destructive_stepping.py
index 7dfde11..a64ecbc 100644
--- a/rdiff-backup/src/destructive_stepping.py
+++ b/rdiff-backup/src/destructive_stepping.py
@@ -1,6 +1,7 @@
from __future__ import generators
import types
-execfile("rorpiter.py")
+from rpath import *
+from lazy import *
#######################################################################
#
@@ -206,3 +207,6 @@ class DestructiveSteppingFinalizer(ErrorITR):
if self.dsrpath: self.dsrpath.write_changes()
+from log import *
+from robust import *
+import Globals
diff --git a/rdiff-backup/src/filename_mapping.py b/rdiff-backup/src/filename_mapping.py
deleted file mode 100644
index b8110be..0000000
--- a/rdiff-backup/src/filename_mapping.py
+++ /dev/null
@@ -1,93 +0,0 @@
-execfile("selection.py")
-import re
-
-#######################################################################
-#
-# filename_mapping - used to coordinate related filenames
-#
-# For instance, some source filenames may contain characters not
-# allowed on the mirror end. Also, if a source filename is very long
-# (say 240 characters), the extra characters added to related
-# increments may put them over the usual 255 character limit.
-#
-
-class FilenameMapping:
- """Contains class methods which coordinate related filenames"""
- max_filename_length = 255
-
- # If true, enable character quoting, and set characters making
- # regex-style range.
- chars_to_quote = None
-
- # These compiled regular expressions are used in quoting and unquoting
- chars_to_quote_regexp = None
- unquoting_regexp = None
-
- # Use given char to quote. Default is set in Globals.
- quoting_char = None
-
- def set_init_quote_vals(cls):
- """Set quoting value from Globals on all conns"""
- for conn in Globals.connections:
- conn.FilenameMapping.set_init_quote_vals_local()
-
- def set_init_quote_vals_local(cls):
- """Set value on local connection, initialize regexps"""
- cls.chars_to_quote = Globals.chars_to_quote
- if len(Globals.quoting_char) != 1:
- Log.FatalError("Expected single character for quoting char,"
- "got '%s' instead" % (Globals.quoting_char,))
- cls.quoting_char = Globals.quoting_char
- cls.init_quoting_regexps()
-
- def init_quoting_regexps(cls):
- """Compile quoting regular expressions"""
- try:
- cls.chars_to_quote_regexp = \
- re.compile("[%s%s]" % (cls.chars_to_quote,
- cls.quoting_char), re.S)
- cls.unquoting_regexp = \
- re.compile("%s[0-9]{3}" % cls.quoting_char, re.S)
- except re.error:
- Log.FatalError("Error '%s' when processing char quote list %s" %
- (re.error, cls.chars_to_quote))
-
- def quote(cls, path):
- """Return quoted version of given path
-
- Any characters quoted will be replaced by the quoting char and
- the ascii number of the character. For instance, "10:11:12"
- would go to "10;05811;05812" if ":" were quoted and ";" were
- the quoting character.
-
- """
- return cls.chars_to_quote_regexp.sub(cls.quote_single, path)
-
- def quote_single(cls, match):
- """Return replacement for a single character"""
- return "%s%03d" % (cls.quoting_char, ord(match.group()))
-
- def unquote(cls, path):
- """Return original version of quoted filename"""
- return cls.unquoting_regexp.sub(cls.unquote_single, path)
-
- def unquote_single(cls, match):
- """Unquote a single quoted character"""
- assert len(match.group()) == 4
- return chr(int(match.group()[1:]))
-
- def get_quoted_dir_children(cls, rpath):
- """For rpath directory, return list of quoted children in dir"""
- if not rpath.isdir(): return []
- dir_pairs = [(cls.unquote(filename), filename)
- for filename in Robust.listrp(rpath)]
- dir_pairs.sort() # sort by real index, not quoted part
- child_list = []
- for unquoted, filename in dir_pairs:
- childrp = rpath.append(unquoted)
- childrp.quote_path()
- child_list.append(childrp)
- return child_list
-
-MakeClass(FilenameMapping)
-
diff --git a/rdiff-backup/src/globals.py b/rdiff-backup/src/globals.py
deleted file mode 100644
index 17345d3..0000000
--- a/rdiff-backup/src/globals.py
+++ /dev/null
@@ -1,237 +0,0 @@
-import re, os
-
-#######################################################################
-#
-# globals - aggregate some configuration options
-#
-
-class Globals:
-
- # The current version of rdiff-backup
- version = "0.8.0"
-
- # If this is set, use this value in seconds as the current time
- # instead of reading it from the clock.
- current_time = None
-
- # This determines how many bytes to read at a time when copying
- blocksize = 32768
-
- # This is used by the BufferedRead class to determine how many
- # bytes to request from the underlying file per read(). Larger
- # values may save on connection overhead and latency.
- conn_bufsize = 98304
-
- # True if script is running as a server
- server = None
-
- # uid and gid of the owner of the rdiff-backup process. This can
- # vary depending on the connection.
- process_uid = os.getuid()
- process_gid = os.getgid()
-
- # If true, when copying attributes, also change target's uid/gid
- change_ownership = None
-
- # If true, change the permissions of unwriteable mirror files
- # (such as directories) so that they can be written, and then
- # change them back. This defaults to 1 just in case the process
- # is not running as root (root doesn't need to change
- # permissions).
- change_mirror_perms = (process_uid != 0)
-
- # If true, temporarily change permissions of unreadable files in
- # the source directory to make sure we can read all files.
- change_source_perms = None
-
- # If true, try to reset the atimes of the source partition.
- preserve_atime = None
-
- # This will be set as soon as the LocalConnection class loads
- local_connection = None
-
- # All connections should be added to the following list, so
- # further global changes can be propagated to the remote systems.
- # The first element should be Globals.local_connection. For a
- # server, the second is the connection to the client.
- connections = []
-
- # Each process should have a connection number unique to the
- # session. The client has connection number 0.
- connection_number = 0
-
- # Dictionary pairing connection numbers with connections. Set in
- # SetConnections for all connections.
- connection_dict = {}
-
- # True if the script is the end that reads the source directory
- # for backups. It is true for purely local sessions.
- isbackup_reader = None
-
- # Connection of the real backup reader (for which isbackup_reader
- # is true)
- backup_reader = None
-
- # True if the script is the end that writes to the increment and
- # mirror directories. True for purely local sessions.
- isbackup_writer = None
-
- # Connection of the backup writer
- backup_writer = None
-
- # True if this process is the client invoked by the user
- isclient = None
-
- # Connection of the client
- client_conn = None
-
- # This list is used by the set function below. When a new
- # connection is created with init_connection, its Globals class
- # will match this one for all the variables mentioned in this
- # list.
- changed_settings = []
-
- # rdiff-backup will try to checkpoint its state every
- # checkpoint_interval seconds. Then when resuming, at most this
- # amount of time is lost.
- checkpoint_interval = 20
-
- # The RPath of the rdiff-backup-data directory.
- rbdir = None
-
- # Indicates if a resume or a lack of resume is forced. This
- # should be None for the default. 0 means don't resume, and 1
- # means resume.
- resume = None
-
- # If there has been an aborted backup fewer than this many seconds
- # ago, attempt to resume it where it left off instead of starting
- # a new one.
- resume_window = 7200
-
- # This string is used when recognizing and creating time strings.
- # If the time_separator is ":", then W3 datetime strings like
- # 2001-12-07T04:22:01-07:00 are produced. It can be set to "_" to
- # make filenames that don't contain colons, which aren't allowed
- # under MS windows NT.
- time_separator = ":"
-
- # quoting_enabled is true if we should quote certain characters in
- # filenames on the source side (see FilenameMapping for more
- # info). chars_to_quote is a string whose characters should be
- # quoted, and quoting_char is the character to quote with.
- quoting_enabled = None
- chars_to_quote = ""
- quoting_char = ';'
-
- # If true, emit output intended to be easily readable by a
- # computer. False means output is intended for humans.
- parsable_output = None
-
- # If true, then hardlinks will be preserved to mirror and recorded
- # in the increments directory. There is also a difference here
- # between None and 0. When restoring, None or 1 means to preserve
- # hardlinks iff can find a hardlink dictionary. 0 means ignore
- # hardlink information regardless.
- preserve_hardlinks = 1
-
- # If this is false, then rdiff-backup will not compress any
- # increments. Default is to compress based on regexp below.
- compression = 1
-
- # Increments based on files whose names match this
- # case-insensitive regular expression won't be compressed (applies
- # to .snapshots and .diffs). The second below will be the
- # compiled version of the first.
- no_compression_regexp_string = "(?i).*\\.(gz|z|bz|bz2|tgz|zip|rpm|deb|" \
- "jpg|gif|png|jp2|mp3|ogg|avi|wmv|mpeg|mpg|rm|mov)$"
- no_compression_regexp = None
-
- # If true, filelists and directory statistics will be split on
- # nulls instead of newlines.
- null_separator = None
-
- # Determines whether or not ssh will be run with the -C switch
- ssh_compression = 1
-
- # If true, print statistics after successful backup
- print_statistics = None
-
- # On the reader and writer connections, the following will be
- # replaced by the source and mirror Select objects respectively.
- select_source, select_mirror = None, None
-
- # On the backup writer connection, holds the main incrementing
- # function. Access is provided to increment error counts.
- ITR = None
-
- def get(cls, name):
- """Return the value of something in this class"""
- return cls.__dict__[name]
- get = classmethod(get)
-
- def is_not_None(cls, name):
- """Returns true if value is not None"""
- return cls.__dict__[name] is not None
- is_not_None = classmethod(is_not_None)
-
- def set(cls, name, val):
- """Set the value of something in this class
-
- Use this instead of writing the values directly if the setting
- matters to remote sides. This function updates the
- changed_settings list, so other connections know to copy the
- changes.
-
- """
- cls.changed_settings.append(name)
- cls.__dict__[name] = val
- set = classmethod(set)
-
- def set_integer(cls, name, val):
- """Like set, but make sure val is an integer"""
- try: intval = int(val)
- except ValueError:
- Log.FatalError("Variable %s must be set to an integer -\n"
- "received %s instead." % (name, val))
- cls.set(name, intval)
- set_integer = classmethod(set_integer)
-
- def get_dict_val(cls, name, key):
- """Return val from dictionary in this class"""
- return cls.__dict__[name][key]
- get_dict_val = classmethod(get_dict_val)
-
- def set_dict_val(cls, name, key, val):
- """Set value for dictionary in this class"""
- cls.__dict__[name][key] = val
- set_dict_val = classmethod(set_dict_val)
-
- def postset_regexp(cls, name, re_string, flags = None):
- """Compile re_string on all existing connections, set to name"""
- for conn in Globals.connections:
- conn.Globals.postset_regexp_local(name, re_string, flags)
- postset_regexp = classmethod(postset_regexp)
-
- def postset_regexp_local(cls, name, re_string, flags):
- """Set name to compiled re_string locally"""
- if flags: cls.__dict__[name] = re.compile(re_string, flags)
- else: cls.__dict__[name] = re.compile(re_string)
- postset_regexp_local = classmethod(postset_regexp_local)
-
- def set_select(cls, dsrpath, tuplelist, quote_mode, *filelists):
- """Initialize select object using tuplelist
-
- Note that each list in filelists must each be passed as
- separate arguments, so each is recognized as a file by the
- connection. Otherwise we will get an error because a list
- containing files can't be pickled.
-
- """
- if dsrpath.source:
- cls.select_source = Select(dsrpath, quote_mode)
- cls.select_source.ParseArgs(tuplelist, filelists)
- else:
- cls.select_mirror = Select(dsrpath, quote_mode)
- cls.select_mirror.ParseArgs(tuplelist, filelists)
- set_select = classmethod(set_select)
diff --git a/rdiff-backup/src/hardlink.py b/rdiff-backup/src/hardlink.py
deleted file mode 100644
index 431207d..0000000
--- a/rdiff-backup/src/hardlink.py
+++ /dev/null
@@ -1,257 +0,0 @@
-execfile("rpath.py")
-
-#######################################################################
-#
-# hardlink - code for preserving and restoring hardlinks
-#
-# If the preserve_hardlinks option is selected, linked files in the
-# source directory will be linked in the mirror directory. Linked
-# files are treated like any other with respect to incrementing, but a
-# database of all links will be recorded at each session, so linked
-# files can still be restored from the increments.
-#
-
-class Hardlink:
- """Hardlink class methods and data
-
- All these functions are meant to be executed on the destination
- side. The source side should only transmit inode information.
-
- """
- # In all of these lists of indicies are the values. The keys in
- # _inode_ ones are (inode, devloc) pairs.
- _src_inode_indicies = {}
- _dest_inode_indicies = {}
-
- # The keys for these two are just indicies. They share values
- # with the earlier dictionaries.
- _src_index_indicies = {}
- _dest_index_indicies = {}
-
- # When a linked file is restored, its path is added to this dict,
- # so it can be found when later paths being restored are linked to
- # it.
- _restore_index_path = {}
-
- def get_inode_key(cls, rorp):
- """Return rorp's key for _inode_ dictionaries"""
- return (rorp.getinode(), rorp.getdevloc())
-
- def get_indicies(cls, rorp, source):
- """Return a list of similarly linked indicies, using rorp's index"""
- if source: dict = cls._src_index_indicies
- else: dict = cls._dest_index_indicies
- try: return dict[rorp.index]
- except KeyError: return []
-
- def add_rorp(cls, rorp, source):
- """Process new rorp and update hard link dictionaries
-
- First enter it into src_inode_indicies. If we have already
- seen all the hard links, then we can delete the entry.
- Everything must stay recorded in src_index_indicies though.
-
- """
- if not rorp.isreg() or rorp.getnumlinks() < 2: return
-
- if source: inode_dict, index_dict = (cls._src_inode_indicies,
- cls._src_index_indicies)
- else: inode_dict, index_dict = (cls._dest_inode_indicies,
- cls._dest_index_indicies)
-
- rp_inode_key = cls.get_inode_key(rorp)
- if inode_dict.has_key(rp_inode_key):
- index_list = inode_dict[rp_inode_key]
- index_list.append(rorp.index)
- if len(index_list) == rorp.getnumlinks():
- del inode_dict[rp_inode_key]
- else: # make new entry in both src dicts
- index_list = [rorp.index]
- inode_dict[rp_inode_key] = index_list
- index_dict[rorp.index] = index_list
-
- def add_rorp_iter(cls, iter, source):
- """Return new rorp iterator like iter that cls.add_rorp's first"""
- for rorp in iter:
- cls.add_rorp(rorp, source)
- yield rorp
-
- def rorp_eq(cls, src_rorp, dest_rorp):
- """Compare hardlinked for equality
-
- Two files may otherwise seem equal but be hardlinked in
- different ways. This function considers them equal enough if
- they have been hardlinked correctly to the previously seen
- indicies.
-
- """
- assert src_rorp.index == dest_rorp.index
- if (not src_rorp.isreg() or not dest_rorp.isreg() or
- src_rorp.getnumlinks() == dest_rorp.getnumlinks() == 1):
- return 1 # Hard links don't apply
-
- src_index_list = cls.get_indicies(src_rorp, 1)
- dest_index_list = cls.get_indicies(dest_rorp, None)
-
- # If a list only has one element, then it is only hardlinked
- # to itself so far, so that is not a genuine difference yet.
- if not src_index_list or len(src_index_list) == 1:
- return not dest_index_list or len(dest_index_list) == 1
- if not dest_index_list or len(dest_index_list) == 1: return None
-
- # Both index lists exist and are non-empty
- return src_index_list == dest_index_list # they are always sorted
-
- def islinked(cls, rorp):
- """True if rorp's index is already linked to something on src side"""
- return len(cls.get_indicies(rorp, 1)) >= 2
-
- def restore_link(cls, index, rpath):
- """Restores a linked file by linking it
-
- When restoring, all the hardlink data is already present, and
- we can only link to something already written. In either
- case, add to the _restore_index_path dict, so we know later
- that the file is available for hard
- linking.
-
- Returns true if succeeded in creating rpath, false if must
- restore rpath normally.
-
- """
- if index not in cls._src_index_indicies: return None
- for linked_index in cls._src_index_indicies[index]:
- if linked_index in cls._restore_index_path:
- srcpath = cls._restore_index_path[linked_index]
- Log("Restoring %s by hard linking to %s" %
- (rpath.path, srcpath), 6)
- rpath.hardlink(srcpath)
- return 1
- cls._restore_index_path[index] = rpath.path
- return None
-
- def link_rp(cls, src_rorp, dest_rpath, dest_root = None):
- """Make dest_rpath into a link analogous to that of src_rorp"""
- if not dest_root: dest_root = dest_rpath # use base of dest_rpath
- dest_link_rpath = RPath(dest_root.conn, dest_root.base,
- cls.get_indicies(src_rorp, 1)[0])
- dest_rpath.hardlink(dest_link_rpath.path)
-
- def write_linkdict(cls, rpath, dict, compress = None):
- """Write link data to the rbdata dir
-
- It is stored as the a big pickled dictionary dated to match
- the current hardlinks.
-
- """
- assert (Globals.isbackup_writer and
- rpath.conn is Globals.local_connection)
- tf = TempFileManager.new(rpath)
- def init():
- fp = tf.open("wb", compress)
- cPickle.dump(dict, fp)
- assert not fp.close()
- tf.setdata()
- Robust.make_tf_robustaction(init, (tf,), (rpath,)).execute()
-
- def get_linkrp(cls, data_rpath, time, prefix):
- """Return RPath of linkdata, or None if cannot find"""
- for rp in map(data_rpath.append, data_rpath.listdir()):
- if (rp.isincfile() and rp.getincbase_str() == prefix and
- (rp.getinctype() == 'snapshot' or rp.getinctype() == 'data')
- and Time.stringtotime(rp.getinctime()) == time):
- return rp
- return None
-
- def get_linkdata(cls, data_rpath, time, prefix = 'hardlink_data'):
- """Return index dictionary written by write_linkdata at time"""
- rp = cls.get_linkrp(data_rpath, time, prefix)
- if not rp: return None
- fp = rp.open("rb", rp.isinccompressed())
- index_dict = cPickle.load(fp)
- assert not fp.close()
- return index_dict
-
- def final_writedata(cls):
- """Write final checkpoint data to rbdir after successful backup"""
- if not cls._src_index_indicies: # no hardlinks, so writing unnecessary
- cls.final_inc = None
- return
- Log("Writing hard link data", 6)
- if Globals.compression:
- cls.final_inc = Globals.rbdir.append("hardlink_data.%s.data.gz" %
- Time.curtimestr)
- else: cls.final_inc = Globals.rbdir.append("hardlink_data.%s.data" %
- Time.curtimestr)
- cls.write_linkdict(cls.final_inc,
- cls._src_index_indicies, Globals.compression)
-
- def retrieve_final(cls, time):
- """Set source index dictionary from hardlink_data file if avail"""
- hd = cls.get_linkdata(Globals.rbdir, time)
- if hd is None: return None
- cls._src_index_indicies = hd
- return 1
-
- def final_checkpoint(cls, data_rpath):
- """Write contents of the four dictionaries to the data dir
-
- If rdiff-backup receives a fatal error, it may still be able
- to save the contents of the four hard link dictionaries.
- Because these dictionaries may be big, they are not saved
- after every 20 seconds or whatever, but just at the end.
-
- """
- Log("Writing intermediate hard link data to disk", 2)
- src_inode_rp = data_rpath.append("hardlink_source_inode_checkpoint."
- "%s.data" % Time.curtimestr)
- src_index_rp = data_rpath.append("hardlink_source_index_checkpoint."
- "%s.data" % Time.curtimestr)
- dest_inode_rp = data_rpath.append("hardlink_dest_inode_checkpoint."
- "%s.data" % Time.curtimestr)
- dest_index_rp = data_rpath.append("hardlink_dest_index_checkpoint."
- "%s.data" % Time.curtimestr)
- for (rp, dict) in ((src_inode_rp, cls._src_inode_indicies),
- (src_index_rp, cls._src_index_indicies),
- (dest_inode_rp, cls._dest_inode_indicies),
- (dest_index_rp, cls._dest_index_indicies)):
- cls.write_linkdict(rp, dict)
-
- def retrieve_checkpoint(cls, data_rpath, time):
- """Retrieve hardlink data from final checkpoint
-
- Return true if the retrieval worked, false otherwise.
-
- """
- try:
- src_inode = cls.get_linkdata(data_rpath, time,
- "hardlink_source_inode_checkpoint")
- src_index = cls.get_linkdata(data_rpath, time,
- "hardlink_source_index_checkpoint")
- dest_inode = cls.get_linkdata(data_rpath, time,
- "hardlink_dest_inode_checkpoint")
- dest_index = cls.get_linkdata(data_rpath, time,
- "hardlink_dest_index_checkpoint")
- except cPickle.UnpicklingError:
- Log("Unpickling Error", 2)
- return None
- if (src_inode is None or src_index is None or
- dest_inode is None or dest_index is None): return None
- cls._src_inode_indicies = src_inode
- cls._src_index_indicies = src_index
- cls._dest_inode_indicies = dest_inode
- cls._dest_index_indicies = dest_index
- return 1
-
- def remove_all_checkpoints(cls):
- """Remove all hardlink checkpoint information from directory"""
- prefix_list = ["hardlink_source_inode_checkpoint",
- "hardlink_source_index_checkpoint",
- "hardlink_dest_inode_checkpoint",
- "hardlink_dest_index_checkpoint"]
- for rp in map(Globals.rbdir.append, Globals.rbdir.listdir()):
- if (rp.isincfile() and rp.getincbase_str() in prefix_list and
- (rp.getinctype() == 'snapshot' or rp.getinctype() == 'data')):
- rp.delete()
-
-MakeClass(Hardlink)
diff --git a/rdiff-backup/src/highlevel.py b/rdiff-backup/src/highlevel.py
index 8c95a1f..0b477d8 100644
--- a/rdiff-backup/src/highlevel.py
+++ b/rdiff-backup/src/highlevel.py
@@ -1,5 +1,12 @@
from __future__ import generators
-execfile("manage.py")
+from static import *
+from log import *
+from rpath import *
+from robust import *
+from increment import *
+from destructive_stepping import *
+from rorpiter import *
+import Globals, Hardlink, MiscStats
#######################################################################
#
@@ -248,7 +255,7 @@ class HLDestinationStruct:
"""Apply diffs and finalize, with checkpointing and statistics"""
collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath)
- Stats.open_dir_stats_file()
+ MiscStats.open_dir_stats_file()
dsrp, finished_dsrp = None, None
try:
@@ -266,15 +273,15 @@ class HLDestinationStruct:
except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
if Globals.preserve_hardlinks: Hardlink.final_writedata()
- Stats.close_dir_stats_file()
- Stats.write_session_statistics(ITR)
+ MiscStats.close_dir_stats_file()
+ MiscStats.write_session_statistics(ITR)
SaveState.checkpoint_remove()
def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath):
"""Apply diffs, write increment if necessary, and finalize"""
collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath)
- Stats.open_dir_stats_file()
+ MiscStats.open_dir_stats_file()
dsrp, finished_dsrp = None, None
try:
@@ -293,8 +300,8 @@ class HLDestinationStruct:
except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
if Globals.preserve_hardlinks: Hardlink.final_writedata()
- Stats.close_dir_stats_file()
- Stats.write_session_statistics(ITR)
+ MiscStats.close_dir_stats_file()
+ MiscStats.write_session_statistics(ITR)
SaveState.checkpoint_remove()
def handle_last_error(cls, dsrp, finalizer, ITR):
diff --git a/rdiff-backup/src/increment.py b/rdiff-backup/src/increment.py
index b03b464..d5543a0 100644
--- a/rdiff-backup/src/increment.py
+++ b/rdiff-backup/src/increment.py
@@ -1,5 +1,7 @@
import traceback
-execfile("statistics.py")
+from static import *
+from statistics import *
+from lazy import *
#######################################################################
#
@@ -256,7 +258,7 @@ class IncrementITR(ErrorITR, StatsITR):
self.end_stats(diff_rorp, dsrp, self.incrp)
if self.mirror_isdirectory or dsrp.isdir():
- Stats.write_dir_stats_line(self, dsrp.index)
+ MiscStats.write_dir_stats_line(self, dsrp.index)
def branch_process(self, subinstance):
"""Update statistics, and the has_changed flag if change in branch"""
@@ -286,8 +288,15 @@ class MirrorITR(ErrorITR, StatsITR):
"""Update statistics when leaving"""
self.end_stats(self.diff_rorp, self.mirror_dsrp)
if self.mirror_dsrp.isdir():
- Stats.write_dir_stats_line(self, self.mirror_dsrp.index)
+ MiscStats.write_dir_stats_line(self, self.mirror_dsrp.index)
def branch_process(self, subinstance):
"""Update statistics with subdirectory results"""
self.add_file_stats(subinstance)
+
+
+from log import *
+from rpath import *
+from robust import *
+from rorpiter import *
+import Globals, Time, MiscStats
diff --git a/rdiff-backup/src/iterfile.py b/rdiff-backup/src/iterfile.py
index 21629b2..26cc952 100644
--- a/rdiff-backup/src/iterfile.py
+++ b/rdiff-backup/src/iterfile.py
@@ -1,5 +1,5 @@
-execfile("ttime.py")
import cPickle
+import Globals
#######################################################################
#
diff --git a/rdiff-backup/src/lazy.py b/rdiff-backup/src/lazy.py
index 425a9c4..1eb0211 100644
--- a/rdiff-backup/src/lazy.py
+++ b/rdiff-backup/src/lazy.py
@@ -1,6 +1,6 @@
from __future__ import generators
-execfile("static.py")
import os, stat, types
+from static import *
#######################################################################
#
@@ -324,3 +324,7 @@ class ErrorITR(IterTreeReducer):
Log("Error '%s' processing %s" % (exc, filename), 2)
+# Put at bottom to prevent (viciously) circular module dependencies
+from robust import *
+from log import *
+
diff --git a/rdiff-backup/src/log.py b/rdiff-backup/src/log.py
index 6b4ba06..aade607 100644
--- a/rdiff-backup/src/log.py
+++ b/rdiff-backup/src/log.py
@@ -1,5 +1,4 @@
-import time, sys, traceback
-execfile("lazy.py")
+import time, sys, traceback, types
#######################################################################
#
@@ -132,7 +131,7 @@ class Logger:
def FatalError(self, message):
self("Fatal Error: " + message, 1)
- Globals.Main.cleanup()
+ Main.cleanup()
sys.exit(1)
def exception_to_string(self):
@@ -158,3 +157,4 @@ class Logger:
logging_func(self.exception_to_string(), verbosity)
Log = Logger()
+import Globals, Main
diff --git a/rdiff-backup/src/main.py b/rdiff-backup/src/main.py
deleted file mode 100755
index e8a15af..0000000
--- a/rdiff-backup/src/main.py
+++ /dev/null
@@ -1,498 +0,0 @@
-#!/usr/bin/python
-
-execfile("setconnections.py")
-import getopt, sys, re
-
-#######################################################################
-#
-# main - Start here: Read arguments, set global settings, etc.
-#
-
-class Main:
- def __init__(self):
- self.action = None
- self.remote_cmd, self.remote_schema = None, None
- self.force = None
- self.select_opts, self.select_mirror_opts = [], []
- self.select_files = []
-
- def parse_cmdlineoptions(self, arglist):
- """Parse argument list and set global preferences"""
- def sel_fl(filename):
- """Helper function for including/excluding filelists below"""
- try: return open(filename, "r")
- except IOError: Log.FatalError("Error opening file %s" % filename)
-
- try: optlist, self.args = getopt.getopt(arglist, "blmr:sv:V",
- ["backup-mode", "calculate-average",
- "change-source-perms", "chars-to-quote=",
- "checkpoint-interval=", "current-time=", "exclude=",
- "exclude-device-files", "exclude-filelist=",
- "exclude-filelist-stdin", "exclude-mirror=",
- "exclude-regexp=", "force", "include=",
- "include-filelist=", "include-filelist-stdin",
- "include-regexp=", "list-increments", "mirror-only",
- "no-compression", "no-compression-regexp=",
- "no-hard-links", "no-resume", "null-separator",
- "parsable-output", "print-statistics", "quoting-char=",
- "remote-cmd=", "remote-schema=", "remove-older-than=",
- "restore-as-of=", "resume", "resume-window=", "server",
- "ssh-no-compression", "terminal-verbosity=",
- "test-server", "verbosity", "version", "windows-mode",
- "windows-time-format"])
- except getopt.error, e:
- self.commandline_error("Bad commandline options: %s" % str(e))
-
- for opt, arg in optlist:
- if opt == "-b" or opt == "--backup-mode": self.action = "backup"
- elif opt == "--calculate-average":
- self.action = "calculate-average"
- elif opt == "--change-source-perms":
- Globals.set('change_source_perms', 1)
- elif opt == "--chars-to-quote":
- Globals.set('chars_to_quote', arg)
- Globals.set('quoting_enabled', 1)
- elif opt == "--checkpoint-interval":
- Globals.set_integer('checkpoint_interval', arg)
- elif opt == "--current-time":
- Globals.set_integer('current_time', arg)
- elif opt == "--exclude": self.select_opts.append((opt, arg))
- elif opt == "--exclude-device-files":
- self.select_opts.append((opt, arg))
- elif opt == "--exclude-filelist":
- self.select_opts.append((opt, arg))
- self.select_files.append(sel_fl(arg))
- elif opt == "--exclude-filelist-stdin":
- self.select_opts.append(("--exclude-filelist",
- "standard input"))
- self.select_files.append(sys.stdin)
- elif opt == "--exclude-mirror":
- self.select_mirror_opts.append(("--exclude", arg))
- elif opt == "--exclude-regexp": self.select_opts.append((opt, arg))
- elif opt == "--force": self.force = 1
- elif opt == "--include": self.select_opts.append((opt, arg))
- elif opt == "--include-filelist":
- self.select_opts.append((opt, arg))
- self.select_files.append(sel_fl(arg))
- elif opt == "--include-filelist-stdin":
- self.select_opts.append(("--include-filelist",
- "standard input"))
- self.select_files.append(sys.stdin)
- elif opt == "--include-regexp":
- self.select_opts.append((opt, arg))
- elif opt == "-l" or opt == "--list-increments":
- self.action = "list-increments"
- elif opt == "-m" or opt == "--mirror-only": self.action = "mirror"
- elif opt == "--no-compression": Globals.set("compression", None)
- elif opt == "--no-compression-regexp":
- Globals.set("no_compression_regexp_string", arg)
- elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0)
- elif opt == '--no-resume': Globals.resume = 0
- elif opt == "--null-separator": Globals.set("null_separator", 1)
- elif opt == "-r" or opt == "--restore-as-of":
- self.restore_timestr = arg
- self.action = "restore-as-of"
- elif opt == "--parsable-output": Globals.set('parsable_output', 1)
- elif opt == "--print-statistics":
- Globals.set('print_statistics', 1)
- elif opt == "--quoting-char":
- Globals.set('quoting_char', arg)
- Globals.set('quoting_enabled', 1)
- elif opt == "--remote-cmd": self.remote_cmd = arg
- elif opt == "--remote-schema": self.remote_schema = arg
- elif opt == "--remove-older-than":
- self.remove_older_than_string = arg
- self.action = "remove-older-than"
- elif opt == '--resume': Globals.resume = 1
- elif opt == '--resume-window':
- Globals.set_integer('resume_window', arg)
- elif opt == "-s" or opt == "--server": self.action = "server"
- elif opt == "--ssh-no-compression":
- Globals.set('ssh_compression', None)
- elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg)
- elif opt == "--test-server": self.action = "test-server"
- elif opt == "-V" or opt == "--version":
- print "rdiff-backup " + Globals.version
- sys.exit(0)
- elif opt == "-v" or opt == "--verbosity": Log.setverbosity(arg)
- elif opt == "--windows-mode":
- Globals.set('time_separator', "_")
- Globals.set('chars_to_quote', ":")
- Globals.set('quoting_enabled', 1)
- elif opt == '--windows-time-format':
- Globals.set('time_separator', "_")
- else: Log.FatalError("Unknown option %s" % opt)
-
- def set_action(self):
- """Check arguments and try to set self.action"""
- l = len(self.args)
- if not self.action:
- if l == 0: self.commandline_error("No arguments given")
- elif l == 1: self.action = "restore"
- elif l == 2:
- if RPath(Globals.local_connection, self.args[0]).isincfile():
- self.action = "restore"
- else: self.action = "backup"
- else: self.commandline_error("Too many arguments given")
-
- if l == 0 and self.action != "server" and self.action != "test-server":
- self.commandline_error("No arguments given")
- if l > 0 and self.action == "server":
- self.commandline_error("Too many arguments given")
- if l < 2 and (self.action == "backup" or self.action == "mirror" or
- self.action == "restore-as-of"):
- self.commandline_error("Two arguments are required "
- "(source, destination).")
- if l == 2 and (self.action == "list-increments" or
- self.action == "remove-older-than"):
- self.commandline_error("Only use one argument, "
- "the root of the backup directory")
- if l > 2 and self.action != "calculate-average":
- self.commandline_error("Too many arguments given")
-
- def commandline_error(self, message):
- sys.stderr.write("Error: %s\n" % message)
- sys.stderr.write("See the rdiff-backup manual page for instructions\n")
- sys.exit(1)
-
- def misc_setup(self, rps):
- """Set default change ownership flag, umask, relay regexps"""
- if ((len(rps) == 2 and rps[1].conn.os.getuid() == 0) or
- (len(rps) < 2 and os.getuid() == 0)):
- # Allow change_ownership if destination connection is root
- for conn in Globals.connections:
- conn.Globals.set('change_ownership', 1)
- for rp in rps: rp.setdata() # Update with userinfo
-
- os.umask(077)
- Time.setcurtime(Globals.current_time)
- FilenameMapping.set_init_quote_vals()
- Globals.set("isclient", 1)
- SetConnections.UpdateGlobal("client_conn", Globals.local_connection)
-
- # This is because I originally didn't think compiled regexps
- # could be pickled, and so must be compiled on remote side.
- Globals.postset_regexp('no_compression_regexp',
- Globals.no_compression_regexp_string)
-
- for conn in Globals.connections: Robust.install_signal_handlers()
-
- def take_action(self, rps):
- """Do whatever self.action says"""
- if self.action == "server":
- PipeConnection(sys.stdin, sys.stdout).Server()
- elif self.action == "backup": self.Backup(rps[0], rps[1])
- elif self.action == "restore": self.Restore(*rps)
- elif self.action == "restore-as-of": self.RestoreAsOf(rps[0], rps[1])
- elif self.action == "mirror": self.Mirror(rps[0], rps[1])
- elif self.action == "test-server": SetConnections.TestConnections()
- elif self.action == "list-increments": self.ListIncrements(rps[0])
- elif self.action == "remove-older-than": self.RemoveOlderThan(rps[0])
- elif self.action == "calculate-average": self.CalculateAverage(rps)
- else: raise AssertionError("Unknown action " + self.action)
-
- def cleanup(self):
- """Do any last minute cleaning before exiting"""
- Log("Cleaning up", 6)
- Log.close_logfile()
- if not Globals.server: SetConnections.CloseConnections()
-
- def Main(self, arglist):
- """Start everything up!"""
- self.parse_cmdlineoptions(arglist)
- self.set_action()
- rps = SetConnections.InitRPs(self.args,
- self.remote_schema, self.remote_cmd)
- self.misc_setup(rps)
- self.take_action(rps)
- self.cleanup()
-
-
- def Mirror(self, src_rp, dest_rp):
- """Turn dest_path into a copy of src_path"""
- Log("Mirroring %s to %s" % (src_rp.path, dest_rp.path), 5)
- self.mirror_check_paths(src_rp, dest_rp)
- # Since no "rdiff-backup-data" dir, use root of destination.
- SetConnections.UpdateGlobal('rbdir', dest_rp)
- SetConnections.BackupInitConnections(src_rp.conn, dest_rp.conn)
- HighLevel.Mirror(src_rp, dest_rp)
-
- def mirror_check_paths(self, rpin, rpout):
- """Check paths and return rpin, rpout"""
- if not rpin.lstat():
- Log.FatalError("Source directory %s does not exist" % rpin.path)
- if rpout.lstat() and not self.force:
- Log.FatalError(
-"""Destination %s exists so continuing could mess it up. Run
-rdiff-backup with the --force option if you want to mirror anyway.""" %
- rpout.path)
-
-
- def Backup(self, rpin, rpout):
- """Backup, possibly incrementally, src_path to dest_path."""
- SetConnections.BackupInitConnections(rpin.conn, rpout.conn)
- self.backup_init_select(rpin, rpout)
- self.backup_init_dirs(rpin, rpout)
- RSI = Globals.backup_writer.Resume.ResumeCheck()
- SaveState.init_filenames()
- if self.prevtime:
- Time.setprevtime(self.prevtime)
- HighLevel.Mirror_and_increment(rpin, rpout, self.incdir, RSI)
- else: HighLevel.Mirror(rpin, rpout, self.incdir, RSI)
- self.backup_touch_curmirror(rpin, rpout)
-
- def backup_init_select(self, rpin, rpout):
- """Create Select objects on source and dest connections"""
- rpin.conn.Globals.set_select(DSRPath(1, rpin), self.select_opts,
- None, *self.select_files)
- rpout.conn.Globals.set_select(DSRPath(None, rpout),
- self.select_mirror_opts, 1)
-
- def backup_init_dirs(self, rpin, rpout):
- """Make sure rpin and rpout are valid, init data dir and logging"""
- if rpout.lstat() and not rpout.isdir():
- if not self.force:
- Log.FatalError("Destination %s exists and is not a "
- "directory" % rpout.path)
- else:
- Log("Deleting %s" % rpout.path, 3)
- rpout.delete()
-
- if not rpin.lstat():
- Log.FatalError("Source directory %s does not exist" % rpin.path)
- elif not rpin.isdir():
- Log.FatalError("Source %s is not a directory" % rpin.path)
-
- self.datadir = rpout.append("rdiff-backup-data")
- SetConnections.UpdateGlobal('rbdir', self.datadir)
- self.incdir = RPath(rpout.conn, os.path.join(self.datadir.path,
- "increments"))
- self.prevtime = self.backup_get_mirrortime()
-
- if rpout.lstat():
- if rpout.isdir() and not rpout.listdir(): # rpout is empty dir
- rpout.chmod(0700) # just make sure permissions aren't too lax
- elif not self.datadir.lstat() and not self.force:
- Log.FatalError(
-"""Destination directory %s exists, but does not look like a
-rdiff-backup directory. Running rdiff-backup like this could mess up
-what is currently in it. If you want to overwrite it, run
-rdiff-backup with the --force option.""" % rpout.path)
-
- if not rpout.lstat():
- try: rpout.mkdir()
- except os.error:
- Log.FatalError("Unable to create directory %s" % rpout.path)
- if not self.datadir.lstat(): self.datadir.mkdir()
- if Log.verbosity > 0:
- Log.open_logfile(self.datadir.append("backup.log"))
- self.backup_warn_if_infinite_regress(rpin, rpout)
-
- def backup_warn_if_infinite_regress(self, rpin, rpout):
- """Warn user if destination area contained in source area"""
- if rpout.conn is rpin.conn: # it's meaningful to compare paths
- if ((len(rpout.path) > len(rpin.path)+1 and
- rpout.path[:len(rpin.path)] == rpin.path and
- rpout.path[len(rpin.path)] == '/') or
- (rpin.path == "." and rpout.path[0] != '/' and
- rpout.path[:2] != '..')):
- # Just a few heuristics, we don't have to get every case
- if Globals.backup_reader.Globals.select_source \
- .Select(rpout): Log(
-"""Warning: The destination directory '%s' may be contained in the
-source directory '%s'. This could cause an infinite regress. You
-may need to use the --exclude option.""" % (rpout.path, rpin.path), 2)
-
- def backup_get_mirrorrps(self):
- """Return list of current_mirror rps"""
- if not self.datadir.isdir(): return []
- mirrorrps = [self.datadir.append(fn) for fn in self.datadir.listdir()
- if fn.startswith("current_mirror.")]
- return filter(lambda rp: rp.isincfile(), mirrorrps)
-
- def backup_get_mirrortime(self):
- """Return time in seconds of previous mirror, or None if cannot"""
- mirrorrps = self.backup_get_mirrorrps()
- if not mirrorrps: return None
- if len(mirrorrps) > 1:
- Log(
-"""Warning: duplicate current_mirror files found. Perhaps something
-went wrong during your last backup? Using """ + mirrorrps[-1].path, 2)
-
- timestr = mirrorrps[-1].getinctime()
- return Time.stringtotime(timestr)
-
- def backup_touch_curmirror(self, rpin, rpout):
- """Make a file like current_mirror.time.data to record time
-
- Also updates rpout so mod times don't get messed up.
-
- """
- map(RPath.delete, self.backup_get_mirrorrps())
- mirrorrp = self.datadir.append("current_mirror.%s.%s" %
- (Time.curtimestr, "data"))
- Log("Touching mirror marker %s" % mirrorrp.path, 6)
- mirrorrp.touch()
- RPath.copy_attribs(rpin, rpout)
-
-
- def Restore(self, src_rp, dest_rp = None):
- """Main restoring function
-
- Here src_rp should be an increment file, and if dest_rp is
- missing it defaults to the base of the increment.
-
- """
- rpin, rpout = self.restore_check_paths(src_rp, dest_rp)
- time = Time.stringtotime(rpin.getinctime())
- self.restore_common(rpin, rpout, time)
-
- def RestoreAsOf(self, rpin, target):
- """Secondary syntax for restore operation
-
- rpin - RPath of mirror file to restore (not nec. with correct index)
- target - RPath of place to put restored file
-
- """
- self.restore_check_paths(rpin, target, 1)
- try: time = Time.genstrtotime(self.restore_timestr)
- except TimeException, exc: Log.FatalError(str(exc))
- self.restore_common(rpin, target, time)
-
- def restore_common(self, rpin, target, time):
- """Restore operation common to Restore and RestoreAsOf"""
- Log("Starting Restore", 5)
- mirror_root, index = self.restore_get_root(rpin)
- mirror = mirror_root.new_index(index)
- inc_rpath = self.datadir.append_path('increments', index)
- self.restore_init_select(mirror_root, target)
- Log.open_logfile(self.datadir.append("restore.log"))
- Restore.Restore(inc_rpath, mirror, target, time)
-
- def restore_check_paths(self, rpin, rpout, restoreasof = None):
- """Check paths and return pair of corresponding rps"""
- if not restoreasof:
- if not rpin.lstat():
- Log.FatalError("Source file %s does not exist" % rpin.path)
- elif not rpin.isincfile():
- Log.FatalError("""File %s does not look like an increment file.
-
-Try restoring from an increment file (the filenames look like
-"foobar.2001-09-01T04:49:04-07:00.diff").""" % rpin.path)
-
- if not rpout: rpout = RPath(Globals.local_connection,
- rpin.getincbase_str())
- if rpout.lstat():
- Log.FatalError("Restore target %s already exists, "
- "and will not be overwritten." % rpout.path)
- return rpin, rpout
-
- def restore_init_select(self, rpin, rpout):
- """Initialize Select
-
- Unlike the backup selections, here they are on the local
- connection, because the backup operation is pipelined in a way
- the restore operation isn't.
-
- """
- Globals.set_select(DSRPath(1, rpin), self.select_mirror_opts, None)
- Globals.set_select(DSRPath(None, rpout), self.select_opts, None,
- *self.select_files)
-
- def restore_get_root(self, rpin):
- """Return (mirror root, index) and set the data dir
-
- The idea here is to keep backing up on the path until we find
- a directory that contains "rdiff-backup-data". That is the
- mirror root. If the path from there starts
- "rdiff-backup-data/increments*", then the index is the
- remainder minus that. Otherwise the index is just the path
- minus the root.
-
- All this could fail if the increment file is pointed to in a
- funny way, using symlinks or somesuch.
-
- """
- if rpin.isincfile(): relpath = rpin.getincbase().path
- else: relpath = rpin.path
- pathcomps = os.path.join(rpin.conn.os.getcwd(), relpath).split("/")
- assert len(pathcomps) >= 2 # path should be relative to /
-
- i = len(pathcomps)
- while i >= 2:
- parent_dir = RPath(rpin.conn, "/".join(pathcomps[:i]))
- if (parent_dir.isdir() and
- "rdiff-backup-data" in parent_dir.listdir()): break
- i = i-1
- else: Log.FatalError("Unable to find rdiff-backup-data directory")
-
- self.rootrp = rootrp = parent_dir
- Log("Using mirror root directory %s" % rootrp.path, 6)
-
- self.datadir = rootrp.append_path("rdiff-backup-data")
- SetConnections.UpdateGlobal('rbdir', self.datadir)
- if not self.datadir.isdir():
- Log.FatalError("Unable to read rdiff-backup-data directory %s" %
- self.datadir.path)
-
- from_datadir = tuple(pathcomps[i:])
- if not from_datadir or from_datadir[0] != "rdiff-backup-data":
- return (rootrp, from_datadir) # in mirror, not increments
- assert from_datadir[1] == "increments"
- return (rootrp, from_datadir[2:])
-
-
- def ListIncrements(self, rp):
- """Print out a summary of the increments and their times"""
- mirror_root, index = self.restore_get_root(rp)
- Globals.rbdir = datadir = \
- mirror_root.append_path("rdiff-backup-data")
- mirrorrp = mirror_root.new_index(index)
- inc_rpath = datadir.append_path('increments', index)
- incs = Restore.get_inclist(inc_rpath)
- mirror_time = Restore.get_mirror_time()
- if Globals.parsable_output:
- print Manage.describe_incs_parsable(incs, mirror_time, mirrorrp)
- else: print Manage.describe_incs_human(incs, mirror_time, mirrorrp)
-
-
- def CalculateAverage(self, rps):
- """Print out the average of the given statistics files"""
- statobjs = map(lambda rp: StatsObj().read_stats_from_rp(rp), rps)
- average_stats = StatsObj().set_to_average(statobjs)
- print average_stats.get_stats_logstring(
- "Average of %d stat files" % len(rps))
-
-
- def RemoveOlderThan(self, rootrp):
- """Remove all increment files older than a certain time"""
- datadir = rootrp.append("rdiff-backup-data")
- if not datadir.lstat() or not datadir.isdir():
- Log.FatalError("Unable to open rdiff-backup-data dir %s" %
- (datadir.path,))
-
- try: time = Time.genstrtotime(self.remove_older_than_string)
- except TimeError, exc: Log.FatalError(str(exc))
- timep = Time.timetopretty(time)
- Log("Deleting increment(s) before %s" % timep, 4)
-
- itimes = [Time.stringtopretty(inc.getinctime())
- for inc in Restore.get_inclist(datadir.append("increments"))
- if Time.stringtotime(inc.getinctime()) < time]
-
- if not itimes:
- Log.FatalError("No increments older than %s found" % timep)
- inc_pretty_time = "\n".join(itimes)
- if len(itimes) > 1 and not self.force:
- Log.FatalError("Found %d relevant increments, dated:\n%s"
- "\nIf you want to delete multiple increments in this way, "
- "use the --force." % (len(itimes), inc_pretty_time))
-
- Log("Deleting increment%sat times:\n%s" %
- (len(itimes) == 1 and " " or "s ", inc_pretty_time), 3)
- Manage.delete_earlier_than(datadir, time)
-
-
-Globals.Main = Main()
-if __name__ == "__main__" and not globals().has_key('__no_execute__'):
- Globals.Main.Main(sys.argv[1:])
diff --git a/rdiff-backup/src/manage.py b/rdiff-backup/src/manage.py
index 0c08872..4dad8b1 100644
--- a/rdiff-backup/src/manage.py
+++ b/rdiff-backup/src/manage.py
@@ -1,4 +1,7 @@
-execfile("restore.py")
+from __future__ import generators
+from static import *
+from log import *
+import Globals, Time
#######################################################################
#
diff --git a/rdiff-backup/src/rdiff.py b/rdiff-backup/src/rdiff.py
deleted file mode 100644
index e4552ce..0000000
--- a/rdiff-backup/src/rdiff.py
+++ /dev/null
@@ -1,192 +0,0 @@
-execfile("iterfile.py")
-import os, popen2
-
-#######################################################################
-#
-# rdiff - Invoke rdiff utility to make signatures, deltas, or patch
-#
-
-class RdiffException(Exception): pass
-
-class Rdiff:
- """Contains static methods for rdiff operations
-
- All these operations should be done in a relatively safe manner
- using RobustAction and the like.
-
- """
- def get_signature(rp):
- """Take signature of rpin file and return in file object"""
- Log("Getting signature of %s" % rp.path, 7)
- return rp.conn.RdiffPopen(['rdiff', 'signature', rp.path])
-
- def get_delta_sigfileobj(sig_fileobj, rp_new):
- """Like get_delta but signature is in a file object"""
- sig_tf = TempFileManager.new(rp_new, None)
- sig_tf.write_from_fileobj(sig_fileobj)
- rdiff_popen_obj = Rdiff.get_delta_sigrp(sig_tf, rp_new)
- rdiff_popen_obj.set_thunk(sig_tf.delete)
- return rdiff_popen_obj
-
- def get_delta_sigrp(rp_signature, rp_new):
- """Take signature rp and new rp, return delta file object"""
- assert rp_signature.conn is rp_new.conn
- Log("Getting delta of %s with signature %s" %
- (rp_new.path, rp_signature.path), 7)
- return rp_new.conn.RdiffPopen(['rdiff', 'delta',
- rp_signature.path, rp_new.path])
-
- def write_delta_action(basis, new, delta, compress = None):
- """Return action writing delta which brings basis to new
-
- If compress is true, the output of rdiff will be gzipped
- before written to delta.
-
- """
- sig_tf = TempFileManager.new(new, None)
- delta_tf = TempFileManager.new(delta)
- def init(): Rdiff.write_delta(basis, new, delta_tf, compress, sig_tf)
- return Robust.make_tf_robustaction(init, (sig_tf, delta_tf),
- (None, delta))
-
- def write_delta(basis, new, delta, compress = None, sig_tf = None):
- """Write rdiff delta which brings basis to new"""
- Log("Writing delta %s from %s -> %s" %
- (basis.path, new.path, delta.path), 7)
- if not sig_tf: sig_tf = TempFileManager.new(new, None)
- sig_tf.write_from_fileobj(Rdiff.get_signature(basis))
- delta.write_from_fileobj(Rdiff.get_delta_sigrp(sig_tf, new), compress)
- sig_tf.delete()
-
- def patch_action(rp_basis, rp_delta, rp_out = None,
- out_tf = None, delta_compressed = None):
- """Return RobustAction which patches rp_basis with rp_delta
-
- If rp_out is None, put output in rp_basis. Will use TempFile
- out_tf it is specified. If delta_compressed is true, the
- delta file will be decompressed before processing with rdiff.
-
- """
- if not rp_out: rp_out = rp_basis
- else: assert rp_out.conn is rp_basis.conn
- if (delta_compressed or
- not (isinstance(rp_delta, RPath) and isinstance(rp_basis, RPath)
- and rp_basis.conn is rp_delta.conn)):
- if delta_compressed:
- assert isinstance(rp_delta, RPath)
- return Rdiff.patch_fileobj_action(rp_basis,
- rp_delta.open('rb', 1),
- rp_out, out_tf)
- else: return Rdiff.patch_fileobj_action(rp_basis,
- rp_delta.open('rb'),
- rp_out, out_tf)
-
- # Files are uncompressed on same connection, run rdiff
- if out_tf is None: out_tf = TempFileManager.new(rp_out)
- def init():
- Log("Patching %s using %s to %s via %s" %
- (rp_basis.path, rp_delta.path, rp_out.path, out_tf.path), 7)
- cmdlist = ["rdiff", "patch", rp_basis.path,
- rp_delta.path, out_tf.path]
- return_val = rp_basis.conn.os.spawnvp(os.P_WAIT, 'rdiff', cmdlist)
- out_tf.setdata()
- if return_val != 0 or not out_tf.lstat():
- RdiffException("Error running %s" % cmdlist)
- return Robust.make_tf_robustaction(init, (out_tf,), (rp_out,))
-
- def patch_fileobj_action(rp_basis, delta_fileobj, rp_out = None,
- out_tf = None, delta_compressed = None):
- """Like patch_action but diff is given in fileobj form
-
- Nest a writing of a tempfile with the actual patching to
- create a new action. We have to nest so that the tempfile
- will be around until the patching finishes.
-
- """
- if not rp_out: rp_out = rp_basis
- delta_tf = TempFileManager.new(rp_out, None)
- def init(): delta_tf.write_from_fileobj(delta_fileobj)
- def final(init_val): delta_tf.delete()
- def error(exc, ran_init, init_val): delta_tf.delete()
- write_delta_action = RobustAction(init, final, error)
- return Robust.chain(write_delta_action,
- Rdiff.patch_action(rp_basis, delta_tf,
- rp_out, out_tf))
-
- def patch_with_attribs_action(rp_basis, rp_delta, rp_out = None):
- """Like patch_action, but also transfers attributs from rp_delta"""
- if not rp_out: rp_out = rp_basis
- tf = TempFileManager.new(rp_out)
- return Robust.chain_nested(
- Rdiff.patch_action(rp_basis, rp_delta, rp_out, tf),
- Robust.copy_attribs_action(rp_delta, tf))
-
- def copy_action(rpin, rpout):
- """Use rdiff to copy rpin to rpout, conserving bandwidth"""
- if not rpin.isreg() or not rpout.isreg() or rpin.conn is rpout.conn:
- # rdiff not applicable, fallback to regular copying
- return Robust.copy_action(rpin, rpout)
-
- Log("Rdiff copying %s to %s" % (rpin.path, rpout.path), 6)
- delta_tf = TempFileManager.new(rpout, None)
- return Robust.chain(Rdiff.write_delta_action(rpout, rpin, delta_tf),
- Rdiff.patch_action(rpout, delta_tf),
- RobustAction(lambda: None, delta_tf.delete,
- lambda exc: delta_tf.delete))
-
-MakeStatic(Rdiff)
-
-
-class RdiffPopen:
- """Spawn process and treat stdout as file object
-
- Instead of using popen, which evaluates arguments with the shell
- and thus may lead to security holes (thanks to Jamie Heilman for
- this point), use the popen2 class and discard stdin.
-
- When closed, this object checks to make sure the process exited
- cleanly, and executes closing_thunk.
-
- """
- def __init__(self, cmdlist, closing_thunk = None):
- """RdiffFilehook initializer
-
- fileobj is the file we are emulating
- thunk is called with no parameters right after the file is closed
-
- """
- assert type(cmdlist) is types.ListType
- self.p3obj = popen2.Popen3(cmdlist)
- self.fileobj = self.p3obj.fromchild
- self.closing_thunk = closing_thunk
- self.cmdlist = cmdlist
-
- def set_thunk(self, closing_thunk):
- """Set closing_thunk if not already"""
- assert not self.closing_thunk
- self.closing_thunk = closing_thunk
-
- def read(self, length = -1): return self.fileobj.read(length)
-
- def close(self):
- closeval = self.fileobj.close()
- if self.closing_thunk: self.closing_thunk()
- exitval = self.p3obj.poll()
- if exitval == 0: return closeval
- elif exitval == 256:
- Log("Failure probably because %s couldn't be found in PATH."
- % self.cmdlist[0], 2)
- assert 0, "rdiff not found"
- elif exitval == -1:
- # There may a race condition where a process closes
- # but doesn't provide its exitval fast enough.
- Log("Waiting for process to close", 8)
- time.sleep(0.2)
- exitval = self.p3obj.poll()
- if exitval == 0: return closeval
- raise RdiffException("%s exited with non-zero value %d" %
- (self.cmdlist, exitval))
-
-
-
-
diff --git a/rdiff-backup/src/restore.py b/rdiff-backup/src/restore.py
index 30820b8..5202854 100644
--- a/rdiff-backup/src/restore.py
+++ b/rdiff-backup/src/restore.py
@@ -1,6 +1,6 @@
from __future__ import generators
-execfile("increment.py")
import tempfile
+from static import *
#######################################################################
#
@@ -362,3 +362,10 @@ class RestoreCombinedData:
else: RPath.copy(inc, target)
else: raise RestoreError("Unknown inctype %s" % inctype)
RPath.copy_attribs(inc, target)
+
+
+from log import *
+from destructive_stepping import *
+from rpath import *
+from rorpiter import *
+import Globals, Time, Rdiff, Hardlink, FilenameMapping, SetConnections
diff --git a/rdiff-backup/src/robust.py b/rdiff-backup/src/robust.py
index e539827..3c9851c 100644
--- a/rdiff-backup/src/robust.py
+++ b/rdiff-backup/src/robust.py
@@ -1,5 +1,5 @@
-import tempfile, errno, signal
-execfile("hardlink.py")
+import tempfile, errno, signal, cPickle
+from static import *
#######################################################################
#
@@ -243,7 +243,7 @@ class Robust:
"""
try: return function(*args)
except (EnvironmentError, SkipFileException, DSRPPermError,
- RPathException, RdiffException), exc:
+ RPathException, Rdiff.RdiffException), exc:
TracebackArchive.add()
if (not isinstance(exc, EnvironmentError) or
(errno.errorcode[exc[0]] in
@@ -356,6 +356,8 @@ class TempFileManager:
MakeClass(TempFileManager)
+from rpath import *
+
class TempFile(RPath):
"""Like an RPath, but keep track of which ones are still here"""
def rename(self, rp_dest):
@@ -642,3 +644,9 @@ class ResumeSessionInfo:
self.last_index = last_index
self.last_definitive = last_definitive
self.ITR, self.finalizer, = ITR, finalizer
+
+
+from log import *
+from destructive_stepping import *
+import Time, Rdiff
+from highlevel import *
diff --git a/rdiff-backup/src/rorpiter.py b/rdiff-backup/src/rorpiter.py
index efa0303..03705aa 100644
--- a/rdiff-backup/src/rorpiter.py
+++ b/rdiff-backup/src/rorpiter.py
@@ -1,6 +1,11 @@
-execfile("robust.py")
from __future__ import generators
-import tempfile, UserList
+import tempfile, UserList, types
+from static import *
+from log import *
+from rpath import *
+from robust import *
+from iterfile import *
+import Globals, Rdiff, Hardlink
#######################################################################
#
diff --git a/rdiff-backup/src/rpath.py b/rdiff-backup/src/rpath.py
index c6bcca6..73910be 100644
--- a/rdiff-backup/src/rpath.py
+++ b/rdiff-backup/src/rpath.py
@@ -1,5 +1,5 @@
-execfile("connection.py")
import os, stat, re, sys, shutil, gzip
+from static import *
#######################################################################
#
@@ -778,6 +778,10 @@ class RPathFileHook:
self.closing_thunk()
return result
+# Import these late to avoid circular dependencies
+from lazy import *
+from selection import *
+from destructive_stepping import *
class RpathDeleter(IterTreeReducer):
"""Delete a directory. Called by RPath.delete()"""
diff --git a/rdiff-backup/src/selection.py b/rdiff-backup/src/selection.py
index 3d1f0e2..4fee9ee 100644
--- a/rdiff-backup/src/selection.py
+++ b/rdiff-backup/src/selection.py
@@ -1,6 +1,9 @@
from __future__ import generators
-execfile("destructive_stepping.py")
import re
+from log import *
+from robust import *
+from destructive_stepping import *
+
#######################################################################
#
@@ -521,3 +524,4 @@ probably isn't what you meant.""" %
else: res = res + re.escape(c)
return res
+
diff --git a/rdiff-backup/src/setconnections.py b/rdiff-backup/src/setconnections.py
deleted file mode 100644
index a32c68e..0000000
--- a/rdiff-backup/src/setconnections.py
+++ /dev/null
@@ -1,219 +0,0 @@
-execfile("highlevel.py")
-
-#######################################################################
-#
-# setconnections - Parse initial arguments and establish connections
-#
-
-class SetConnectionsException(Exception): pass
-
-class SetConnections:
- """Parse args and setup connections
-
- The methods in this class are used once by Main to parse file
- descriptions like bescoto@folly.stanford.edu:/usr/bin/ls and to
- set up the related connections.
-
- """
- # This is the schema that determines how rdiff-backup will open a
- # pipe to the remote system. If the file is given as A::B, %s will
- # be substituted with A in the schema.
- __cmd_schema = 'ssh -C %s rdiff-backup --server'
- __cmd_schema_no_compress = 'ssh %s rdiff-backup --server'
-
- # This is a list of remote commands used to start the connections.
- # The first is None because it is the local connection.
- __conn_remote_cmds = [None]
-
- def InitRPs(cls, arglist, remote_schema = None, remote_cmd = None):
- """Map the given file descriptions into rpaths and return list"""
- if remote_schema: cls.__cmd_schema = remote_schema
- elif not Globals.ssh_compression:
- cls.__cmd_schema = cls.__cmd_schema_no_compress
-
- if not arglist: return []
- desc_pairs = map(cls.parse_file_desc, arglist)
-
- if filter(lambda x: x[0], desc_pairs): # True if any host_info found
- if remote_cmd:
- Log.FatalError("The --remote-cmd flag is not compatible "
- "with remote file descriptions.")
- elif remote_schema:
- Log("Remote schema option ignored - no remote file "
- "descriptions.", 2)
-
- cmd_pairs = map(cls.desc2cmd_pairs, desc_pairs)
- if remote_cmd: # last file description gets remote_cmd
- cmd_pairs[-1] = (remote_cmd, cmd_pairs[-1][1])
- return map(cls.cmdpair2rp, cmd_pairs)
-
- def cmdpair2rp(cls, cmd_pair):
- """Return RPath from cmd_pair (remote_cmd, filename)"""
- cmd, filename = cmd_pair
- if cmd: conn = cls.init_connection(cmd)
- else: conn = Globals.local_connection
- return RPath(conn, filename)
-
- def desc2cmd_pairs(cls, desc_pair):
- """Return pair (remote_cmd, filename) from desc_pair"""
- host_info, filename = desc_pair
- if not host_info: return (None, filename)
- else: return (cls.fill_schema(host_info), filename)
-
- def parse_file_desc(cls, file_desc):
- """Parse file description returning pair (host_info, filename)
-
- In other words, bescoto@folly.stanford.edu::/usr/bin/ls =>
- ("bescoto@folly.stanford.edu", "/usr/bin/ls"). The
- complication is to allow for quoting of : by a \. If the
- string is not separated by :, then the host_info is None.
-
- """
- def check_len(i):
- if i >= len(file_desc):
- raise SetConnectionsException(
- "Unexpected end to file description %s" % file_desc)
-
- host_info_list, i, last_was_quoted = [], 0, None
- while 1:
- if i == len(file_desc):
- return (None, file_desc)
-
- if file_desc[i] == '\\':
- i = i+1
- check_len(i)
- last_was_quoted = 1
- elif (file_desc[i] == ":" and i > 0 and file_desc[i-1] == ":"
- and not last_was_quoted):
- host_info_list.pop() # Remove last colon from name
- break
- else: last_was_quoted = None
- host_info_list.append(file_desc[i])
- i = i+1
-
- check_len(i+1)
- return ("".join(host_info_list), file_desc[i+1:])
-
- def fill_schema(cls, host_info):
- """Fills host_info into the schema and returns remote command"""
- return cls.__cmd_schema % host_info
-
- def init_connection(cls, remote_cmd):
- """Run remote_cmd, register connection, and then return it
-
- If remote_cmd is None, then the local connection will be
- returned. This also updates some settings on the remote side,
- like global settings, its connection number, and verbosity.
-
- """
- if not remote_cmd: return Globals.local_connection
-
- Log("Executing " + remote_cmd, 4)
- stdin, stdout = os.popen2(remote_cmd)
- conn_number = len(Globals.connections)
- conn = PipeConnection(stdout, stdin, conn_number)
-
- cls.check_connection_version(conn, remote_cmd)
- Log("Registering connection %d" % conn_number, 7)
- cls.init_connection_routing(conn, conn_number, remote_cmd)
- cls.init_connection_settings(conn)
- return conn
-
- def check_connection_version(cls, conn, remote_cmd):
- """Log warning if connection has different version"""
- try: remote_version = conn.Globals.get('version')
- except ConnectionReadError, exception:
- Log.FatalError("""%s
-
-Couldn't start up the remote connection by executing
-
- %s
-
-Remember that, under the default settings, rdiff-backup must be
-installed in the PATH on the remote system. See the man page for more
-information.""" % (exception, remote_cmd))
-
- if remote_version != Globals.version:
- Log("Warning: Local version %s does not match remote version %s."
- % (Globals.version, remote_version), 2)
-
- def init_connection_routing(cls, conn, conn_number, remote_cmd):
- """Called by init_connection, establish routing, conn dict"""
- Globals.connection_dict[conn_number] = conn
-
- conn.SetConnections.init_connection_remote(conn_number)
- for other_remote_conn in Globals.connections[1:]:
- conn.SetConnections.add_redirected_conn(
- other_remote_conn.conn_number)
- other_remote_conn.SetConnections.add_redirected_conn(conn_number)
-
- Globals.connections.append(conn)
- cls.__conn_remote_cmds.append(remote_cmd)
-
- def init_connection_settings(cls, conn):
- """Tell new conn about log settings and updated globals"""
- conn.Log.setverbosity(Log.verbosity)
- conn.Log.setterm_verbosity(Log.term_verbosity)
- for setting_name in Globals.changed_settings:
- conn.Globals.set(setting_name, Globals.get(setting_name))
-
- def init_connection_remote(cls, conn_number):
- """Run on server side to tell self that have given conn_number"""
- Globals.connection_number = conn_number
- Globals.local_connection.conn_number = conn_number
- Globals.connection_dict[0] = Globals.connections[1]
- Globals.connection_dict[conn_number] = Globals.local_connection
-
- def add_redirected_conn(cls, conn_number):
- """Run on server side - tell about redirected connection"""
- Globals.connection_dict[conn_number] = \
- RedirectedConnection(conn_number)
-
- def UpdateGlobal(cls, setting_name, val):
- """Update value of global variable across all connections"""
- for conn in Globals.connections:
- conn.Globals.set(setting_name, val)
-
- def BackupInitConnections(cls, reading_conn, writing_conn):
- """Backup specific connection initialization"""
- reading_conn.Globals.set("isbackup_reader", 1)
- writing_conn.Globals.set("isbackup_writer", 1)
- cls.UpdateGlobal("backup_reader", reading_conn)
- cls.UpdateGlobal("backup_writer", writing_conn)
-
- def CloseConnections(cls):
- """Close all connections. Run by client"""
- assert not Globals.server
- for conn in Globals.connections: conn.quit()
- del Globals.connections[1:] # Only leave local connection
- Globals.connection_dict = {0: Globals.local_connection}
- Globals.backup_reader = Globals.isbackup_reader = \
- Globals.backup_writer = Globals.isbackup_writer = None
-
- def TestConnections(cls):
- """Test connections, printing results"""
- if len(Globals.connections) == 1:
- print "No remote connections specified"
- else:
- for i in range(1, len(Globals.connections)):
- cls.test_connection(i)
-
- def test_connection(cls, conn_number):
- """Test connection. conn_number 0 is the local connection"""
- print "Testing server started by: ", \
- cls.__conn_remote_cmds[conn_number]
- conn = Globals.connections[conn_number]
- try:
- assert conn.pow(2,3) == 8
- assert conn.os.path.join("a", "b") == "a/b"
- version = conn.reval("lambda: Globals.version")
- except:
- sys.stderr.write("Server tests failed\n")
- raise
- if not version == Globals.version:
- print """Server may work, but there is a version mismatch:
-Local version: %s
-Remote version: %s""" % (Globals.version, version)
- else: print "Server OK"
-
-MakeClass(SetConnections)
diff --git a/rdiff-backup/src/static.py b/rdiff-backup/src/static.py
index 2e97cd0..0355f44 100644
--- a/rdiff-backup/src/static.py
+++ b/rdiff-backup/src/static.py
@@ -1,5 +1,3 @@
-execfile("globals.py")
-
#######################################################################
#
# static - MakeStatic and MakeClass
diff --git a/rdiff-backup/src/statistics.py b/rdiff-backup/src/statistics.py
index a91a681..16dd881 100644
--- a/rdiff-backup/src/statistics.py
+++ b/rdiff-backup/src/statistics.py
@@ -1,4 +1,4 @@
-execfile("filename_mapping.py")
+from lazy import *
#######################################################################
#
@@ -277,73 +277,7 @@ class StatsITR(IterTreeReducer, StatsObj):
self.__dict__[attr] += subinstance.__dict__[attr]
-class Stats:
- """Misc statistics methods, pertaining to dir and session stat files"""
- # This is the RPath of the directory statistics file, and the
- # associated open file. It will hold a line of statistics for
- # each directory that is backed up.
- _dir_stats_rp = None
- _dir_stats_fp = None
-
- # This goes at the beginning of the directory statistics file and
- # explains the format.
- _dir_stats_header = """# rdiff-backup directory statistics file
-#
-# Each line is in the following format:
-# RelativeDirName %s
-""" % " ".join(StatsObj.stat_file_attrs)
-
- def open_dir_stats_file(cls):
- """Open directory statistics file, write header"""
- assert not cls._dir_stats_fp, "Directory file already open"
-
- if Globals.compression: suffix = "data.gz"
- else: suffix = "data"
- cls._dir_stats_rp = Inc.get_inc(Globals.rbdir.append(
- "directory_statistics"), Time.curtime, suffix)
-
- if cls._dir_stats_rp.lstat():
- Log("Warning, statistics file %s already exists, appending" %
- cls._dir_stats_rp.path, 2)
- cls._dir_stats_fp = cls._dir_stats_rp.open("ab",
- Globals.compression)
- else: cls._dir_stats_fp = \
- cls._dir_stats_rp.open("wb", Globals.compression)
- cls._dir_stats_fp.write(cls._dir_stats_header)
-
- def write_dir_stats_line(cls, statobj, index):
- """Write info from statobj about rpath to statistics file"""
- if Globals.null_separator:
- cls._dir_stats_fp.write(statobj.get_stats_line(index, None) + "\0")
- else: cls._dir_stats_fp.write(statobj.get_stats_line(index) + "\n")
-
- def close_dir_stats_file(cls):
- """Close directory statistics file if its open"""
- if cls._dir_stats_fp:
- cls._dir_stats_fp.close()
- cls._dir_stats_fp = None
-
- def write_session_statistics(cls, statobj):
- """Write session statistics into file, log"""
- stat_inc = Inc.get_inc(Globals.rbdir.append("session_statistics"),
- Time.curtime, "data")
- statobj.StartTime = Time.curtime
- statobj.EndTime = time.time()
-
- # include hardlink data and dir stats in size of increments
- if Globals.preserve_hardlinks and Hardlink.final_inc:
- # include hardlink data in size of increments
- statobj.IncrementFiles += 1
- statobj.IncrementFileSize += Hardlink.final_inc.getsize()
- if cls._dir_stats_rp and cls._dir_stats_rp.lstat():
- statobj.IncrementFiles += 1
- statobj.IncrementFileSize += cls._dir_stats_rp.getsize()
-
- statobj.write_stats_to_rp(stat_inc)
- if Globals.print_statistics:
- message = statobj.get_stats_logstring("Session statistics")
- Log.log_to_file(message)
- Globals.client_conn.sys.stdout.write(message)
-
-MakeClass(Stats)
-
+from log import *
+from increment import *
+from robust import *
+import Globals
diff --git a/rdiff-backup/src/ttime.py b/rdiff-backup/src/ttime.py
deleted file mode 100644
index 852f6ea..0000000
--- a/rdiff-backup/src/ttime.py
+++ /dev/null
@@ -1,201 +0,0 @@
-execfile("log.py")
-import time, types, re
-
-#######################################################################
-#
-# ttime - Provide Time class, which contains time related functions.
-#
-
-class TimeException(Exception): pass
-
-class Time:
- """Functions which act on the time"""
- _interval_conv_dict = {"s": 1, "m": 60, "h": 3600, "D": 86400,
- "W": 7*86400, "M": 30*86400, "Y": 365*86400}
- _integer_regexp = re.compile("^[0-9]+$")
- _interval_regexp = re.compile("^([0-9]+)([smhDWMY])")
- _genstr_date_regexp1 = re.compile("^(?P<year>[0-9]{4})[-/]"
- "(?P<month>[0-9]{1,2})[-/](?P<day>[0-9]{1,2})$")
- _genstr_date_regexp2 = re.compile("^(?P<month>[0-9]{1,2})[-/]"
- "(?P<day>[0-9]{1,2})[-/](?P<year>[0-9]{4})$")
-
- def setcurtime(cls, curtime = None):
- """Sets the current time in curtime and curtimestr on all systems"""
- t = curtime or time.time()
- for conn in Globals.connections:
- conn.Time.setcurtime_local(t, cls.timetostring(t))
-
- def setcurtime_local(cls, timeinseconds, timestr):
- """Only set the current time locally"""
- cls.curtime = timeinseconds
- cls.curtimestr = timestr
-
- def setprevtime(cls, timeinseconds):
- """Sets the previous inc time in prevtime and prevtimestr"""
- assert timeinseconds > 0, timeinseconds
- timestr = cls.timetostring(timeinseconds)
- for conn in Globals.connections:
- conn.Time.setprevtime_local(timeinseconds, timestr)
-
- def setprevtime_local(cls, timeinseconds, timestr):
- """Like setprevtime but only set the local version"""
- cls.prevtime = timeinseconds
- cls.prevtimestr = timestr
-
- def timetostring(cls, timeinseconds):
- """Return w3 datetime compliant listing of timeinseconds"""
- return time.strftime("%Y-%m-%dT%H" + Globals.time_separator +
- "%M" + Globals.time_separator + "%S",
- time.localtime(timeinseconds)) + cls.gettzd()
-
- def stringtotime(cls, timestring):
- """Return time in seconds from w3 timestring
-
- If there is an error parsing the string, or it doesn't look
- like a w3 datetime string, return None.
-
- """
- try:
- date, daytime = timestring[:19].split("T")
- year, month, day = map(int, date.split("-"))
- hour, minute, second = map(int,
- daytime.split(Globals.time_separator))
- assert 1900 < year < 2100, year
- assert 1 <= month <= 12
- assert 1 <= day <= 31
- assert 0 <= hour <= 23
- assert 0 <= minute <= 59
- assert 0 <= second <= 61 # leap seconds
- timetuple = (year, month, day, hour, minute, second, -1, -1, -1)
- if time.daylight:
- utc_in_secs = time.mktime(timetuple) - time.altzone
- else: utc_in_secs = time.mktime(timetuple) - time.timezone
-
- return long(utc_in_secs) + cls.tzdtoseconds(timestring[19:])
- except (TypeError, ValueError, AssertionError): return None
-
- def timetopretty(cls, timeinseconds):
- """Return pretty version of time"""
- return time.asctime(time.localtime(timeinseconds))
-
- def stringtopretty(cls, timestring):
- """Return pretty version of time given w3 time string"""
- return cls.timetopretty(cls.stringtotime(timestring))
-
- def inttopretty(cls, seconds):
- """Convert num of seconds to readable string like "2 hours"."""
- partlist = []
- hours, seconds = divmod(seconds, 3600)
- if hours > 1: partlist.append("%d hours" % hours)
- elif hours == 1: partlist.append("1 hour")
-
- minutes, seconds = divmod(seconds, 60)
- if minutes > 1: partlist.append("%d minutes" % minutes)
- elif minutes == 1: partlist.append("1 minute")
-
- if seconds == 1: partlist.append("1 second")
- elif not partlist or seconds > 1:
- if isinstance(seconds, int) or isinstance(seconds, long):
- partlist.append("%s seconds" % seconds)
- else: partlist.append("%.2f seconds" % seconds)
- return " ".join(partlist)
-
- def intstringtoseconds(cls, interval_string):
- """Convert a string expressing an interval (e.g. "4D2s") to seconds"""
- def error():
- raise TimeException("""Bad interval string "%s"
-
-Intervals are specified like 2Y (2 years) or 2h30m (2.5 hours). The
-allowed special characters are s, m, h, D, W, M, and Y. See the man
-page for more information.
-""" % interval_string)
- if len(interval_string) < 2: error()
-
- total = 0
- while interval_string:
- match = cls._interval_regexp.match(interval_string)
- if not match: error()
- num, ext = int(match.group(1)), match.group(2)
- if not ext in cls._interval_conv_dict or num < 0: error()
- total += num*cls._interval_conv_dict[ext]
- interval_string = interval_string[match.end(0):]
- return total
-
- def gettzd(cls):
- """Return w3's timezone identification string.
-
- Expresed as [+/-]hh:mm. For instance, PST is -08:00. Zone is
- coincides with what localtime(), etc., use.
-
- """
- if time.daylight: offset = -1 * time.altzone/60
- else: offset = -1 * time.timezone/60
- if offset > 0: prefix = "+"
- elif offset < 0: prefix = "-"
- else: return "Z" # time is already in UTC
-
- hours, minutes = map(abs, divmod(offset, 60))
- assert 0 <= hours <= 23
- assert 0 <= minutes <= 59
- return "%s%02d%s%02d" % (prefix, hours,
- Globals.time_separator, minutes)
-
- def tzdtoseconds(cls, tzd):
- """Given w3 compliant TZD, return how far ahead UTC is"""
- if tzd == "Z": return 0
- assert len(tzd) == 6 # only accept forms like +08:00 for now
- assert (tzd[0] == "-" or tzd[0] == "+") and \
- tzd[3] == Globals.time_separator
- return -60 * (60 * int(tzd[:3]) + int(tzd[4:]))
-
- def cmp(cls, time1, time2):
- """Compare time1 and time2 and return -1, 0, or 1"""
- if type(time1) is types.StringType:
- time1 = cls.stringtotime(time1)
- assert time1 is not None
- if type(time2) is types.StringType:
- time2 = cls.stringtotime(time2)
- assert time2 is not None
-
- if time1 < time2: return -1
- elif time1 == time2: return 0
- else: return 1
-
- def genstrtotime(cls, timestr, curtime = None):
- """Convert a generic time string to a time in seconds"""
- if curtime is None: curtime = cls.curtime
- if timestr == "now": return curtime
-
- def error():
- raise TimeException("""Bad time string "%s"
-
-The acceptible time strings are intervals (like "3D64s"), w3-datetime
-strings, like "2002-04-26T04:22:01-07:00" (strings like
-"2002-04-26T04:22:01" are also acceptable - rdiff-backup will use the
-current time zone), or ordinary dates like 2/4/1997 or 2001-04-23
-(various combinations are acceptable, but the month always precedes
-the day).""" % timestr)
-
- # Test for straight integer
- if cls._integer_regexp.search(timestr): return int(timestr)
-
- # Test for w3-datetime format, possibly missing tzd
- t = cls.stringtotime(timestr) or cls.stringtotime(timestr+cls.gettzd())
- if t: return t
-
- try: # test for an interval, like "2 days ago"
- return curtime - cls.intstringtoseconds(timestr)
- except TimeException: pass
-
- # Now check for dates like 2001/3/23
- match = cls._genstr_date_regexp1.search(timestr) or \
- cls._genstr_date_regexp2.search(timestr)
- if not match: error()
- timestr = "%s-%02d-%02dT00:00:00%s" % \
- (match.group('year'), int(match.group('month')),
- int(match.group('day')), cls.gettzd())
- t = cls.stringtotime(timestr)
- if t: return t
- else: error()
-
-MakeClass(Time)
diff --git a/rdiff-backup/testing/chdir-wrapper b/rdiff-backup/testing/chdir-wrapper
index 073b297..1d0db7d 100755
--- a/rdiff-backup/testing/chdir-wrapper
+++ b/rdiff-backup/testing/chdir-wrapper
@@ -9,10 +9,8 @@ the server. Otherwise will start the server without a chdir.
import os, sys
-#execfile("commontest.py")
-#rbexec("setconnections.py")
-
if len(sys.argv) > 1: os.chdir(sys.argv[1])
#PipeConnection(sys.stdin, sys.stdout).Server()
-os.system("/home/ben/prog/python/rdiff-backup/src/rdiff-backup --server")
+#os.system("/home/ben/prog/python/rdiff-backup/rdiff-backup --server")
+os.system("/home/ben/prog/python/rdiff-backup/testing/server.py /home/ben/prog/python/rdiff-backup/src")
diff --git a/rdiff-backup/testing/chdir-wrapper2 b/rdiff-backup/testing/chdir-wrapper2
new file mode 100755
index 0000000..7fc1312
--- /dev/null
+++ b/rdiff-backup/testing/chdir-wrapper2
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+
+"""Used to emulate a remote connection by changing directories.
+
+Like chdir-wrapper, but this time run the 'rdiff-backup' script, not
+some other special thing.
+
+"""
+
+import os, sys
+
+if len(sys.argv) > 1: os.chdir(sys.argv[1])
+#PipeConnection(sys.stdin, sys.stdout).Server()
+
+os.system("/home/ben/prog/python/rdiff-backup/rdiff-backup --server")
+
diff --git a/rdiff-backup/testing/commontest.py b/rdiff-backup/testing/commontest.py
index 04f2a8a..24eb2cb 100644
--- a/rdiff-backup/testing/commontest.py
+++ b/rdiff-backup/testing/commontest.py
@@ -1,5 +1,10 @@
"""commontest - Some functions and constants common to several test cases"""
-import os
+import os, sys
+sys.path.insert(0, "../src")
+from rpath import *
+from destructive_stepping import *
+from highlevel import *
+import Globals, Hardlink, SetConnections, Main
SourceDir = "../src"
AbsCurdir = os.getcwd() # Absolute path name of current directory
@@ -7,12 +12,6 @@ AbsTFdir = AbsCurdir+"/testfiles"
MiscDir = "../misc"
__no_execute__ = 1 # Keeps the actual rdiff-backup program from running
-def rbexec(src_file):
- """Changes to the source directory, execfile src_file, return"""
- os.chdir(SourceDir)
- execfile(src_file, globals())
- os.chdir(AbsCurdir)
-
def Myrm(dirstring):
"""Run myrm on given directory string"""
assert not os.system("%s/myrm %s" % (MiscDir, dirstring))
@@ -77,9 +76,9 @@ def InternalBackup(source_local, dest_local, src_dir, dest_dir,
% (SourceDir, dest_dir)
rpin, rpout = SetConnections.InitRPs([src_dir, dest_dir], remote_schema)
- _get_main().misc_setup([rpin, rpout])
- _get_main().Backup(rpin, rpout)
- _get_main().cleanup()
+ Main.misc_setup([rpin, rpout])
+ Main.Backup(rpin, rpout)
+ Main.cleanup()
def InternalMirror(source_local, dest_local, src_dir, dest_dir,
write_data = None):
@@ -94,8 +93,8 @@ def InternalMirror(source_local, dest_local, src_dir, dest_dir,
% (SourceDir, dest_dir)
rpin, rpout = SetConnections.InitRPs([src_dir, dest_dir], remote_schema)
- _get_main().misc_setup([rpin, rpout])
- _get_main().backup_init_select(rpin, rpout)
+ Main.misc_setup([rpin, rpout])
+ Main.backup_init_select(rpin, rpout)
if not rpout.lstat(): rpout.mkdir()
if write_data: # use rdiff-backup-data dir to checkpoint
data_dir = rpout.append("rdiff-backup-data")
@@ -109,7 +108,7 @@ def InternalMirror(source_local, dest_local, src_dir, dest_dir,
SaveState.init_filenames()
HighLevel.Mirror(rpin, rpout, Globals.rbdir.append("increments"))
else: HighLevel.Mirror(rpin, rpout)
- _get_main().cleanup()
+ Main.cleanup()
def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time):
"""Restore mirror_dir to dest_dir at given time
@@ -132,12 +131,11 @@ def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time):
remote_schema)
Time.setcurtime()
inc = get_increment_rp(mirror_rp, time)
- if inc:
- _get_main().Restore(get_increment_rp(mirror_rp, time), dest_rp)
+ if inc: Main.restore(get_increment_rp(mirror_rp, time), dest_rp)
else: # use alternate syntax
- _get_main().restore_timestr = str(time)
- _get_main().RestoreAsOf(mirror_rp, dest_rp)
- _get_main().cleanup()
+ Main.restore_timestr = str(time)
+ Main.RestoreAsOf(mirror_rp, dest_rp)
+ Main.cleanup()
def get_increment_rp(mirror_rp, time):
"""Return increment rp matching time in seconds"""
@@ -154,14 +152,7 @@ def _reset_connections(src_rp, dest_rp):
#Globals.connections = [Globals.local_connection]
#Globals.connection_dict = {0: Globals.local_connection}
SetConnections.UpdateGlobal('rbdir', None)
- _get_main().misc_setup([src_rp, dest_rp])
-
-def _get_main():
- """Set Globals.Main if it doesn't exist, and return"""
- try: return Globals.Main
- except AttributeError:
- Globals.Main = Main()
- return Globals.Main
+ Main.misc_setup([src_rp, dest_rp])
def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
equality_func = None, exclude_rbdir = 1,
diff --git a/rdiff-backup/testing/connectiontest.py b/rdiff-backup/testing/connectiontest.py
index dfac612..61a75f4 100644
--- a/rdiff-backup/testing/connectiontest.py
+++ b/rdiff-backup/testing/connectiontest.py
@@ -1,7 +1,7 @@
import unittest, types, tempfile, os, sys
-execfile("commontest.py")
-rbexec("setconnections.py")
-
+from commontest import *
+from connection import *
+import Globals
class LocalConnectionTest(unittest.TestCase):
"""Test the dummy connection"""
diff --git a/rdiff-backup/testing/destructive_steppingtest.py b/rdiff-backup/testing/destructive_steppingtest.py
index 1194835..ab1c23b 100644
--- a/rdiff-backup/testing/destructive_steppingtest.py
+++ b/rdiff-backup/testing/destructive_steppingtest.py
@@ -1,7 +1,9 @@
from __future__ import generators
import unittest
-execfile("commontest.py")
-rbexec("selection.py")
+from commontest import *
+from rpath import *
+from selection import *
+import Globals
Log.setverbosity(4)
diff --git a/rdiff-backup/testing/finaltest.py b/rdiff-backup/testing/finaltest.py
index 150d5ac..0a51485 100644
--- a/rdiff-backup/testing/finaltest.py
+++ b/rdiff-backup/testing/finaltest.py
@@ -1,12 +1,13 @@
import unittest, os, re, sys
-execfile("commontest.py")
-rbexec("restore.py")
+from commontest import *
+from log import *
+from rpath import *
+import Globals
"""Regression tests"""
Globals.exclude_mirror_regexps = [re.compile(".*/rdiff-backup-data")]
Log.setverbosity(7)
-Make()
lc = Globals.local_connection
@@ -43,7 +44,7 @@ class PathSetter(unittest.TestCase):
def reset_schema(self):
self.rb_schema = SourceDir + \
- "/rdiff-backup -v3 --remote-schema './chdir-wrapper %s' "
+ "/../rdiff-backup -v3 --remote-schema './chdir-wrapper2 %s' "
def refresh(self, *rp_list):
"""Reread data for the given rps"""
diff --git a/rdiff-backup/testing/hardlinktest.py b/rdiff-backup/testing/hardlinktest.py
index a45e427..9e6bdf9 100644
--- a/rdiff-backup/testing/hardlinktest.py
+++ b/rdiff-backup/testing/hardlinktest.py
@@ -1,7 +1,7 @@
import os, unittest
-execfile("commontest.py")
-rbexec("main.py")
-
+from commontest import *
+from rpath import *
+import Globals, Hardlink
Log.setverbosity(7)
diff --git a/rdiff-backup/testing/highleveltest.py b/rdiff-backup/testing/highleveltest.py
index 1b79b90..8f24f9d 100644
--- a/rdiff-backup/testing/highleveltest.py
+++ b/rdiff-backup/testing/highleveltest.py
@@ -1,8 +1,6 @@
import unittest
-
-execfile("commontest.py")
-rbexec("main.py")
-
+from commontest import *
+import Globals, SetConnections
class RemoteMirrorTest(unittest.TestCase):
"""Test mirroring"""
diff --git a/rdiff-backup/testing/incrementtest.py b/rdiff-backup/testing/incrementtest.py
index ecafa70..5562fdd 100644
--- a/rdiff-backup/testing/incrementtest.py
+++ b/rdiff-backup/testing/incrementtest.py
@@ -1,8 +1,8 @@
import unittest, os
-
-execfile("commontest.py")
-rbexec("main.py")
-
+from commontest import *
+from log import *
+from rpath import *
+from restore import *
lc = Globals.local_connection
Globals.change_source_perms = 1
diff --git a/rdiff-backup/testing/iterfiletest.py b/rdiff-backup/testing/iterfiletest.py
index 38dca4d..62f622a 100644
--- a/rdiff-backup/testing/iterfiletest.py
+++ b/rdiff-backup/testing/iterfiletest.py
@@ -1,6 +1,6 @@
import unittest, StringIO
-execfile("commontest.py")
-rbexec("iterfile.py")
+from commontest import *
+from iterfile import *
class testIterFile(unittest.TestCase):
diff --git a/rdiff-backup/testing/killtest.py b/rdiff-backup/testing/killtest.py
index d0b1b40..5b9a836 100644
--- a/rdiff-backup/testing/killtest.py
+++ b/rdiff-backup/testing/killtest.py
@@ -1,6 +1,7 @@
import unittest, os, signal, sys, random, time
-execfile("commontest.py")
-rbexec("main.py")
+from commontest import *
+from log import *
+import Globals, Main
"""Test consistency by killing rdiff-backup as it is backing up"""
diff --git a/rdiff-backup/testing/lazytest.py b/rdiff-backup/testing/lazytest.py
index 83f4d20..d0b743c 100644
--- a/rdiff-backup/testing/lazytest.py
+++ b/rdiff-backup/testing/lazytest.py
@@ -1,8 +1,7 @@
from __future__ import generators
import unittest, pickle
-
-execfile("commontest.py")
-rbexec("robust.py")
+from commontest import *
+from lazy import *
class Iterators(unittest.TestCase):
one_to_100 = lambda s: iter(range(1, 101))
diff --git a/rdiff-backup/testing/rdifftest.py b/rdiff-backup/testing/rdifftest.py
index 68f88a2..223e7a1 100644
--- a/rdiff-backup/testing/rdifftest.py
+++ b/rdiff-backup/testing/rdifftest.py
@@ -1,8 +1,8 @@
import unittest, random
-
-execfile("commontest.py")
-rbexec("selection.py")
-
+from commontest import *
+from log import *
+from selection import *
+import Globals, Rdiff
Log.setverbosity(6)
diff --git a/rdiff-backup/testing/regressiontest.py b/rdiff-backup/testing/regressiontest.py
index 80a60d0..c8cd59c 100644
--- a/rdiff-backup/testing/regressiontest.py
+++ b/rdiff-backup/testing/regressiontest.py
@@ -1,7 +1,8 @@
import unittest, os
-
-execfile("commontest.py")
-rbexec("main.py")
+from commontest import *
+from log import *
+from rpath import *
+import Globals, SetConnections
"""Regression tests
@@ -50,7 +51,7 @@ class PathSetter(unittest.TestCase):
"""Return (prefix, connection) tuple"""
if path:
return (return_path,
- SetConnections.init_connection("python ./chdir-wrapper "+path))
+ SetConnections.init_connection("./chdir-wrapper "+path))
else: return ('./', Globals.local_connection)
def get_src_rp(self, path):
@@ -193,25 +194,25 @@ class IncrementTest2(PathSetter):
Time.setcurtime()
SaveState.init_filenames()
- _get_main().backup_init_select(Local.inc1rp, Local.rpout)
+ Main.backup_init_select(Local.inc1rp, Local.rpout)
HighLevel.Mirror(self.inc1rp, self.rpout)
assert CompareRecursive(Local.inc1rp, Local.rpout)
Time.setcurtime()
Time.setprevtime(999500000)
- _get_main().backup_init_select(self.inc2rp, self.rpout)
+ Main.backup_init_select(self.inc2rp, self.rpout)
HighLevel.Mirror_and_increment(self.inc2rp, self.rpout, self.rpout_inc)
assert CompareRecursive(Local.inc2rp, Local.rpout)
Time.setcurtime()
Time.setprevtime(999510000)
- _get_main().backup_init_select(self.inc3rp, self.rpout)
+ Main.backup_init_select(self.inc3rp, self.rpout)
HighLevel.Mirror_and_increment(self.inc3rp, self.rpout, self.rpout_inc)
assert CompareRecursive(Local.inc3rp, Local.rpout)
Time.setcurtime()
Time.setprevtime(999520000)
- _get_main().backup_init_select(self.inc4rp, self.rpout)
+ Main.backup_init_select(self.inc4rp, self.rpout)
HighLevel.Mirror_and_increment(self.inc4rp, self.rpout, self.rpout_inc)
assert CompareRecursive(Local.inc4rp, Local.rpout)
@@ -419,8 +420,8 @@ class MirrorTest(PathSetter):
def Mirror(self, rpin, rpout, write_increments = 1):
"""Like HighLevel.Mirror, but run misc_setup first"""
- _get_main().misc_setup([rpin, rpout])
- _get_main().backup_init_select(rpin, rpout)
+ Main.misc_setup([rpin, rpout])
+ Main.backup_init_select(rpin, rpout)
if write_increments:
HighLevel.Mirror(rpin, rpout,
rpout.append_path("rdiff-backup-data/increments"))
diff --git a/rdiff-backup/testing/restoretest.py b/rdiff-backup/testing/restoretest.py
index e934c4a..a5d7f12 100644
--- a/rdiff-backup/testing/restoretest.py
+++ b/rdiff-backup/testing/restoretest.py
@@ -1,7 +1,9 @@
import unittest
+from commontest import *
+from log import *
+from restore import *
+import Globals
-execfile("commontest.py")
-rbexec("main.py")
Log.setverbosity(3)
diff --git a/rdiff-backup/testing/robusttest.py b/rdiff-backup/testing/robusttest.py
index 912117a..c76054e 100644
--- a/rdiff-backup/testing/robusttest.py
+++ b/rdiff-backup/testing/robusttest.py
@@ -1,7 +1,8 @@
import os, unittest
+from commontest import *
+from rpath import *
+from robust import *
-execfile("commontest.py")
-rbexec("setconnections.py")
class TestRobustAction(unittest.TestCase):
"""Test some robust actions"""
diff --git a/rdiff-backup/testing/roottest.py b/rdiff-backup/testing/roottest.py
index c6d81c6..2320f4c 100644
--- a/rdiff-backup/testing/roottest.py
+++ b/rdiff-backup/testing/roottest.py
@@ -1,6 +1,7 @@
import unittest, os
-execfile("commontest.py")
-rbexec("main.py")
+from commontest import *
+from log import *
+import Globals
"""Root tests
diff --git a/rdiff-backup/testing/rorpitertest.py b/rdiff-backup/testing/rorpitertest.py
index 718f75a..6f9cfeb 100644
--- a/rdiff-backup/testing/rorpitertest.py
+++ b/rdiff-backup/testing/rorpitertest.py
@@ -1,6 +1,9 @@
import unittest
-execfile("commontest.py")
-rbexec("highlevel.py")
+from commontest import *
+from log import *
+from rpath import *
+from rorpiter import *
+import Globals
#Log.setverbosity(8)
diff --git a/rdiff-backup/testing/rpathtest.py b/rdiff-backup/testing/rpathtest.py
index 9a24e6c..d4ffdbe 100644
--- a/rdiff-backup/testing/rpathtest.py
+++ b/rdiff-backup/testing/rpathtest.py
@@ -1,7 +1,6 @@
import os, cPickle, sys, unittest
-execfile("commontest.py")
-rbexec("highlevel.py")
-
+from commontest import *
+from rpath import *
class RPathTest(unittest.TestCase):
diff --git a/rdiff-backup/testing/selectiontest.py b/rdiff-backup/testing/selectiontest.py
index a80830b..945d427 100644
--- a/rdiff-backup/testing/selectiontest.py
+++ b/rdiff-backup/testing/selectiontest.py
@@ -1,7 +1,10 @@
from __future__ import generators
import re, StringIO, unittest
-execfile("commontest.py")
-rbexec("highlevel.py")
+from commontest import *
+from selection import *
+from destructive_stepping import *
+import Globals
+
class MatchingTest(unittest.TestCase):
"""Test matching of file names against various selection functions"""
diff --git a/rdiff-backup/testing/server.py b/rdiff-backup/testing/server.py
index 5baaca2..da7d905 100755
--- a/rdiff-backup/testing/server.py
+++ b/rdiff-backup/testing/server.py
@@ -21,12 +21,10 @@ if len(sys.argv) > 2:
sys.exit(1)
try:
- if len(sys.argv) == 2:
- olddir = os.getcwd()
- os.chdir(sys.argv[1])
- execfile("setconnections.py")
- if len(sys.argv) == 2: os.chdir(olddir)
-except (OSError, IOError):
+ if len(sys.argv) == 2: sys.path.insert(0, sys.argv[1])
+ import Globals
+ from connection import *
+except (OSError, IOError, ImportError):
print_usage()
raise
diff --git a/rdiff-backup/testing/setconnectionstest.py b/rdiff-backup/testing/setconnectionstest.py
index d5d2671..0e862b9 100644
--- a/rdiff-backup/testing/setconnectionstest.py
+++ b/rdiff-backup/testing/setconnectionstest.py
@@ -1,6 +1,6 @@
import unittest
-execfile("commontest.py")
-rbexec("setconnections.py")
+from commontest import *
+import SetConnections
class SetConnectionsTest(unittest.TestCase):
"""Set SetConnections Class"""
@@ -18,9 +18,10 @@ class SetConnectionsTest(unittest.TestCase):
assert pfd("foobar") == (None, "foobar")
assert pfd(r"hello\::there") == (None, "hello\::there")
- self.assertRaises(SetConnectionsException, pfd, r"hello\:there::")
- self.assertRaises(SetConnectionsException, pfd, "foobar\\")
-
+ self.assertRaises(SetConnections.SetConnectionsException,
+ pfd, r"hello\:there::")
+ self.assertRaises(SetConnections.SetConnectionsException,
+ pfd, "foobar\\")
if __name__ == "__main__": unittest.main()
diff --git a/rdiff-backup/testing/statictest.py b/rdiff-backup/testing/statictest.py
index a9ff812..17518fa 100644
--- a/rdiff-backup/testing/statictest.py
+++ b/rdiff-backup/testing/statictest.py
@@ -1,6 +1,6 @@
import unittest, types
-execfile("commontest.py")
-rbexec("static.py")
+from commontest import *
+from static import *
class D:
diff --git a/rdiff-backup/testing/statisticstest.py b/rdiff-backup/testing/statisticstest.py
index ef714f1..62ad0b7 100644
--- a/rdiff-backup/testing/statisticstest.py
+++ b/rdiff-backup/testing/statisticstest.py
@@ -1,6 +1,6 @@
import unittest
-execfile("commontest.py")
-rbexec("statistics.py")
+from commontest import *
+from statistics import *
class StatsObjTest(unittest.TestCase):
"""Test StatsObj class"""
diff --git a/rdiff-backup/testing/timetest.py b/rdiff-backup/testing/timetest.py
index d37862e..089ae0c 100644
--- a/rdiff-backup/testing/timetest.py
+++ b/rdiff-backup/testing/timetest.py
@@ -1,6 +1,6 @@
import unittest
-execfile("commontest.py")
-rbexec("highlevel.py")
+from commontest import *
+import Globals, Time
class TimeTest(unittest.TestCase):
def testConversion(self):
@@ -59,7 +59,7 @@ class TimeTest(unittest.TestCase):
i2s = Time.intstringtoseconds
for s in ["32", "", "d", "231I", "MM", "s", "-2h"]:
try: i2s(s)
- except TimeException: pass
+ except Time.TimeException: pass
else: assert 0, s
assert i2s("7D") == 7*86400
assert i2s("232s") == 232
@@ -104,9 +104,9 @@ class TimeTest(unittest.TestCase):
def testGenericStringErrors(self):
"""Test genstrtotime on some bad strings"""
g2t = Time.genstrtotime
- self.assertRaises(TimeException, g2t, "hello")
- self.assertRaises(TimeException, g2t, "")
- self.assertRaises(TimeException, g2t, "3q")
+ self.assertRaises(Time.TimeException, g2t, "hello")
+ self.assertRaises(Time.TimeException, g2t, "")
+ self.assertRaises(Time.TimeException, g2t, "3q")
if __name__ == '__main__': unittest.main()