From 4c8440ee71ba819c7327913870a615186ef8d386 Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 16 Jul 2002 05:16:42 +0000 Subject: Various changes to 0.9.3, see CHANGELOG git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@157 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109 --- rdiff-backup/rdiff_backup/Globals.py | 45 +++++++- rdiff-backup/rdiff_backup/Main.py | 75 +++++++----- rdiff-backup/rdiff_backup/Security.py | 171 ++++++++++++++++++++++++++++ rdiff-backup/rdiff_backup/SetConnections.py | 14 ++- rdiff-backup/rdiff_backup/Time.py | 28 ++++- rdiff-backup/rdiff_backup/cmodule.c | 4 +- rdiff-backup/rdiff_backup/connection.py | 47 ++++++-- rdiff-backup/rdiff_backup/increment.py | 4 +- rdiff-backup/rdiff_backup/rorpiter.py | 4 +- rdiff-backup/rdiff_backup/selection.py | 13 +++ rdiff-backup/rdiff_backup/statistics.py | 49 ++++++-- rdiff-backup/src/Globals.py | 45 +++++++- rdiff-backup/src/Main.py | 75 +++++++----- rdiff-backup/src/Security.py | 171 ++++++++++++++++++++++++++++ rdiff-backup/src/SetConnections.py | 14 ++- rdiff-backup/src/Time.py | 28 ++++- rdiff-backup/src/cmodule.c | 4 +- rdiff-backup/src/connection.py | 47 ++++++-- rdiff-backup/src/increment.py | 4 +- rdiff-backup/src/rorpiter.py | 4 +- rdiff-backup/src/selection.py | 13 +++ rdiff-backup/src/statistics.py | 49 ++++++-- rdiff-backup/testing/commontest.py | 16 ++- rdiff-backup/testing/connectiontest.py | 13 ++- rdiff-backup/testing/finaltest.py | 31 +++++ rdiff-backup/testing/securitytest.py | 60 ++++++++++ rdiff-backup/testing/selectiontest.py | 12 ++ rdiff-backup/testing/statisticstest.py | 1 + rdiff-backup/testing/timetest.py | 26 ++++- 29 files changed, 938 insertions(+), 129 deletions(-) create mode 100644 rdiff-backup/rdiff_backup/Security.py create mode 100644 rdiff-backup/src/Security.py create mode 100644 rdiff-backup/testing/securitytest.py (limited to 'rdiff-backup') diff --git a/rdiff-backup/rdiff_backup/Globals.py b/rdiff-backup/rdiff_backup/Globals.py index 0fc08f5..bf4a977 100644 --- a/rdiff-backup/rdiff_backup/Globals.py +++ b/rdiff-backup/rdiff_backup/Globals.py @@ -85,9 +85,6 @@ isbackup_writer = None # Connection of the backup writer backup_writer = None -# True if this process is the client invoked by the user -isclient = None - # Connection of the client client_conn = None @@ -171,6 +168,22 @@ select_source, select_mirror = None, None # object. Access is provided to increment error counts. ITRB = None +# Percentage of time to spend sleeping. None means never sleep. +sleep_ratio = None + +# security_level has 4 values and controls which requests from remote +# systems will be honored. "all" means anything goes. "read-only" +# means that the requests must not write to disk. "update-only" means +# that requests shouldn't destructively update the disk (but normal +# incremental updates are OK). "minimal" means only listen to a few +# basic requests. +security_level = "all" + +# If this is set, it indicates that the remote connection should only +# deal with paths inside of restrict_path. +restrict_path = None + + def get(name): """Return the value of something in this module""" return globals()[name] @@ -199,6 +212,32 @@ def set_integer(name, val): "received %s instead." % (name, val)) set(name, intval) +def set_float(name, val, min = None, max = None, inclusive = 1): + """Like set, but make sure val is float within given bounds""" + def error(): + s = "Variable %s must be set to a float" % (name,) + if min is not None and max is not None: + s += " between %s and %s " % (min, max) + if inclusive: s += "inclusive" + else: s += "not inclusive" + elif min is not None or max is not None: + if inclusive: inclusive_string = "or equal to " + else: inclusive_string = "" + if min is not None: + s += " greater than %s%s" % (inclusive_string, min) + else: s+= " less than %s%s" % (inclusive_string, max) + Log.FatalError(s) + + try: f = float(val) + except ValueError: error() + if min is not None: + if inclusive and f < min: error() + elif not inclusive and f <= min: error() + if max is not None: + if inclusive and f > max: error() + elif not inclusive and f >= max: error() + set(name, f) + def get_dict_val(name, key): """Return val from dictionary in this class""" return globals()[name][key] diff --git a/rdiff-backup/rdiff_backup/Main.py b/rdiff-backup/rdiff_backup/Main.py index 35624c2..06a1285 100644 --- a/rdiff-backup/rdiff_backup/Main.py +++ b/rdiff-backup/rdiff_backup/Main.py @@ -44,16 +44,17 @@ def parse_cmdlineoptions(arglist): "checkpoint-interval=", "current-time=", "exclude=", "exclude-device-files", "exclude-filelist=", "exclude-filelist-stdin", "exclude-mirror=", - "exclude-regexp=", "force", "include=", - "include-filelist=", "include-filelist-stdin", + "exclude-other-filesystems", "exclude-regexp=", "force", + "include=", "include-filelist=", "include-filelist-stdin", "include-regexp=", "list-increments", "mirror-only", - "no-compression", "no-compression-regexp=", - "no-hard-links", "no-resume", "null-separator", - "parsable-output", "print-statistics", "quoting-char=", - "remote-cmd=", "remote-schema=", "remove-older-than=", - "restore-as-of=", "resume", "resume-window=", "server", - "ssh-no-compression", "terminal-verbosity=", - "test-server", "verbosity", "version", "windows-mode", + "no-compression", "no-compression-regexp=", "no-hard-links", + "no-resume", "null-separator", "parsable-output", + "print-statistics", "quoting-char=", "remote-cmd=", + "remote-schema=", "remove-older-than=", "restore-as-of=", + "restrict=", "restrict-read-only=", "restrict-update-only=", + "resume", "resume-window=", "server", "sleep-ratio=", + "ssh-no-compression", "terminal-verbosity=", "test-server", + "verbosity", "version", "windows-mode", "windows-time-format"]) except getopt.error, e: commandline_error("Bad commandline options: %s" % str(e)) @@ -80,6 +81,8 @@ def parse_cmdlineoptions(arglist): select_files.append(sys.stdin) elif opt == "--exclude-mirror": select_mirror_opts.append(("--exclude", arg)) + elif opt == "--exclude-other-filesystems": + select_opts.append((opt, arg)) elif opt == "--exclude-regexp": select_opts.append((opt, arg)) elif opt == "--force": force = 1 elif opt == "--include": select_opts.append((opt, arg)) @@ -99,23 +102,34 @@ def parse_cmdlineoptions(arglist): elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0) elif opt == '--no-resume': Globals.resume = 0 elif opt == "--null-separator": Globals.set("null_separator", 1) - elif opt == "-r" or opt == "--restore-as-of": - restore_timestr, action = arg, "restore-as-of" elif opt == "--parsable-output": Globals.set('parsable_output', 1) elif opt == "--print-statistics": Globals.set('print_statistics', 1) elif opt == "--quoting-char": Globals.set('quoting_char', arg) Globals.set('quoting_enabled', 1) + elif opt == "-r" or opt == "--restore-as-of": + restore_timestr, action = arg, "restore-as-of" elif opt == "--remote-cmd": remote_cmd = arg elif opt == "--remote-schema": remote_schema = arg elif opt == "--remove-older-than": remove_older_than_string = arg action = "remove-older-than" + elif opt == "--restrict": Globals.restrict_path = arg + elif opt == "--restrict-read-only": + Globals.security_level = "read-only" + Globals.restrict_path = arg + elif opt == "--restrict-update-only": + Globals.security_level = "update-only" + Globals.restrict_path = arg elif opt == '--resume': Globals.resume = 1 elif opt == '--resume-window': Globals.set_integer('resume_window', arg) - elif opt == "-s" or opt == "--server": action = "server" + elif opt == "-s" or opt == "--server": + action = "server" + Globals.server = 1 + elif opt == "--sleep-ratio": + Globals.set_float("sleep_ratio", arg, 0, 1, inclusive=0) elif opt == "--ssh-no-compression": Globals.set('ssh_compression', None) elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg) @@ -176,7 +190,6 @@ def misc_setup(rps): os.umask(077) Time.setcurtime(Globals.current_time) FilenameMapping.set_init_quote_vals() - Globals.set("isclient", 1) SetConnections.UpdateGlobal("client_conn", Globals.local_connection) # This is because I originally didn't think compiled regexps @@ -209,7 +222,9 @@ def Main(arglist): """Start everything up!""" parse_cmdlineoptions(arglist) set_action() - rps = SetConnections.InitRPs(args, remote_schema, remote_cmd) + cmdpairs = SetConnections.get_cmd_pairs(args, remote_schema, remote_cmd) + Security.initialize(action, cmdpairs) + rps = map(SetConnections.cmdpair2rp, cmdpairs) misc_setup(rps) take_action(rps) cleanup() @@ -222,6 +237,7 @@ def Mirror(src_rp, dest_rp): # Since no "rdiff-backup-data" dir, use root of destination. SetConnections.UpdateGlobal('rbdir', dest_rp) SetConnections.BackupInitConnections(src_rp.conn, dest_rp.conn) + backup_init_select(src_rp, dest_rp) HighLevel.Mirror(src_rp, dest_rp) def mirror_check_paths(rpin, rpout): @@ -245,7 +261,7 @@ def Backup(rpin, rpout): Time.setprevtime(prevtime) HighLevel.Mirror_and_increment(rpin, rpout, incdir, RSI) else: HighLevel.Mirror(rpin, rpout, incdir, RSI) - backup_touch_curmirror(rpin, rpout) + rpout.conn.Main.backup_touch_curmirror_local(rpin, rpout) def backup_init_select(rpin, rpout): """Create Select objects on source and dest connections""" @@ -307,6 +323,7 @@ may need to use the --exclude option.""" % (rpout.path, rpin.path), 2) def backup_get_mirrorrps(): """Return list of current_mirror rps""" + datadir = Globals.rbdir if not datadir.isdir(): return [] mirrorrps = [datadir.append(fn) for fn in datadir.listdir() if fn.startswith("current_mirror.")] @@ -324,12 +341,14 @@ went wrong during your last backup? Using """ + mirrorrps[-1].path, 2) timestr = mirrorrps[-1].getinctime() return Time.stringtotime(timestr) -def backup_touch_curmirror(rpin, rpout): +def backup_touch_curmirror_local(rpin, rpout): """Make a file like current_mirror.time.data to record time - Also updates rpout so mod times don't get messed up. + Also updates rpout so mod times don't get messed up. This should + be run on the destination connection. """ + datadir = Globals.rbdir map(RPath.delete, backup_get_mirrorrps()) mirrorrp = datadir.append("current_mirror.%s.%s" % (Time.curtimestr, "data")) @@ -337,7 +356,6 @@ def backup_touch_curmirror(rpin, rpout): mirrorrp.touch() RPath.copy_attribs(rpin, rpout) - def restore(src_rp, dest_rp = None): """Main restoring function @@ -474,23 +492,24 @@ def RemoveOlderThan(rootrp): (datadir.path,)) try: time = Time.genstrtotime(remove_older_than_string) - except TimeError, exc: Log.FatalError(str(exc)) + except Time.TimeException, exc: Log.FatalError(str(exc)) timep = Time.timetopretty(time) Log("Deleting increment(s) before %s" % timep, 4) - itimes = [Time.stringtopretty(inc.getinctime()) - for inc in Restore.get_inclist(datadir.append("increments")) - if Time.stringtotime(inc.getinctime()) < time] - - if not itimes: + times_in_secs = map(lambda inc: Time.stringtotime(inc.getinctime()), + Restore.get_inclist(datadir.append("increments"))) + times_in_secs = filter(lambda t: t < time, times_in_secs) + if not times_in_secs: Log.FatalError("No increments older than %s found" % timep) - inc_pretty_time = "\n".join(itimes) - if len(itimes) > 1 and not force: + + times_in_secs.sort() + inc_pretty_time = "\n".join(map(Time.timetopretty, times_in_secs)) + if len(times_in_secs) > 1 and not force: Log.FatalError("Found %d relevant increments, dated:\n%s" "\nIf you want to delete multiple increments in this way, " - "use the --force." % (len(itimes), inc_pretty_time)) + "use the --force." % (len(times_in_secs), inc_pretty_time)) Log("Deleting increment%sat times:\n%s" % - (len(itimes) == 1 and " " or "s ", inc_pretty_time), 3) + (len(times_in_secs) == 1 and " " or "s ", inc_pretty_time), 3) Manage.delete_earlier_than(datadir, time) diff --git a/rdiff-backup/rdiff_backup/Security.py b/rdiff-backup/rdiff_backup/Security.py new file mode 100644 index 0000000..e4630d7 --- /dev/null +++ b/rdiff-backup/rdiff_backup/Security.py @@ -0,0 +1,171 @@ +# Copyright 2002 Ben Escoto +# +# This file is part of rdiff-backup. +# +# rdiff-backup is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA +# 02139, USA; either version 2 of the License, or (at your option) any +# later version; incorporated herein by reference. + +"""Functions to make sure remote requests are kosher""" + +import sys +import Globals, tempfile +from rpath import * + +class Violation(Exception): + """Exception that indicates an improper request has been received""" + pass + + +# This will store the list of functions that will be honored from +# remote connections. +allowed_requests = None + +# This stores the list of global variables that the client can not +# set on the server. +disallowed_server_globals = ["server", "security_level", "restrict_path"] + +def initialize(action, cmdpairs): + """Initialize allowable request list and chroot""" + global allowed_requests + set_security_level(action, cmdpairs) + set_allowed_requests(Globals.security_level) + +def set_security_level(action, cmdpairs): + """If running client, set security level and restrict_path + + To find these settings, we must look at the action to see what is + supposed to happen, and then look at the cmdpairs to see what end + the client is on. + + """ + def islocal(cmdpair): return not cmdpair[0] + def bothlocal(cp1, cp2): return islocal(cp1) and islocal(cp2) + def bothremote(cp1, cp2): return not islocal(cp1) and not islocal(cp2) + def getpath(cmdpair): return cmdpair[1] + + if Globals.server: return + cp1 = cmdpairs[0] + if len(cmdpairs) > 1: cp2 = cmdpairs[1] + + if action == "backup": + if bothlocal(cp1, cp2) or bothremote(cp1, cp2): + sec_level = "minimal" + rdir = tempfile.gettempdir() + elif islocal(cp1): + sec_level = "read-only" + rdir = getpath(cp1) + else: + assert islocal(cp2) + sec_level = "update-only" + rdir = getpath(cp2) + elif action == "restore" or action == "restore-as-of": + if len(cmdpairs) == 1 or bothlocal(cp1, cp2) or bothremote(cp1, cp2): + sec_level = "minimal" + rdir = tempfile.gettempdir() + elif islocal(cp1): + sec_level = "read-only" + else: + assert islocal(cp2) + sec_level = "all" + rdir = getpath(cp2) + elif action == "mirror": + if bothlocal(cp1, cp2) or bothremote(cp1, cp2): + sec_level = "minimal" + rdir = tempfile.gettempdir() + elif islocal(cp1): + sec_level = "read-only" + rdir = getpath(cp1) + else: + assert islocal(cp2) + sec_level = "all" + rdir = getpath(cp2) + elif (action == "test-server" or action == "list-increments" or + action == "calculate-average" or action == "remove-older-than"): + sec_level = "minimal" + rdir = tempfile.gettempdir() + else: assert 0, "Unknown action %s" % action + + Globals.security_level = sec_level + Globals.restrict_path = rdir + +def set_allowed_requests(sec_level): + """Set the allowed requests list using the security level""" + global allowed_requests + if sec_level == "all": return + allowed_requests = ["VirtualFile.readfromid", "VirtualFile.closebyid", + "Globals.get", "Globals.is_not_None", + "Globals.get_dict_val", + "Log.open_logfile_allconn", + "Log.close_logfile_allconn", + "SetConnections.add_redirected_conn", + "RedirectedRun"] + if sec_level == "minimal": pass + elif sec_level == "read-only" or sec_level == "update-only": + allowed_requests.extend(["C.make_file_dict", + "os.getuid", + "os.listdir", + "Resume.ResumeCheck", + "HLSourceStruct.split_initial_dsiter", + "HLSourceStruct.get_diffs_and_finalize"]) + if sec_level == "update-only": + allowed_requests. \ + extend(["Log.open_logfile_local", "Log.close_logfile_local", + "Log.close_logfile_allconn", "Log.log_to_file", + "SaveState.init_filenames", + "SaveState.touch_last_file", + "HLDestinationStruct.get_sigs", + "HLDestinationStruct.patch_w_datadir_writes", + "HLDestinationStruct.patch_and_finalize", + "HLDestinationStruct.patch_increment_and_finalize", + "Main.backup_touch_curmirror_local", + "Globals.ITRB.increment_stat"]) + if Globals.server: + allowed_requests.extend(["SetConnections.init_connection_remote", + "Log.setverbosity", + "Log.setterm_verbosity", + "Time.setcurtime_local", + "Time.setprevtime_local", + "FilenameMapping.set_init_quote_vals_local", + "Globals.postset_regexp_local", + "Globals.set_select", + "HLSourceStruct.set_session_info", + "HLDestinationStruct.set_session_info"]) + +def vet_request(request, arglist): + """Examine request for security violations""" + #if Globals.server: sys.stderr.write(str(request) + "\n") + security_level = Globals.security_level + if Globals.restrict_path: + for arg in arglist: + if isinstance(arg, RPath): vet_rpath(arg) + if security_level == "all": return + if request.function_string in allowed_requests: return + if request.function_string == "Globals.set": + if Globals.server and arglist[0] not in disallowed_server_globals: + return + raise Violation("\nWarning Security Violation!\n" + "Bad request for function: %s\n" + "with arguments: %s\n" % (request.function_string, + arglist)) + +def vet_rpath(rpath): + """Require rpath not to step outside retricted directory""" + if Globals.restrict_path and rpath.conn is Globals.local_connection: + normalized, restrict = rpath.normalize().path, Globals.restrict_path + components = normalized.split("/") + # 3 cases for restricted dir /usr/foo: /var, /usr/foobar, /usr/foo/.. + if (not normalized.startswith(restrict) or + (len(normalized) > len(restrict) and + normalized[len(restrict)] != "/") or + ".." in components): + raise Violation("\nWarning Security Violation!\n" + "Request to handle path %s\n" + "which doesn't appear to be within " + "restrict path %s.\n" % (normalized, restrict)) + + + + diff --git a/rdiff-backup/rdiff_backup/SetConnections.py b/rdiff-backup/rdiff_backup/SetConnections.py index 91edd7d..8d763a1 100644 --- a/rdiff-backup/rdiff_backup/SetConnections.py +++ b/rdiff-backup/rdiff_backup/SetConnections.py @@ -28,8 +28,13 @@ __conn_remote_cmds = [None] class SetConnectionsException(Exception): pass -def InitRPs(arglist, remote_schema = None, remote_cmd = None): - """Map the given file descriptions into rpaths and return list""" +def get_cmd_pairs(arglist, remote_schema = None, remote_cmd = None): + """Map the given file descriptions into command pairs + + Command pairs are tuples cmdpair with length 2. cmdpair[0] is + None iff it describes a local path, and cmdpair[1] is the path. + + """ global __cmd_schema if remote_schema: __cmd_schema = remote_schema elif not Globals.ssh_compression: __cmd_schema = __cmd_schema_no_compress @@ -44,11 +49,10 @@ def InitRPs(arglist, remote_schema = None, remote_cmd = None): elif remote_schema: Log("Remote schema option ignored - no remote file " "descriptions.", 2) - - cmd_pairs = map(desc2cmd_pairs, desc_pairs) + cmdpairs = map(desc2cmd_pairs, desc_pairs) if remote_cmd: # last file description gets remote_cmd cmd_pairs[-1] = (remote_cmd, cmd_pairs[-1][1]) - return map(cmdpair2rp, cmd_pairs) + return cmdpairs def cmdpair2rp(cmd_pair): """Return RPath from cmd_pair (remote_cmd, filename)""" diff --git a/rdiff-backup/rdiff_backup/Time.py b/rdiff-backup/rdiff_backup/Time.py index 6220514..90d3ae8 100644 --- a/rdiff-backup/rdiff_backup/Time.py +++ b/rdiff-backup/rdiff_backup/Time.py @@ -25,17 +25,18 @@ _genstr_date_regexp1 = re.compile("^(?P[0-9]{4})[-/]" _genstr_date_regexp2 = re.compile("^(?P[0-9]{1,2})[-/]" "(?P[0-9]{1,2})[-/](?P[0-9]{4})$") curtime = curtimestr = None +been_awake_since = None # stores last time sleep() was run def setcurtime(curtime = None): """Sets the current time in curtime and curtimestr on all systems""" t = curtime or time.time() for conn in Globals.connections: - conn.Time.setcurtime_local(t, timetostring(t)) + conn.Time.setcurtime_local(t) -def setcurtime_local(timeinseconds, timestr): +def setcurtime_local(timeinseconds): """Only set the current time locally""" global curtime, curtimestr - curtime, curtimestr = timeinseconds, timestr + curtime, curtimestr = timeinseconds, timetostring(timeinseconds) def setprevtime(timeinseconds): """Sets the previous inc time in prevtime and prevtimestr""" @@ -168,6 +169,25 @@ def cmp(time1, time2): elif time1 == time2: return 0 else: return 1 + +def sleep(sleep_ratio): + """Sleep for period to maintain given sleep_ratio + + On my system sleeping for periods less than 1/20th of a second + doesn't seem to work very accurately, so accumulate at least that + much time before sleeping. + + """ + global been_awake_since + if been_awake_since is None: # first running + been_awake_since = time.time() + else: + elapsed_time = time.time() - been_awake_since + sleep_time = elapsed_time * (sleep_ratio/(1-sleep_ratio)) + if sleep_time >= 0.05: + time.sleep(sleep_time) + been_awake_since = time.time() + def genstrtotime(timestr, curtime = None): """Convert a generic time string to a time in seconds""" if curtime is None: curtime = globals()['curtime'] @@ -203,5 +223,3 @@ the day).""" % timestr) t = stringtotime(timestr) if t: return t else: error() - - diff --git a/rdiff-backup/rdiff_backup/cmodule.c b/rdiff-backup/rdiff_backup/cmodule.c index cce87da..6a55ab4 100644 --- a/rdiff-backup/rdiff_backup/cmodule.c +++ b/rdiff-backup/rdiff_backup/cmodule.c @@ -49,7 +49,7 @@ static PyObject *c_make_file_dict(self, args) size = PyLong_FromLongLong((LONG_LONG)sbuf.st_size); inode = PyLong_FromLongLong((LONG_LONG)sbuf.st_ino); #else - size = PyInt_FromLong((long)sbuf.st_size); + size = PyInt_FromLong(sbuf.st_size); inode = PyInt_FromLong((long)sbuf.st_ino); #endif mode = (long)sbuf.st_mode; @@ -64,7 +64,7 @@ static PyObject *c_make_file_dict(self, args) atime = PyLong_FromLongLong((LONG_LONG)sbuf.st_atime); #else mtime = PyInt_FromLong((long)sbuf.st_mtime); - atime = PyLong_FromLongLong((long)sbuf.st_atime); + atime = PyInt_FromLong((long)sbuf.st_atime); #endif /* Build return dictionary from stat struct */ diff --git a/rdiff-backup/rdiff_backup/connection.py b/rdiff-backup/rdiff_backup/connection.py index 91c44fe..20dd3db 100644 --- a/rdiff-backup/rdiff_backup/connection.py +++ b/rdiff-backup/rdiff_backup/connection.py @@ -48,11 +48,9 @@ class LocalConnection(Connection): elif isinstance(__builtins__, dict): return __builtins__[name] else: return __builtins__.__dict__[name] - def __setattr__(self, name, value): - globals()[name] = value + def __setattr__(self, name, value): globals()[name] = value - def __delattr__(self, name): - del globals()[name] + def __delattr__(self, name): del globals()[name] def __str__(self): return "LocalConnection" @@ -329,7 +327,9 @@ class PipeConnection(LowLevelPipeConnection): arg_req_num, arg = self._get() assert arg_req_num == req_num argument_list.append(arg) - try: result = apply(eval(request.function_string), argument_list) + try: + Security.vet_request(request, argument_list) + result = apply(eval(request.function_string), argument_list) except: result = self.extract_exception() self._put(result, req_num) self.unused_request_numbers[req_num] = None @@ -407,14 +407,31 @@ class RedirectedConnection(Connection): self.routing_number = routing_number self.routing_conn = Globals.connection_dict[routing_number] + def reval(self, function_string, *args): + """Evalution function_string on args on remote connection""" + return self.routing_conn.reval("RedirectedRun", self.conn_number, + function_string, *args) + def __str__(self): return "RedirectedConnection %d,%d" % (self.conn_number, self.routing_number) def __getattr__(self, name): - return EmulateCallable(self.routing_conn, - "Globals.get_dict_val('connection_dict', %d).%s" - % (self.conn_number, name)) + return EmulateCallableRedirected(self.conn_number, self.routing_conn, + name) + +def RedirectedRun(conn_number, func, *args): + """Run func with args on connection with conn number conn_number + + This function is meant to redirect requests from one connection to + another, so conn_number must not be the local connection (and also + for security reasons since this function is always made + available). + + """ + conn = Globals.connection_dict[conn_number] + assert conn is not Globals.local_connection, conn + return conn.reval(func, *args) class EmulateCallable: @@ -428,6 +445,18 @@ class EmulateCallable: return EmulateCallable(self.connection, "%s.%s" % (self.name, attr_name)) +class EmulateCallableRedirected: + """Used by RedirectedConnection in calls like conn.os.chmod(foo)""" + def __init__(self, conn_number, routing_conn, name): + self.conn_number, self.routing_conn = conn_number, routing_conn + self.name = name + def __call__(self, *args): + return apply(self.routing_conn.reval, + ("RedirectedRun", self.conn_number, self.name) + args) + def __getattr__(self, attr_name): + return EmulateCallableRedirected(self.conn_number, self.routing_conn, + "%s.%s" % (self.name, attr_name)) + class VirtualFile: """When the client asks for a file over the connection, it gets this @@ -499,7 +528,7 @@ class VirtualFile: # everything has to be available here for remote connection's use, but # put at bottom to reduce circularities. -import Globals, Time, Rdiff, Hardlink, FilenameMapping, C +import Globals, Time, Rdiff, Hardlink, FilenameMapping, C, Security, Main from static import * from lazy import * from log import * diff --git a/rdiff-backup/rdiff_backup/increment.py b/rdiff-backup/rdiff_backup/increment.py index e3b7f5a..814322d 100644 --- a/rdiff-backup/rdiff_backup/increment.py +++ b/rdiff-backup/rdiff_backup/increment.py @@ -274,6 +274,7 @@ class IncrementITRB(StatsITRB): def branch_process(self, branch): """Update statistics, and the has_changed flag if change in branch""" + if Globals.sleep_ratio is not None: Time.sleep(Globals.sleep_ratio) if branch.changed: self.changed = 1 self.add_file_stats(branch) @@ -288,7 +289,7 @@ class MirrorITRB(StatsITRB): StatsITRB.__init__(self) def start_process(self, index, diff_rorp, mirror_dsrp): - """Initialize statistics, do actual writing to mirror""" + """Initialize statistics and do actual writing to mirror""" self.start_stats(mirror_dsrp) if diff_rorp and not diff_rorp.isplaceholder(): RORPIter.patchonce_action(None, mirror_dsrp, diff_rorp).execute() @@ -312,6 +313,7 @@ class MirrorITRB(StatsITRB): def branch_process(self, branch): """Update statistics with subdirectory results""" + if Globals.sleep_ratio is not None: Time.sleep(Globals.sleep_ratio) self.add_file_stats(branch) diff --git a/rdiff-backup/rdiff_backup/rorpiter.py b/rdiff-backup/rdiff_backup/rorpiter.py index cfd2d5f..1e85d55 100644 --- a/rdiff-backup/rdiff_backup/rorpiter.py +++ b/rdiff-backup/rdiff_backup/rorpiter.py @@ -242,7 +242,9 @@ class RORPIter: def init(): Hardlink.link_rp(diff_rorp, tf, basisrp) return Robust.make_tf_robustaction(init, tf, basisrp) elif basisrp and basisrp.isreg() and diff_rorp.isreg(): - assert diff_rorp.get_attached_filetype() == 'diff' + if diff_rorp.get_attached_filetype() != 'diff': + raise RPathException("File %s appears to have changed during" + " processing, skipping" % (basisrp.path,)) return Rdiff.patch_with_attribs_action(basisrp, diff_rorp) else: # Diff contains whole file, just copy it over if not basisrp: basisrp = base_rp.new_index(diff_rorp.index) diff --git a/rdiff-backup/rdiff_backup/selection.py b/rdiff-backup/rdiff_backup/selection.py index 58abdf0..c70857c 100644 --- a/rdiff-backup/rdiff_backup/selection.py +++ b/rdiff-backup/rdiff_backup/selection.py @@ -256,6 +256,8 @@ class Select: self.add_selection_func(self.filelist_get_sf( filelists[filelists_index], 0, arg)) filelists_index += 1 + elif opt == "--exclude-other-filesystems": + self.add_selection_func(self.other_filesystems_get_sf(0)) elif opt == "--exclude-regexp": self.add_selection_func(self.regexp_get_sf(arg, 0)) elif opt == "--include": @@ -416,6 +418,17 @@ probably isn't what you meant.""" % else: return (None, None) # dsrp greater, not initial sequence else: assert 0, "Include is %s, should be 0 or 1" % (include,) + def other_filesystems_get_sf(self, include): + """Return selection function matching files on other filesystems""" + assert include == 0 or include == 1 + root_devloc = self.dsrpath.getdevloc() + def sel_func(dsrp): + if dsrp.getdevloc() == root_devloc: return None + else: return include + sel_func.exclude = not include + sel_func.name = "Match other filesystems" + return sel_func + def regexp_get_sf(self, regexp_string, include): """Return selection function given by regexp_string""" assert include == 0 or include == 1 diff --git a/rdiff-backup/rdiff_backup/statistics.py b/rdiff-backup/rdiff_backup/statistics.py index e9f43dc..83a8858 100644 --- a/rdiff-backup/rdiff_backup/statistics.py +++ b/rdiff-backup/rdiff_backup/statistics.py @@ -28,7 +28,7 @@ class StatsObj: 'ChangedFiles', 'ChangedSourceSize', 'ChangedMirrorSize', 'IncrementFiles', 'IncrementFileSize') - stat_misc_attrs = ('Errors',) + stat_misc_attrs = ('Errors', 'TotalDestinationSizeChange') stat_time_attrs = ('StartTime', 'EndTime', 'ElapsedTime') stat_attrs = (('Filename',) + stat_time_attrs + stat_misc_attrs + stat_file_attrs) @@ -65,6 +65,26 @@ class StatsObj: """Add 1 to value of attribute""" self.__dict__[attr] += 1 + def get_total_dest_size_change(self): + """Return total destination size change + + This represents the total change in the size of the + rdiff-backup destination directory. + + """ + addvals = [self.NewFileSize, self.ChangedSourceSize, + self.IncrementFileSize] + subtractvals = [self.DeletedFileSize, self.ChangedMirrorSize] + for val in addvals + subtractvals: + if val is None: + result = None + break + else: + def addlist(l): return reduce(lambda x,y: x+y, l) + result = addlist(addvals) - addlist(subtractvals) + self.TotalDestinationSizeChange = result + return result + def get_stats_line(self, index, use_repr = 1): """Return one line abbreviated version of full stats string""" file_attrs = map(lambda attr: str(self.get_stat(attr)), @@ -95,7 +115,9 @@ class StatsObj: def get_stats_string(self): """Return extended string printing out statistics""" - return self.get_timestats_string() + self.get_filestats_string() + return "%s%s%s" % (self.get_timestats_string(), + self.get_filestats_string(), + self.get_miscstats_string()) def get_timestats_string(self): """Return portion of statistics string dealing with time""" @@ -112,8 +134,6 @@ class StatsObj: self.ElapsedTime = self.EndTime - self.StartTime timelist.append("ElapsedTime %.2f (%s)\n" % (self.ElapsedTime, Time.inttopretty(self.ElapsedTime))) - if self.Errors is not None: - timelist.append("Errors %d\n" % self.Errors) return "".join(timelist) def get_filestats_string(self): @@ -130,8 +150,23 @@ class StatsObj: return "".join(map(fileline, self.stat_file_pairs)) + def get_miscstats_string(self): + """Return portion of extended stat string about misc attributes""" + misc_string = "" + tdsc = self.get_total_dest_size_change() + if tdsc is not None: + misc_string += ("TotalDestinationSizeChange %s (%s)\n" % + (tdsc, self.get_byte_summary_string(tdsc))) + if self.Errors is not None: misc_string += "Errors %d\n" % self.Errors + return misc_string + def get_byte_summary_string(self, byte_count): """Turn byte count into human readable string like "7.23GB" """ + if byte_count < 0: + sign = "-" + byte_count = -byte_count + else: sign = "" + for abbrev_bytes, abbrev_string in self.byte_abbrev_list: if byte_count >= abbrev_bytes: # Now get 3 significant figures @@ -139,11 +174,11 @@ class StatsObj: if abbrev_count >= 100: precision = 0 elif abbrev_count >= 10: precision = 1 else: precision = 2 - return "%%.%df %s" % (precision, abbrev_string) \ + return "%s%%.%df %s" % (sign, precision, abbrev_string) \ % (abbrev_count,) byte_count = round(byte_count) - if byte_count == 1: return "1 byte" - else: return "%d bytes" % (byte_count,) + if byte_count == 1: return sign + "1 byte" + else: return "%s%d bytes" % (sign, byte_count) def get_stats_logstring(self, title): """Like get_stats_string, but add header and footer""" diff --git a/rdiff-backup/src/Globals.py b/rdiff-backup/src/Globals.py index 0fc08f5..bf4a977 100644 --- a/rdiff-backup/src/Globals.py +++ b/rdiff-backup/src/Globals.py @@ -85,9 +85,6 @@ isbackup_writer = None # Connection of the backup writer backup_writer = None -# True if this process is the client invoked by the user -isclient = None - # Connection of the client client_conn = None @@ -171,6 +168,22 @@ select_source, select_mirror = None, None # object. Access is provided to increment error counts. ITRB = None +# Percentage of time to spend sleeping. None means never sleep. +sleep_ratio = None + +# security_level has 4 values and controls which requests from remote +# systems will be honored. "all" means anything goes. "read-only" +# means that the requests must not write to disk. "update-only" means +# that requests shouldn't destructively update the disk (but normal +# incremental updates are OK). "minimal" means only listen to a few +# basic requests. +security_level = "all" + +# If this is set, it indicates that the remote connection should only +# deal with paths inside of restrict_path. +restrict_path = None + + def get(name): """Return the value of something in this module""" return globals()[name] @@ -199,6 +212,32 @@ def set_integer(name, val): "received %s instead." % (name, val)) set(name, intval) +def set_float(name, val, min = None, max = None, inclusive = 1): + """Like set, but make sure val is float within given bounds""" + def error(): + s = "Variable %s must be set to a float" % (name,) + if min is not None and max is not None: + s += " between %s and %s " % (min, max) + if inclusive: s += "inclusive" + else: s += "not inclusive" + elif min is not None or max is not None: + if inclusive: inclusive_string = "or equal to " + else: inclusive_string = "" + if min is not None: + s += " greater than %s%s" % (inclusive_string, min) + else: s+= " less than %s%s" % (inclusive_string, max) + Log.FatalError(s) + + try: f = float(val) + except ValueError: error() + if min is not None: + if inclusive and f < min: error() + elif not inclusive and f <= min: error() + if max is not None: + if inclusive and f > max: error() + elif not inclusive and f >= max: error() + set(name, f) + def get_dict_val(name, key): """Return val from dictionary in this class""" return globals()[name][key] diff --git a/rdiff-backup/src/Main.py b/rdiff-backup/src/Main.py index 35624c2..06a1285 100644 --- a/rdiff-backup/src/Main.py +++ b/rdiff-backup/src/Main.py @@ -44,16 +44,17 @@ def parse_cmdlineoptions(arglist): "checkpoint-interval=", "current-time=", "exclude=", "exclude-device-files", "exclude-filelist=", "exclude-filelist-stdin", "exclude-mirror=", - "exclude-regexp=", "force", "include=", - "include-filelist=", "include-filelist-stdin", + "exclude-other-filesystems", "exclude-regexp=", "force", + "include=", "include-filelist=", "include-filelist-stdin", "include-regexp=", "list-increments", "mirror-only", - "no-compression", "no-compression-regexp=", - "no-hard-links", "no-resume", "null-separator", - "parsable-output", "print-statistics", "quoting-char=", - "remote-cmd=", "remote-schema=", "remove-older-than=", - "restore-as-of=", "resume", "resume-window=", "server", - "ssh-no-compression", "terminal-verbosity=", - "test-server", "verbosity", "version", "windows-mode", + "no-compression", "no-compression-regexp=", "no-hard-links", + "no-resume", "null-separator", "parsable-output", + "print-statistics", "quoting-char=", "remote-cmd=", + "remote-schema=", "remove-older-than=", "restore-as-of=", + "restrict=", "restrict-read-only=", "restrict-update-only=", + "resume", "resume-window=", "server", "sleep-ratio=", + "ssh-no-compression", "terminal-verbosity=", "test-server", + "verbosity", "version", "windows-mode", "windows-time-format"]) except getopt.error, e: commandline_error("Bad commandline options: %s" % str(e)) @@ -80,6 +81,8 @@ def parse_cmdlineoptions(arglist): select_files.append(sys.stdin) elif opt == "--exclude-mirror": select_mirror_opts.append(("--exclude", arg)) + elif opt == "--exclude-other-filesystems": + select_opts.append((opt, arg)) elif opt == "--exclude-regexp": select_opts.append((opt, arg)) elif opt == "--force": force = 1 elif opt == "--include": select_opts.append((opt, arg)) @@ -99,23 +102,34 @@ def parse_cmdlineoptions(arglist): elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0) elif opt == '--no-resume': Globals.resume = 0 elif opt == "--null-separator": Globals.set("null_separator", 1) - elif opt == "-r" or opt == "--restore-as-of": - restore_timestr, action = arg, "restore-as-of" elif opt == "--parsable-output": Globals.set('parsable_output', 1) elif opt == "--print-statistics": Globals.set('print_statistics', 1) elif opt == "--quoting-char": Globals.set('quoting_char', arg) Globals.set('quoting_enabled', 1) + elif opt == "-r" or opt == "--restore-as-of": + restore_timestr, action = arg, "restore-as-of" elif opt == "--remote-cmd": remote_cmd = arg elif opt == "--remote-schema": remote_schema = arg elif opt == "--remove-older-than": remove_older_than_string = arg action = "remove-older-than" + elif opt == "--restrict": Globals.restrict_path = arg + elif opt == "--restrict-read-only": + Globals.security_level = "read-only" + Globals.restrict_path = arg + elif opt == "--restrict-update-only": + Globals.security_level = "update-only" + Globals.restrict_path = arg elif opt == '--resume': Globals.resume = 1 elif opt == '--resume-window': Globals.set_integer('resume_window', arg) - elif opt == "-s" or opt == "--server": action = "server" + elif opt == "-s" or opt == "--server": + action = "server" + Globals.server = 1 + elif opt == "--sleep-ratio": + Globals.set_float("sleep_ratio", arg, 0, 1, inclusive=0) elif opt == "--ssh-no-compression": Globals.set('ssh_compression', None) elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg) @@ -176,7 +190,6 @@ def misc_setup(rps): os.umask(077) Time.setcurtime(Globals.current_time) FilenameMapping.set_init_quote_vals() - Globals.set("isclient", 1) SetConnections.UpdateGlobal("client_conn", Globals.local_connection) # This is because I originally didn't think compiled regexps @@ -209,7 +222,9 @@ def Main(arglist): """Start everything up!""" parse_cmdlineoptions(arglist) set_action() - rps = SetConnections.InitRPs(args, remote_schema, remote_cmd) + cmdpairs = SetConnections.get_cmd_pairs(args, remote_schema, remote_cmd) + Security.initialize(action, cmdpairs) + rps = map(SetConnections.cmdpair2rp, cmdpairs) misc_setup(rps) take_action(rps) cleanup() @@ -222,6 +237,7 @@ def Mirror(src_rp, dest_rp): # Since no "rdiff-backup-data" dir, use root of destination. SetConnections.UpdateGlobal('rbdir', dest_rp) SetConnections.BackupInitConnections(src_rp.conn, dest_rp.conn) + backup_init_select(src_rp, dest_rp) HighLevel.Mirror(src_rp, dest_rp) def mirror_check_paths(rpin, rpout): @@ -245,7 +261,7 @@ def Backup(rpin, rpout): Time.setprevtime(prevtime) HighLevel.Mirror_and_increment(rpin, rpout, incdir, RSI) else: HighLevel.Mirror(rpin, rpout, incdir, RSI) - backup_touch_curmirror(rpin, rpout) + rpout.conn.Main.backup_touch_curmirror_local(rpin, rpout) def backup_init_select(rpin, rpout): """Create Select objects on source and dest connections""" @@ -307,6 +323,7 @@ may need to use the --exclude option.""" % (rpout.path, rpin.path), 2) def backup_get_mirrorrps(): """Return list of current_mirror rps""" + datadir = Globals.rbdir if not datadir.isdir(): return [] mirrorrps = [datadir.append(fn) for fn in datadir.listdir() if fn.startswith("current_mirror.")] @@ -324,12 +341,14 @@ went wrong during your last backup? Using """ + mirrorrps[-1].path, 2) timestr = mirrorrps[-1].getinctime() return Time.stringtotime(timestr) -def backup_touch_curmirror(rpin, rpout): +def backup_touch_curmirror_local(rpin, rpout): """Make a file like current_mirror.time.data to record time - Also updates rpout so mod times don't get messed up. + Also updates rpout so mod times don't get messed up. This should + be run on the destination connection. """ + datadir = Globals.rbdir map(RPath.delete, backup_get_mirrorrps()) mirrorrp = datadir.append("current_mirror.%s.%s" % (Time.curtimestr, "data")) @@ -337,7 +356,6 @@ def backup_touch_curmirror(rpin, rpout): mirrorrp.touch() RPath.copy_attribs(rpin, rpout) - def restore(src_rp, dest_rp = None): """Main restoring function @@ -474,23 +492,24 @@ def RemoveOlderThan(rootrp): (datadir.path,)) try: time = Time.genstrtotime(remove_older_than_string) - except TimeError, exc: Log.FatalError(str(exc)) + except Time.TimeException, exc: Log.FatalError(str(exc)) timep = Time.timetopretty(time) Log("Deleting increment(s) before %s" % timep, 4) - itimes = [Time.stringtopretty(inc.getinctime()) - for inc in Restore.get_inclist(datadir.append("increments")) - if Time.stringtotime(inc.getinctime()) < time] - - if not itimes: + times_in_secs = map(lambda inc: Time.stringtotime(inc.getinctime()), + Restore.get_inclist(datadir.append("increments"))) + times_in_secs = filter(lambda t: t < time, times_in_secs) + if not times_in_secs: Log.FatalError("No increments older than %s found" % timep) - inc_pretty_time = "\n".join(itimes) - if len(itimes) > 1 and not force: + + times_in_secs.sort() + inc_pretty_time = "\n".join(map(Time.timetopretty, times_in_secs)) + if len(times_in_secs) > 1 and not force: Log.FatalError("Found %d relevant increments, dated:\n%s" "\nIf you want to delete multiple increments in this way, " - "use the --force." % (len(itimes), inc_pretty_time)) + "use the --force." % (len(times_in_secs), inc_pretty_time)) Log("Deleting increment%sat times:\n%s" % - (len(itimes) == 1 and " " or "s ", inc_pretty_time), 3) + (len(times_in_secs) == 1 and " " or "s ", inc_pretty_time), 3) Manage.delete_earlier_than(datadir, time) diff --git a/rdiff-backup/src/Security.py b/rdiff-backup/src/Security.py new file mode 100644 index 0000000..e4630d7 --- /dev/null +++ b/rdiff-backup/src/Security.py @@ -0,0 +1,171 @@ +# Copyright 2002 Ben Escoto +# +# This file is part of rdiff-backup. +# +# rdiff-backup is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA +# 02139, USA; either version 2 of the License, or (at your option) any +# later version; incorporated herein by reference. + +"""Functions to make sure remote requests are kosher""" + +import sys +import Globals, tempfile +from rpath import * + +class Violation(Exception): + """Exception that indicates an improper request has been received""" + pass + + +# This will store the list of functions that will be honored from +# remote connections. +allowed_requests = None + +# This stores the list of global variables that the client can not +# set on the server. +disallowed_server_globals = ["server", "security_level", "restrict_path"] + +def initialize(action, cmdpairs): + """Initialize allowable request list and chroot""" + global allowed_requests + set_security_level(action, cmdpairs) + set_allowed_requests(Globals.security_level) + +def set_security_level(action, cmdpairs): + """If running client, set security level and restrict_path + + To find these settings, we must look at the action to see what is + supposed to happen, and then look at the cmdpairs to see what end + the client is on. + + """ + def islocal(cmdpair): return not cmdpair[0] + def bothlocal(cp1, cp2): return islocal(cp1) and islocal(cp2) + def bothremote(cp1, cp2): return not islocal(cp1) and not islocal(cp2) + def getpath(cmdpair): return cmdpair[1] + + if Globals.server: return + cp1 = cmdpairs[0] + if len(cmdpairs) > 1: cp2 = cmdpairs[1] + + if action == "backup": + if bothlocal(cp1, cp2) or bothremote(cp1, cp2): + sec_level = "minimal" + rdir = tempfile.gettempdir() + elif islocal(cp1): + sec_level = "read-only" + rdir = getpath(cp1) + else: + assert islocal(cp2) + sec_level = "update-only" + rdir = getpath(cp2) + elif action == "restore" or action == "restore-as-of": + if len(cmdpairs) == 1 or bothlocal(cp1, cp2) or bothremote(cp1, cp2): + sec_level = "minimal" + rdir = tempfile.gettempdir() + elif islocal(cp1): + sec_level = "read-only" + else: + assert islocal(cp2) + sec_level = "all" + rdir = getpath(cp2) + elif action == "mirror": + if bothlocal(cp1, cp2) or bothremote(cp1, cp2): + sec_level = "minimal" + rdir = tempfile.gettempdir() + elif islocal(cp1): + sec_level = "read-only" + rdir = getpath(cp1) + else: + assert islocal(cp2) + sec_level = "all" + rdir = getpath(cp2) + elif (action == "test-server" or action == "list-increments" or + action == "calculate-average" or action == "remove-older-than"): + sec_level = "minimal" + rdir = tempfile.gettempdir() + else: assert 0, "Unknown action %s" % action + + Globals.security_level = sec_level + Globals.restrict_path = rdir + +def set_allowed_requests(sec_level): + """Set the allowed requests list using the security level""" + global allowed_requests + if sec_level == "all": return + allowed_requests = ["VirtualFile.readfromid", "VirtualFile.closebyid", + "Globals.get", "Globals.is_not_None", + "Globals.get_dict_val", + "Log.open_logfile_allconn", + "Log.close_logfile_allconn", + "SetConnections.add_redirected_conn", + "RedirectedRun"] + if sec_level == "minimal": pass + elif sec_level == "read-only" or sec_level == "update-only": + allowed_requests.extend(["C.make_file_dict", + "os.getuid", + "os.listdir", + "Resume.ResumeCheck", + "HLSourceStruct.split_initial_dsiter", + "HLSourceStruct.get_diffs_and_finalize"]) + if sec_level == "update-only": + allowed_requests. \ + extend(["Log.open_logfile_local", "Log.close_logfile_local", + "Log.close_logfile_allconn", "Log.log_to_file", + "SaveState.init_filenames", + "SaveState.touch_last_file", + "HLDestinationStruct.get_sigs", + "HLDestinationStruct.patch_w_datadir_writes", + "HLDestinationStruct.patch_and_finalize", + "HLDestinationStruct.patch_increment_and_finalize", + "Main.backup_touch_curmirror_local", + "Globals.ITRB.increment_stat"]) + if Globals.server: + allowed_requests.extend(["SetConnections.init_connection_remote", + "Log.setverbosity", + "Log.setterm_verbosity", + "Time.setcurtime_local", + "Time.setprevtime_local", + "FilenameMapping.set_init_quote_vals_local", + "Globals.postset_regexp_local", + "Globals.set_select", + "HLSourceStruct.set_session_info", + "HLDestinationStruct.set_session_info"]) + +def vet_request(request, arglist): + """Examine request for security violations""" + #if Globals.server: sys.stderr.write(str(request) + "\n") + security_level = Globals.security_level + if Globals.restrict_path: + for arg in arglist: + if isinstance(arg, RPath): vet_rpath(arg) + if security_level == "all": return + if request.function_string in allowed_requests: return + if request.function_string == "Globals.set": + if Globals.server and arglist[0] not in disallowed_server_globals: + return + raise Violation("\nWarning Security Violation!\n" + "Bad request for function: %s\n" + "with arguments: %s\n" % (request.function_string, + arglist)) + +def vet_rpath(rpath): + """Require rpath not to step outside retricted directory""" + if Globals.restrict_path and rpath.conn is Globals.local_connection: + normalized, restrict = rpath.normalize().path, Globals.restrict_path + components = normalized.split("/") + # 3 cases for restricted dir /usr/foo: /var, /usr/foobar, /usr/foo/.. + if (not normalized.startswith(restrict) or + (len(normalized) > len(restrict) and + normalized[len(restrict)] != "/") or + ".." in components): + raise Violation("\nWarning Security Violation!\n" + "Request to handle path %s\n" + "which doesn't appear to be within " + "restrict path %s.\n" % (normalized, restrict)) + + + + diff --git a/rdiff-backup/src/SetConnections.py b/rdiff-backup/src/SetConnections.py index 91edd7d..8d763a1 100644 --- a/rdiff-backup/src/SetConnections.py +++ b/rdiff-backup/src/SetConnections.py @@ -28,8 +28,13 @@ __conn_remote_cmds = [None] class SetConnectionsException(Exception): pass -def InitRPs(arglist, remote_schema = None, remote_cmd = None): - """Map the given file descriptions into rpaths and return list""" +def get_cmd_pairs(arglist, remote_schema = None, remote_cmd = None): + """Map the given file descriptions into command pairs + + Command pairs are tuples cmdpair with length 2. cmdpair[0] is + None iff it describes a local path, and cmdpair[1] is the path. + + """ global __cmd_schema if remote_schema: __cmd_schema = remote_schema elif not Globals.ssh_compression: __cmd_schema = __cmd_schema_no_compress @@ -44,11 +49,10 @@ def InitRPs(arglist, remote_schema = None, remote_cmd = None): elif remote_schema: Log("Remote schema option ignored - no remote file " "descriptions.", 2) - - cmd_pairs = map(desc2cmd_pairs, desc_pairs) + cmdpairs = map(desc2cmd_pairs, desc_pairs) if remote_cmd: # last file description gets remote_cmd cmd_pairs[-1] = (remote_cmd, cmd_pairs[-1][1]) - return map(cmdpair2rp, cmd_pairs) + return cmdpairs def cmdpair2rp(cmd_pair): """Return RPath from cmd_pair (remote_cmd, filename)""" diff --git a/rdiff-backup/src/Time.py b/rdiff-backup/src/Time.py index 6220514..90d3ae8 100644 --- a/rdiff-backup/src/Time.py +++ b/rdiff-backup/src/Time.py @@ -25,17 +25,18 @@ _genstr_date_regexp1 = re.compile("^(?P[0-9]{4})[-/]" _genstr_date_regexp2 = re.compile("^(?P[0-9]{1,2})[-/]" "(?P[0-9]{1,2})[-/](?P[0-9]{4})$") curtime = curtimestr = None +been_awake_since = None # stores last time sleep() was run def setcurtime(curtime = None): """Sets the current time in curtime and curtimestr on all systems""" t = curtime or time.time() for conn in Globals.connections: - conn.Time.setcurtime_local(t, timetostring(t)) + conn.Time.setcurtime_local(t) -def setcurtime_local(timeinseconds, timestr): +def setcurtime_local(timeinseconds): """Only set the current time locally""" global curtime, curtimestr - curtime, curtimestr = timeinseconds, timestr + curtime, curtimestr = timeinseconds, timetostring(timeinseconds) def setprevtime(timeinseconds): """Sets the previous inc time in prevtime and prevtimestr""" @@ -168,6 +169,25 @@ def cmp(time1, time2): elif time1 == time2: return 0 else: return 1 + +def sleep(sleep_ratio): + """Sleep for period to maintain given sleep_ratio + + On my system sleeping for periods less than 1/20th of a second + doesn't seem to work very accurately, so accumulate at least that + much time before sleeping. + + """ + global been_awake_since + if been_awake_since is None: # first running + been_awake_since = time.time() + else: + elapsed_time = time.time() - been_awake_since + sleep_time = elapsed_time * (sleep_ratio/(1-sleep_ratio)) + if sleep_time >= 0.05: + time.sleep(sleep_time) + been_awake_since = time.time() + def genstrtotime(timestr, curtime = None): """Convert a generic time string to a time in seconds""" if curtime is None: curtime = globals()['curtime'] @@ -203,5 +223,3 @@ the day).""" % timestr) t = stringtotime(timestr) if t: return t else: error() - - diff --git a/rdiff-backup/src/cmodule.c b/rdiff-backup/src/cmodule.c index cce87da..6a55ab4 100644 --- a/rdiff-backup/src/cmodule.c +++ b/rdiff-backup/src/cmodule.c @@ -49,7 +49,7 @@ static PyObject *c_make_file_dict(self, args) size = PyLong_FromLongLong((LONG_LONG)sbuf.st_size); inode = PyLong_FromLongLong((LONG_LONG)sbuf.st_ino); #else - size = PyInt_FromLong((long)sbuf.st_size); + size = PyInt_FromLong(sbuf.st_size); inode = PyInt_FromLong((long)sbuf.st_ino); #endif mode = (long)sbuf.st_mode; @@ -64,7 +64,7 @@ static PyObject *c_make_file_dict(self, args) atime = PyLong_FromLongLong((LONG_LONG)sbuf.st_atime); #else mtime = PyInt_FromLong((long)sbuf.st_mtime); - atime = PyLong_FromLongLong((long)sbuf.st_atime); + atime = PyInt_FromLong((long)sbuf.st_atime); #endif /* Build return dictionary from stat struct */ diff --git a/rdiff-backup/src/connection.py b/rdiff-backup/src/connection.py index 91c44fe..20dd3db 100644 --- a/rdiff-backup/src/connection.py +++ b/rdiff-backup/src/connection.py @@ -48,11 +48,9 @@ class LocalConnection(Connection): elif isinstance(__builtins__, dict): return __builtins__[name] else: return __builtins__.__dict__[name] - def __setattr__(self, name, value): - globals()[name] = value + def __setattr__(self, name, value): globals()[name] = value - def __delattr__(self, name): - del globals()[name] + def __delattr__(self, name): del globals()[name] def __str__(self): return "LocalConnection" @@ -329,7 +327,9 @@ class PipeConnection(LowLevelPipeConnection): arg_req_num, arg = self._get() assert arg_req_num == req_num argument_list.append(arg) - try: result = apply(eval(request.function_string), argument_list) + try: + Security.vet_request(request, argument_list) + result = apply(eval(request.function_string), argument_list) except: result = self.extract_exception() self._put(result, req_num) self.unused_request_numbers[req_num] = None @@ -407,14 +407,31 @@ class RedirectedConnection(Connection): self.routing_number = routing_number self.routing_conn = Globals.connection_dict[routing_number] + def reval(self, function_string, *args): + """Evalution function_string on args on remote connection""" + return self.routing_conn.reval("RedirectedRun", self.conn_number, + function_string, *args) + def __str__(self): return "RedirectedConnection %d,%d" % (self.conn_number, self.routing_number) def __getattr__(self, name): - return EmulateCallable(self.routing_conn, - "Globals.get_dict_val('connection_dict', %d).%s" - % (self.conn_number, name)) + return EmulateCallableRedirected(self.conn_number, self.routing_conn, + name) + +def RedirectedRun(conn_number, func, *args): + """Run func with args on connection with conn number conn_number + + This function is meant to redirect requests from one connection to + another, so conn_number must not be the local connection (and also + for security reasons since this function is always made + available). + + """ + conn = Globals.connection_dict[conn_number] + assert conn is not Globals.local_connection, conn + return conn.reval(func, *args) class EmulateCallable: @@ -428,6 +445,18 @@ class EmulateCallable: return EmulateCallable(self.connection, "%s.%s" % (self.name, attr_name)) +class EmulateCallableRedirected: + """Used by RedirectedConnection in calls like conn.os.chmod(foo)""" + def __init__(self, conn_number, routing_conn, name): + self.conn_number, self.routing_conn = conn_number, routing_conn + self.name = name + def __call__(self, *args): + return apply(self.routing_conn.reval, + ("RedirectedRun", self.conn_number, self.name) + args) + def __getattr__(self, attr_name): + return EmulateCallableRedirected(self.conn_number, self.routing_conn, + "%s.%s" % (self.name, attr_name)) + class VirtualFile: """When the client asks for a file over the connection, it gets this @@ -499,7 +528,7 @@ class VirtualFile: # everything has to be available here for remote connection's use, but # put at bottom to reduce circularities. -import Globals, Time, Rdiff, Hardlink, FilenameMapping, C +import Globals, Time, Rdiff, Hardlink, FilenameMapping, C, Security, Main from static import * from lazy import * from log import * diff --git a/rdiff-backup/src/increment.py b/rdiff-backup/src/increment.py index e3b7f5a..814322d 100644 --- a/rdiff-backup/src/increment.py +++ b/rdiff-backup/src/increment.py @@ -274,6 +274,7 @@ class IncrementITRB(StatsITRB): def branch_process(self, branch): """Update statistics, and the has_changed flag if change in branch""" + if Globals.sleep_ratio is not None: Time.sleep(Globals.sleep_ratio) if branch.changed: self.changed = 1 self.add_file_stats(branch) @@ -288,7 +289,7 @@ class MirrorITRB(StatsITRB): StatsITRB.__init__(self) def start_process(self, index, diff_rorp, mirror_dsrp): - """Initialize statistics, do actual writing to mirror""" + """Initialize statistics and do actual writing to mirror""" self.start_stats(mirror_dsrp) if diff_rorp and not diff_rorp.isplaceholder(): RORPIter.patchonce_action(None, mirror_dsrp, diff_rorp).execute() @@ -312,6 +313,7 @@ class MirrorITRB(StatsITRB): def branch_process(self, branch): """Update statistics with subdirectory results""" + if Globals.sleep_ratio is not None: Time.sleep(Globals.sleep_ratio) self.add_file_stats(branch) diff --git a/rdiff-backup/src/rorpiter.py b/rdiff-backup/src/rorpiter.py index cfd2d5f..1e85d55 100644 --- a/rdiff-backup/src/rorpiter.py +++ b/rdiff-backup/src/rorpiter.py @@ -242,7 +242,9 @@ class RORPIter: def init(): Hardlink.link_rp(diff_rorp, tf, basisrp) return Robust.make_tf_robustaction(init, tf, basisrp) elif basisrp and basisrp.isreg() and diff_rorp.isreg(): - assert diff_rorp.get_attached_filetype() == 'diff' + if diff_rorp.get_attached_filetype() != 'diff': + raise RPathException("File %s appears to have changed during" + " processing, skipping" % (basisrp.path,)) return Rdiff.patch_with_attribs_action(basisrp, diff_rorp) else: # Diff contains whole file, just copy it over if not basisrp: basisrp = base_rp.new_index(diff_rorp.index) diff --git a/rdiff-backup/src/selection.py b/rdiff-backup/src/selection.py index 58abdf0..c70857c 100644 --- a/rdiff-backup/src/selection.py +++ b/rdiff-backup/src/selection.py @@ -256,6 +256,8 @@ class Select: self.add_selection_func(self.filelist_get_sf( filelists[filelists_index], 0, arg)) filelists_index += 1 + elif opt == "--exclude-other-filesystems": + self.add_selection_func(self.other_filesystems_get_sf(0)) elif opt == "--exclude-regexp": self.add_selection_func(self.regexp_get_sf(arg, 0)) elif opt == "--include": @@ -416,6 +418,17 @@ probably isn't what you meant.""" % else: return (None, None) # dsrp greater, not initial sequence else: assert 0, "Include is %s, should be 0 or 1" % (include,) + def other_filesystems_get_sf(self, include): + """Return selection function matching files on other filesystems""" + assert include == 0 or include == 1 + root_devloc = self.dsrpath.getdevloc() + def sel_func(dsrp): + if dsrp.getdevloc() == root_devloc: return None + else: return include + sel_func.exclude = not include + sel_func.name = "Match other filesystems" + return sel_func + def regexp_get_sf(self, regexp_string, include): """Return selection function given by regexp_string""" assert include == 0 or include == 1 diff --git a/rdiff-backup/src/statistics.py b/rdiff-backup/src/statistics.py index e9f43dc..83a8858 100644 --- a/rdiff-backup/src/statistics.py +++ b/rdiff-backup/src/statistics.py @@ -28,7 +28,7 @@ class StatsObj: 'ChangedFiles', 'ChangedSourceSize', 'ChangedMirrorSize', 'IncrementFiles', 'IncrementFileSize') - stat_misc_attrs = ('Errors',) + stat_misc_attrs = ('Errors', 'TotalDestinationSizeChange') stat_time_attrs = ('StartTime', 'EndTime', 'ElapsedTime') stat_attrs = (('Filename',) + stat_time_attrs + stat_misc_attrs + stat_file_attrs) @@ -65,6 +65,26 @@ class StatsObj: """Add 1 to value of attribute""" self.__dict__[attr] += 1 + def get_total_dest_size_change(self): + """Return total destination size change + + This represents the total change in the size of the + rdiff-backup destination directory. + + """ + addvals = [self.NewFileSize, self.ChangedSourceSize, + self.IncrementFileSize] + subtractvals = [self.DeletedFileSize, self.ChangedMirrorSize] + for val in addvals + subtractvals: + if val is None: + result = None + break + else: + def addlist(l): return reduce(lambda x,y: x+y, l) + result = addlist(addvals) - addlist(subtractvals) + self.TotalDestinationSizeChange = result + return result + def get_stats_line(self, index, use_repr = 1): """Return one line abbreviated version of full stats string""" file_attrs = map(lambda attr: str(self.get_stat(attr)), @@ -95,7 +115,9 @@ class StatsObj: def get_stats_string(self): """Return extended string printing out statistics""" - return self.get_timestats_string() + self.get_filestats_string() + return "%s%s%s" % (self.get_timestats_string(), + self.get_filestats_string(), + self.get_miscstats_string()) def get_timestats_string(self): """Return portion of statistics string dealing with time""" @@ -112,8 +134,6 @@ class StatsObj: self.ElapsedTime = self.EndTime - self.StartTime timelist.append("ElapsedTime %.2f (%s)\n" % (self.ElapsedTime, Time.inttopretty(self.ElapsedTime))) - if self.Errors is not None: - timelist.append("Errors %d\n" % self.Errors) return "".join(timelist) def get_filestats_string(self): @@ -130,8 +150,23 @@ class StatsObj: return "".join(map(fileline, self.stat_file_pairs)) + def get_miscstats_string(self): + """Return portion of extended stat string about misc attributes""" + misc_string = "" + tdsc = self.get_total_dest_size_change() + if tdsc is not None: + misc_string += ("TotalDestinationSizeChange %s (%s)\n" % + (tdsc, self.get_byte_summary_string(tdsc))) + if self.Errors is not None: misc_string += "Errors %d\n" % self.Errors + return misc_string + def get_byte_summary_string(self, byte_count): """Turn byte count into human readable string like "7.23GB" """ + if byte_count < 0: + sign = "-" + byte_count = -byte_count + else: sign = "" + for abbrev_bytes, abbrev_string in self.byte_abbrev_list: if byte_count >= abbrev_bytes: # Now get 3 significant figures @@ -139,11 +174,11 @@ class StatsObj: if abbrev_count >= 100: precision = 0 elif abbrev_count >= 10: precision = 1 else: precision = 2 - return "%%.%df %s" % (precision, abbrev_string) \ + return "%s%%.%df %s" % (sign, precision, abbrev_string) \ % (abbrev_count,) byte_count = round(byte_count) - if byte_count == 1: return "1 byte" - else: return "%d bytes" % (byte_count,) + if byte_count == 1: return sign + "1 byte" + else: return "%s%d bytes" % (sign, byte_count) def get_stats_logstring(self, title): """Like get_stats_string, but add header and footer""" diff --git a/rdiff-backup/testing/commontest.py b/rdiff-backup/testing/commontest.py index 24eb2cb..dd49394 100644 --- a/rdiff-backup/testing/commontest.py +++ b/rdiff-backup/testing/commontest.py @@ -54,6 +54,15 @@ def rdiff_backup(source_local, dest_local, src_dir, dest_dir, os.system(" ".join(cmdargs)) +def cmd_schemas2rps(schema_list, remote_schema): + """Input list of file descriptions and the remote schema, return rps + + File descriptions should be strings of the form 'hostname.net::foo' + + """ + return map(SetConnections.cmdpair2rp, + SetConnections.get_cmd_pairs(schema_list, remote_schema)) + def InternalBackup(source_local, dest_local, src_dir, dest_dir, current_time = None): """Backup src to dest internally @@ -75,7 +84,7 @@ def InternalBackup(source_local, dest_local, src_dir, dest_dir, dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \ % (SourceDir, dest_dir) - rpin, rpout = SetConnections.InitRPs([src_dir, dest_dir], remote_schema) + rpin, rpout = cmd_schemas2rps([src_dir, dest_dir], remote_schema) Main.misc_setup([rpin, rpout]) Main.Backup(rpin, rpout) Main.cleanup() @@ -92,7 +101,7 @@ def InternalMirror(source_local, dest_local, src_dir, dest_dir, dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \ % (SourceDir, dest_dir) - rpin, rpout = SetConnections.InitRPs([src_dir, dest_dir], remote_schema) + rpin, rpout = cmd_schemas2rps([src_dir, dest_dir], remote_schema) Main.misc_setup([rpin, rpout]) Main.backup_init_select(rpin, rpout) if not rpout.lstat(): rpout.mkdir() @@ -127,8 +136,7 @@ def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time): dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \ % (SourceDir, dest_dir) - mirror_rp, dest_rp = SetConnections.InitRPs([mirror_dir, dest_dir], - remote_schema) + mirror_rp, dest_rp = cmd_schemas2rps([mirror_dir, dest_dir], remote_schema) Time.setcurtime() inc = get_increment_rp(mirror_rp, time) if inc: Main.restore(get_increment_rp(mirror_rp, time), dest_rp) diff --git a/rdiff-backup/testing/connectiontest.py b/rdiff-backup/testing/connectiontest.py index 1deadbe..ab80256 100644 --- a/rdiff-backup/testing/connectiontest.py +++ b/rdiff-backup/testing/connectiontest.py @@ -168,13 +168,22 @@ class RedirectedConnectionTest(unittest.TestCase): def testBasic(self): """Test basic operations with redirection""" + self.conna.Globals.set("tmp_val", 1) + self.connb.Globals.set("tmp_val", 2) + assert self.conna.Globals.get("tmp_val") == 1 + assert self.connb.Globals.get("tmp_val") == 2 + self.conna.Globals.set("tmp_connb", self.connb) self.connb.Globals.set("tmp_conna", self.conna) assert self.conna.Globals.get("tmp_connb") is self.connb assert self.connb.Globals.get("tmp_conna") is self.conna - #self.conna.Test_SetConnGlobals(self.connb, "tmp_settest", 1) - #assert self.connb.Globals.get("tmp_settest") + val = self.conna.reval("Globals.get('tmp_connb').Globals.get", + "tmp_val") + assert val == 2, val + val = self.connb.reval("Globals.get('tmp_conna').Globals.get", + "tmp_val") + assert val == 1, val assert self.conna.reval("Globals.get('tmp_connb').pow", 2, 3) == 8 self.conna.reval("Globals.tmp_connb.reval", diff --git a/rdiff-backup/testing/finaltest.py b/rdiff-backup/testing/finaltest.py index 0a51485..43daef6 100644 --- a/rdiff-backup/testing/finaltest.py +++ b/rdiff-backup/testing/finaltest.py @@ -17,6 +17,7 @@ class Local: def get_local_rp(extension): return RPath(Globals.local_connection, "testfiles/" + extension) + vftrp = get_local_rp('various_file_types') inc1rp = get_local_rp('increment1') inc2rp = get_local_rp('increment2') inc3rp = get_local_rp('increment3') @@ -71,6 +72,19 @@ class PathSetter(unittest.TestCase): print "executing " + cmdstr assert not os.system(cmdstr) + def exec_rb_extra_args(self, time, extra_args, *args): + """Run rdiff-backup on given arguments""" + arglist = [] + if time: arglist.append("--current-time %s" % str(time)) + arglist.append(self.src_prefix + args[0]) + if len(args) > 1: + arglist.append(self.dest_prefix + args[1]) + assert len(args) == 2 + + cmdstr = "%s %s %s" % (self.rb_schema, extra_args, ' '.join(arglist)) + print "executing " + cmdstr + assert not os.system(cmdstr) + def exec_rb_restore(self, time, *args): """Restore using rdiff-backup's new syntax and given time""" arglist = [] @@ -174,6 +188,23 @@ class Final(PathSetter): self.set_connections("test1/", '../', 'test2/tmp/', '../../') self.runtest() + def testMirroringLocal(self): + """Run mirroring only everything remote""" + self.delete_tmpdirs() + self.set_connections(None, None, None, None) + self.exec_rb_extra_args(10000, "-m", + "testfiles/various_file_types", + "testfiles/output") + assert CompareRecursive(Local.vftrp, Local.rpout, exclude_rbdir = None) + + def testMirroringRemote(self): + """Run mirroring only everything remote""" + self.delete_tmpdirs() + self.set_connections("test1/", "../", "test2/tmp/", "../../") + self.exec_rb_extra_args(10000, "-m", + "testfiles/various_file_types", + "testfiles/output") + assert CompareRecursive(Local.vftrp, Local.rpout, exclude_rbdir = None) class FinalSelection(PathSetter): """Test selection options""" diff --git a/rdiff-backup/testing/securitytest.py b/rdiff-backup/testing/securitytest.py new file mode 100644 index 0000000..689544d --- /dev/null +++ b/rdiff-backup/testing/securitytest.py @@ -0,0 +1,60 @@ +import os, unittest +from commontest import * +import rdiff_backup.Security, Security + +#Log.setverbosity(5) + +class SecurityTest(unittest.TestCase): + def assert_exc_sec(self, exc): + """Fudge - make sure exception is a security violation + + This is necessary because of some kind of pickling/module + problem. + + """ + assert isinstance(exc, rdiff_backup.Security.Violation) + #assert str(exc).find("Security") >= 0, "%s\n%s" % (exc, repr(exc)) + + def test_vet_request_ro(self): + """Test vetting of ConnectionRequests on read-only server""" + remote_cmd = "rdiff-backup --server --restrict-read-only foo" + conn = SetConnections.init_connection(remote_cmd) + assert type(conn.os.getuid()) is type(5) + try: conn.os.remove("/tmp/foobar") + except Exception, e: self.assert_exc_sec(e) + else: assert 0, "No exception raised" + SetConnections.CloseConnections() + + def test_vet_request_minimal(self): + """Test vetting of ConnectionRequests on minimal server""" + remote_cmd = "rdiff-backup --server --restrict-update-only foo" + conn = SetConnections.init_connection(remote_cmd) + assert type(conn.os.getuid()) is type(5) + try: conn.os.remove("/tmp/foobar") + except Exception, e: self.assert_exc_sec(e) + else: assert 0, "No exception raised" + SetConnections.CloseConnections() + + def test_vet_rpath(self): + """Test to make sure rpaths not in restricted path will be rejected""" + remote_cmd = "rdiff-backup --server --restrict-update-only foo" + conn = SetConnections.init_connection(remote_cmd) + + for rp in [RPath(Globals.local_connection, "blahblah"), + RPath(conn, "foo/bar")]: + conn.Globals.set("TEST_var", rp) + assert conn.Globals.get("TEST_var").path == rp.path + + for rp in [RPath(conn, "foobar"), + RPath(conn, "/usr/local"), + RPath(conn, "foo/../bar")]: + try: conn.Globals.set("TEST_var", rp) + except Exception, e: + self.assert_exc_sec(e) + continue + assert 0, "No violation raised by rp %s" % (rp,) + + SetConnections.CloseConnections() + +if __name__ == "__main__": unittest.main() + diff --git a/rdiff-backup/testing/selectiontest.py b/rdiff-backup/testing/selectiontest.py index 76af96f..f512f12 100644 --- a/rdiff-backup/testing/selectiontest.py +++ b/rdiff-backup/testing/selectiontest.py @@ -221,6 +221,18 @@ testfiles/select/1/1 select.filelist_get_sf(StringIO.StringIO("/foo/bar"), 0, "test")(root) == None + def testOtherFilesystems(self): + """Test to see if --exclude-other-filesystems works correctly""" + root = DSRPath(1, Globals.local_connection, "/") + select = Select(root) + sf = select.other_filesystems_get_sf(0) + assert sf(root) is None + assert sf(RPath(Globals.local_connection, "/usr/bin")) is None, \ + "Assumption: /usr/bin is on the same filesystem as /" + assert sf(RPath(Globals.local_connection, "/proc")) == 0, \ + "Assumption: /proc is on a different filesystem" + assert sf(RPath(Globals.local_connection, "/boot")) == 0, \ + "Assumption: /boot is on a different filesystem" class ParseArgsTest(unittest.TestCase): """Test argument parsing""" diff --git a/rdiff-backup/testing/statisticstest.py b/rdiff-backup/testing/statisticstest.py index 819bb85..cc1f675 100644 --- a/rdiff-backup/testing/statisticstest.py +++ b/rdiff-backup/testing/statisticstest.py @@ -57,6 +57,7 @@ ChangedSourceSize 8 (8 bytes) ChangedMirrorSize 9 (9 bytes) IncrementFiles 15 IncrementFileSize 10 (10 bytes) +TotalDestinationSizeChange 7 (7 bytes) """, "'%s'" % stats_string def test_line_string(self): diff --git a/rdiff-backup/testing/timetest.py b/rdiff-backup/testing/timetest.py index 089ae0c..b6d545f 100644 --- a/rdiff-backup/testing/timetest.py +++ b/rdiff-backup/testing/timetest.py @@ -1,4 +1,4 @@ -import unittest +import unittest, time from commontest import * import Globals, Time @@ -108,5 +108,29 @@ class TimeTest(unittest.TestCase): self.assertRaises(Time.TimeException, g2t, "") self.assertRaises(Time.TimeException, g2t, "3q") + def testSleeping(self): + """Test sleep and sleep ratio""" + sleep_ratio = 0.5 + time1 = time.time() + Time.sleep(0) # set initial time + time.sleep(1) + time2 = time.time() + Time.sleep(sleep_ratio) + time3 = time.time() + time.sleep(0.5) + time4 = time.time() + Time.sleep(sleep_ratio) + time5 = time.time() + + sleep_ratio = 0.25 + time.sleep(0.75) + time6 = time.time() + Time.sleep(sleep_ratio) + time7 = time.time() + + assert 0.9 < time3 - time2 < 1.1, time3 - time2 + assert 0.4 < time5 - time4 < 0.6, time5 - time4 + assert 0.2 < time7 - time6 < 0.3, time7 - time6 + if __name__ == '__main__': unittest.main() -- cgit v1.2.1