diff options
Diffstat (limited to 'rdiff-backup')
21 files changed, 905 insertions, 444 deletions
diff --git a/rdiff-backup/TODO b/rdiff-backup/TODO index 54d091b..33191ee 100644 --- a/rdiff-backup/TODO +++ b/rdiff-backup/TODO @@ -1,4 +1,6 @@ +Use ctime to check whether files have been changed +Include some option to summarize space taken up ---------[ Medium term ]--------------------------------------- diff --git a/rdiff-backup/rdiff-backup.1 b/rdiff-backup/rdiff-backup.1 index b6ecbe4..08469bf 100644 --- a/rdiff-backup/rdiff-backup.1 +++ b/rdiff-backup/rdiff-backup.1 @@ -56,23 +56,14 @@ ability to restore previous versions of that file. .SH OPTIONS .TP .B -b, --backup-mode -Force backup mode even if first argument appears to be an increment file. +Force backup mode even if first argument appears to be an increment or +mirror file. .TP .B --calculate-average Enter calculate average mode. The arguments should be a number of statistics files. rdiff-backup will print the average of the listed statistics files and exit. .TP -.BI "--chars-to-quote " chars -If this option is set, any characters in -.I chars -present in filenames on the source side will be quoted on the -destination side, so that they do not appear in filenames on the -remote side. See -.B --quoting-char -and -.BR --windows-mode . -.TP .B --check-destination-dir If an rdiff-backup session fails, running rdiff-backup with this option on the destination dir will undo the failed directory. This @@ -132,8 +123,7 @@ See the section for more information. .TP .B --exclude-special-files -Exclude all device files, fifos, sockets, and symlinks. This option -is implied by --windows-mode. +Exclude all device files, fifos, sockets, and symlinks. .TP .B --force Authorize the updating or overwriting of a destination path. @@ -202,7 +192,7 @@ In this mode rdiff-backup is similar to rsync (but usually slower). .TP .B --no-compare-inode -This relative esoteric option prevents rdiff-backup from flagging a +This relatively esoteric option prevents rdiff-backup from flagging a file as changed when its inode changes. This option may be useful if you are backing up two different directories to the same rdiff-backup destination directory. The downside is that hard link information may @@ -255,13 +245,6 @@ session statistics file. See the .B STATISTICS section for more information. .TP -.BI "--quoting-char " char -Use the specified character for quoting characters specified to be -escaped by the -.B --chars-to-quote -option. The default is the semicolon ";". See also -.BR --windows-mode . -.TP .BI "-r, --restore-as-of " restore_time Restore the specified directory as it was as of .IR restore_time . @@ -273,10 +256,6 @@ and see the .B RESTORING section for more information on restoring. .TP -.BI "--remote-cmd " command -This command has been depreciated as of version 0.4.1. Use ---remote-schema instead. -.TP .BI "--remote-schema " schema Specify an alternate method of connecting to a remote computer. This is necessary to get rdiff-backup not to use ssh for remote backups, or @@ -354,20 +333,6 @@ is noisiest). This determines how much is written to the log file. .TP .B "-V, --version" Print the current version and exit -.TP -.B --windows-mode -This option quotes characters not allowable on windows, and does not -try to preserve ownership, hardlinks, or permissions on the -destination side. It is appropriate when backing up a normal unix -file system to a windows one such as VFS, or a file system with -similar limitations. Because metadata is stored in a separate regular -file, this option does not prevent all data from being restored. -.TP -.B --windows-restore -This option turns on windows quoting, but does not disable -permissions, hard linking, or ownership. Use this when restoring from -an rdiff-backup directory on a windows file system to a unix file -system. .SH EXAMPLES Simplest case---backup directory foo to directory bar, with increments diff --git a/rdiff-backup/rdiff_backup/FilenameMapping.py b/rdiff-backup/rdiff_backup/FilenameMapping.py index 117d711..066b86a 100644 --- a/rdiff-backup/rdiff_backup/FilenameMapping.py +++ b/rdiff-backup/rdiff_backup/FilenameMapping.py @@ -1,4 +1,4 @@ -# Copyright 2002 Ben Escoto +# Copyright 2002, 2003 Ben Escoto # # This file is part of rdiff-backup. # @@ -29,7 +29,7 @@ handle that error.) """ -import re +import re, types import Globals, log, rpath max_filename_length = 255 @@ -66,12 +66,14 @@ def set_init_quote_vals_local(): def init_quoting_regexps(): """Compile quoting regular expressions""" global chars_to_quote_regexp, unquoting_regexp + assert chars_to_quote and type(chars_to_quote) is types.StringType, \ + "Chars to quote: '%s'" % (chars_to_quote,) try: chars_to_quote_regexp = \ re.compile("[%s]|%s" % (chars_to_quote, quoting_char), re.S) unquoting_regexp = re.compile("%s[0-9]{3}" % quoting_char, re.S) except re.error: - log.Log.FatalError("Error '%s' when processing char quote list %s" % + log.Log.FatalError("Error '%s' when processing char quote list '%s'" % (re.error, chars_to_quote)) def quote(path): @@ -131,8 +133,16 @@ class QuotedRPath(rpath.RPath): def isincfile(self): """Return true if path indicates increment, sets various variables""" - result = rpath.RPath.isincfile(self) - if result: self.inc_basestr = unquote(self.inc_basestr) + if not self.index: # consider the last component as quoted + dirname, basename = self.dirsplit() + temp_rp = rpath.RPath(self.conn, dirname, (unquote(basename),)) + result = temp_rp.isincfile() + if result: + self.inc_basestr = unquote(temp_rp.inc_basestr) + self.inc_timestr = unquote(temp_rp.inc_timestr) + else: + result = rpath.RPath.isincfile(self) + if result: self.inc_basestr = unquote(self.inc_basestr) return result def get_quotedrpath(rp, separate_basename = 0): diff --git a/rdiff-backup/rdiff_backup/Globals.py b/rdiff-backup/rdiff_backup/Globals.py index 0d83d4c..2f60ab7 100644 --- a/rdiff-backup/rdiff_backup/Globals.py +++ b/rdiff-backup/rdiff_backup/Globals.py @@ -66,6 +66,14 @@ change_mirror_perms = (process_uid != 0) # If true, try to reset the atimes of the source partition. preserve_atime = None +# If true, save the extended attributes when backing up. +read_eas = None + +# If true, preserve the extended attributes on the mirror directory +# when backing up, or write them to the restore directory. This +# implies read_eas. +write_eas = None + # This will be set as soon as the LocalConnection class loads local_connection = None @@ -112,10 +120,12 @@ rbdir = None # quoting_enabled is true if we should quote certain characters in # filenames on the source side (see FilenameMapping for more -# info). chars_to_quote is a string whose characters should be -# quoted, and quoting_char is the character to quote with. -quoting_enabled = None -chars_to_quote = "A-Z:" +# info). + +# chars_to_quote is a string whose characters should be quoted. It +# should be true if certain characters in filenames on the source side +# should be escaped (see FilenameMapping for more info). +chars_to_quote = None quoting_char = ';' # If true, emit output intended to be easily readable by a diff --git a/rdiff-backup/rdiff_backup/Main.py b/rdiff-backup/rdiff_backup/Main.py index 5a03fb5..505025d 100644 --- a/rdiff-backup/rdiff_backup/Main.py +++ b/rdiff-backup/rdiff_backup/Main.py @@ -1,4 +1,4 @@ -# Copyright 2002 Ben Escoto +# Copyright 2002, 2003 Ben Escoto # # This file is part of rdiff-backup. # @@ -24,7 +24,7 @@ import getopt, sys, re, os from log import Log, LoggerError, ErrorLog import Globals, Time, SetConnections, selection, robust, rpath, \ manage, backup, connection, restore, FilenameMapping, \ - Security, Hardlink, regress, C + Security, Hardlink, regress, C, fs_abilities action = None @@ -32,6 +32,9 @@ remote_cmd, remote_schema = None, None force = None select_opts = [] select_files = [] +# These are global because they are set while we are trying to figure +# whether to restore or to backup +restore_root, restore_index, restore_root_set = None, None, 0 def parse_cmdlineoptions(arglist): """Parse argument list and set global preferences""" @@ -43,24 +46,24 @@ def parse_cmdlineoptions(arglist): except IOError: Log.FatalError("Error opening file %s" % filename) try: optlist, args = getopt.getopt(arglist, "blr:sv:V", - ["backup-mode", "calculate-average", "chars-to-quote=", - "check-destination-dir", "current-time=", "exclude=", - "exclude-device-files", "exclude-filelist=", - "exclude-filelist-stdin", "exclude-globbing-filelist=", - "exclude-mirror=", "exclude-other-filesystems", - "exclude-regexp=", "exclude-special-files", "force", - "include=", "include-filelist=", "include-filelist-stdin", + ["backup-mode", "calculate-average", "check-destination-dir", + "current-time=", "exclude=", "exclude-device-files", + "exclude-filelist=", "exclude-filelist-stdin", + "exclude-globbing-filelist=", "exclude-mirror=", + "exclude-other-filesystems", "exclude-regexp=", + "exclude-special-files", "force", "include=", + "include-filelist=", "include-filelist-stdin", "include-globbing-filelist=", "include-regexp=", "list-at-time=", "list-changed-since=", "list-increments", "no-compare-inode", "no-compression", "no-compression-regexp=", "no-file-statistics", - "no-hard-links", "null-separator", "parsable-output", - "print-statistics", "quoting-char=", "remote-cmd=", - "remote-schema=", "remove-older-than=", "restore-as-of=", - "restrict=", "restrict-read-only=", "restrict-update-only=", - "server", "ssh-no-compression", "terminal-verbosity=", - "test-server", "verbosity=", "version", "windows-mode", - "windows-restore"]) + "no-hard-links", "null-separator", + "override-chars-to-quote=", "parsable-output", + "print-statistics", "remote-cmd=", "remote-schema=", + "remove-older-than=", "restore-as-of=", "restrict=", + "restrict-read-only=", "restrict-update-only=", "server", + "ssh-no-compression", "terminal-verbosity=", "test-server", + "verbosity=", "version"]) except getopt.error, e: commandline_error("Bad commandline options: %s" % str(e)) @@ -68,9 +71,6 @@ def parse_cmdlineoptions(arglist): if opt == "-b" or opt == "--backup-mode": action = "backup" elif opt == "--calculate-average": action = "calculate-average" elif opt == "--check-destination-dir": action = "check-destination-dir" - elif opt == "--chars-to-quote": - Globals.set('chars_to_quote', arg) - Globals.set('quoting_enabled', 1) elif opt == "--current-time": Globals.set_integer('current_time', arg) elif opt == "--exclude": select_opts.append((opt, arg)) @@ -112,11 +112,10 @@ def parse_cmdlineoptions(arglist): elif opt == "--no-file-statistics": Globals.set('file_statistics', 0) elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0) elif opt == "--null-separator": Globals.set("null_separator", 1) + elif opt == "--override-chars-to-quote": + Globals.set('chars_to_quote', arg) elif opt == "--parsable-output": Globals.set('parsable_output', 1) elif opt == "--print-statistics": Globals.set('print_statistics', 1) - elif opt == "--quoting-char": - Globals.set('quoting_char', arg) - Globals.set('quoting_enabled', 1) elif opt == "-r" or opt == "--restore-as-of": restore_timestr, action = arg, "restore-as-of" elif opt == "--remote-cmd": remote_cmd = arg @@ -142,55 +141,30 @@ def parse_cmdlineoptions(arglist): print "rdiff-backup " + Globals.version sys.exit(0) elif opt == "-v" or opt == "--verbosity": Log.setverbosity(arg) - elif opt == "--windows-mode": - Globals.set('chars_to_quote', "^a-z0-9._ -") - Globals.set('quoting_enabled', 1) - Globals.set('preserve_hardlinks', 0) - Globals.set('change_ownership', 0) - Globals.set('change_permissions', 0) - Globals.set('fsync_directories', 0) - elif opt == '--windows-restore': - Globals.set('chars_to_quote', "^a-z0-9._ -") - Globals.set('quoting_enabled', 1) else: Log.FatalError("Unknown option %s" % opt) -def isincfilename(path): - """Return true if path is of a (possibly quoted) increment file""" - rp = rpath.RPath(Globals.local_connection, path) - if Globals.quoting_enabled: - if not FilenameMapping.quoting_char: - FilenameMapping.set_init_quote_vals() - rp = FilenameMapping.get_quotedrpath(rp, separate_basename = 1) - result = rp.isincfile() - return result - -def set_action(): - """Check arguments and try to set action""" +def check_action(): + """Check to make sure action is compatible with args""" global action + arg_action_dict = {0: ['server'], + 1: ['list-increments', 'remove-older-than', + 'list-at-time', 'list-changed-since', + 'check-destination-dir'], + 2: ['backup', 'restore', 'restore-as-of']} l = len(args) - if not action: + if not action: assert l == 2, args # cannot tell backup or restore yet + elif action == 'calculate-average': if l == 0: commandline_error("No arguments given") - elif l == 1: action = "restore" - elif l == 2: - if isincfilename(args[0]): action = "restore" - else: action = "backup" - else: commandline_error("Too many arguments given") - - if l == 0 and action != "server": - commandline_error("No arguments given") - if l > 0 and action == "server": - commandline_error("Too many arguments given") - if l < 2 and (action == "backup" or action == "restore-as-of"): - commandline_error("Two arguments are required (source, destination).") - if l == 2 and (action == "list-increments" or - action == "remove-older-than" or - action == "list-at-time" or - action == "list-changed-since" or - action == "check-destination-dir"): - commandline_error("Only use one argument, " - "the root of the backup directory") - if l > 2 and action != "calculate-average": - commandline_error("Too many arguments given") + elif l > 2 or action not in arg_action_dict[l]: + commandline_error("Wrong number of arguments given. See man page.") + +def final_set_action(rps): + """If no action set, decide between backup and restore at this point""" + global action + if action: return + assert len(rps) == 2, rps + if restore_get_root(rps[0]): action = "restore" + else: action = "backup" def commandline_error(message): sys.stderr.write("Error: %s\n" % message) @@ -201,7 +175,6 @@ def misc_setup(rps): """Set default change ownership flag, umask, relay regexps""" os.umask(077) Time.setcurtime(Globals.current_time) - FilenameMapping.set_init_quote_vals() SetConnections.UpdateGlobal("client_conn", Globals.local_connection) Globals.postset_regexp('no_compression_regexp', Globals.no_compression_regexp_string) @@ -216,7 +189,7 @@ def take_action(rps): sys.exit(0) elif action == "backup": Backup(rps[0], rps[1]) elif action == "restore": Restore(*rps) - elif action == "restore-as-of": RestoreAsOf(rps[0], rps[1]) + elif action == "restore-as-of": Restore(rps[0], rps[1], 1) elif action == "test-server": SetConnections.TestConnections() elif action == "list-at-time": ListAtTime(rps[0]) elif action == "list-changed-since": ListChangedSince(rps[0]) @@ -236,10 +209,11 @@ def cleanup(): def Main(arglist): """Start everything up!""" parse_cmdlineoptions(arglist) - set_action() + check_action() cmdpairs = SetConnections.get_cmd_pairs(args, remote_schema, remote_cmd) - Security.initialize(action, cmdpairs) + Security.initialize(action or "mirror", cmdpairs) rps = map(SetConnections.cmdpair2rp, cmdpairs) + final_set_action(rps) misc_setup(rps) take_action(rps) cleanup() @@ -247,11 +221,15 @@ def Main(arglist): def Backup(rpin, rpout): """Backup, possibly incrementally, src_path to dest_path.""" - if Globals.quoting_enabled: - rpout = FilenameMapping.get_quotedrpath(rpout) SetConnections.BackupInitConnections(rpin.conn, rpout.conn) + backup_check_dirs(rpin, rpout) + backup_set_fs_globals(rpin, rpout) + if Globals.chars_to_quote: + rpout = FilenameMapping.get_quotedrpath(rpout) + SetConnections.UpdateGlobal( + 'rbdir', FilenameMapping.get_quotedrpath(Globals.rbdir)) + backup_set_rbdir(rpin, rpout) backup_set_select(rpin) - backup_init_dirs(rpin, rpout) if prevtime: rpout.conn.Main.backup_touch_curmirror_local(rpin, rpout) Time.setprevtime(prevtime) @@ -266,32 +244,39 @@ def backup_set_select(rpin): rpin.conn.backup.SourceStruct.set_source_select(rpin, select_opts, *select_files) -def backup_init_dirs(rpin, rpout): - """Make sure rpin and rpout are valid, init data dir and logging""" - global datadir, incdir, prevtime +def backup_check_dirs(rpin, rpout): + """Make sure in and out dirs exist and are directories""" if rpout.lstat() and not rpout.isdir(): if not force: Log.FatalError("Destination %s exists and is not a " "directory" % rpout.path) else: Log("Deleting %s" % rpout.path, 3) rpout.delete() + if not rpout.lstat(): + try: rpout.mkdir() + except os.error: + Log.FatalError("Unable to create directory %s" % rpout.path) if not rpin.lstat(): Log.FatalError("Source directory %s does not exist" % rpin.path) elif not rpin.isdir(): Log.FatalError("Source %s is not a directory" % rpin.path) + backup_warn_if_infinite_regress(rpin, rpout) + Globals.rbdir = rpout.append_path("rdiff-backup-data") - datadir = rpout.append_path("rdiff-backup-data") - SetConnections.UpdateGlobal('rbdir', datadir) +def backup_set_rbdir(rpin, rpout): + """Initialize data dir and logging""" + global incdir, prevtime + SetConnections.UpdateGlobal('rbdir', Globals.rbdir) checkdest_if_necessary(rpout) - incdir = datadir.append_path("increments") + incdir = Globals.rbdir.append_path("increments") prevtime = backup_get_mirrortime() - if rpout.lstat(): - if rpout.isdir() and not rpout.listdir(): # rpout is empty dir - if Globals.change_permissions: - rpout.chmod(0700) # just make sure permissions aren't too lax - elif not datadir.lstat() and not force: Log.FatalError( + assert rpout.lstat(), (rpout.path, rpout.lstat()) + if rpout.isdir() and not rpout.listdir(): # rpout is empty dir + if Globals.change_permissions: + rpout.chmod(0700) # just make sure permissions aren't too lax + elif not Globals.rbdir.lstat() and not force: Log.FatalError( """Destination directory %s @@ -301,17 +286,12 @@ rdiff-backup like this could mess up what is currently in it. If you want to update or overwrite it, run rdiff-backup with the --force option.""" % rpout.path) - if not rpout.lstat(): - try: rpout.mkdir() - except os.error: - Log.FatalError("Unable to create directory %s" % rpout.path) - if not datadir.lstat(): datadir.mkdir() - inc_base = datadir.append_path("increments") + if not Globals.rbdir.lstat(): Globals.rbdir.mkdir() + inc_base = Globals.rbdir.append_path("increments") if not inc_base.lstat(): inc_base.mkdir() if Log.verbosity > 0: - Log.open_logfile(datadir.append("backup.log")) + Log.open_logfile(Globals.rbdir.append("backup.log")) ErrorLog.open(Time.curtimestr, compress = Globals.compression) - backup_warn_if_infinite_regress(rpin, rpout) def backup_warn_if_infinite_regress(rpin, rpout): """Warn user if destination area contained in source area""" @@ -336,6 +316,25 @@ def backup_get_mirrortime(): if mirror_rps: return mirror_rps[0].getinctime() else: return None +def backup_set_fs_globals(rpin, rpout): + """Use fs_abilities to set the globals that depend on filesystem""" + src_fsa = fs_abilities.FSAbilities().init_readonly(rpin) + SetConnections.UpdateGlobal('read_acls', src_fsa.acls) + if src_fsa.eas: rpin.get_ea() + SetConnections.UpdateGlobal('read_eas', src_fsa.eas) + + dest_fsa = fs_abilities.FSAbilities().init_readwrite( + Globals.rbdir, override_chars_to_quote = Globals.chars_to_quote) + SetConnections.UpdateGlobal('preserve_hardlinks', dest_fsa.hardlinks) + SetConnections.UpdateGlobal('fsync_directories', dest_fsa.fsync_dirs) + SetConnections.UpdateGlobal('write_acls', dest_fsa.acls) + SetConnections.UpdateGlobal('write_eas', Globals.read_eas and dest_fsa.eas) + SetConnections.UpdateGlobal('change_ownership', dest_fsa.ownership) + SetConnections.UpdateGlobal('chars_to_quote', dest_fsa.chars_to_quote) + if Globals.chars_to_quote: + for conn in Globals.connections: + conn.FilenameMapping.set_init_quote_vals() + def backup_touch_curmirror_local(rpin, rpout): """Make a file like current_mirror.time.data to record time @@ -367,40 +366,56 @@ def backup_remove_curmirror_local(): older_inc.delete() -def Restore(src_rp, dest_rp = None): +def Restore(src_rp, dest_rp, restore_as_of = None): """Main restoring function - Here src_rp should be an increment file, and if dest_rp is - missing it defaults to the base of the increment. + Here src_rp should be the source file (either an increment or + mirror file), dest_rp should be the target rp to be written. """ - rpin, rpout = restore_check_paths(src_rp, dest_rp) - restore_common(rpin, rpout, rpin.getinctime()) - -def RestoreAsOf(rpin, target): - """Secondary syntax for restore operation - - rpin - RPath of mirror file to restore (not nec. with correct index) - target - RPath of place to put restored file - - """ - rpin, rpout = restore_check_paths(rpin, target, 1) - try: time = Time.genstrtotime(restore_timestr) - except Time.TimeException, exc: Log.FatalError(str(exc)) - restore_common(rpin, target, time) - -def restore_common(rpin, target, time): - """Restore operation common to Restore and RestoreAsOf""" - if target.conn.os.getuid() == 0: - SetConnections.UpdateGlobal('change_ownership', 1) - mirror_root, index = restore_get_root(rpin) - restore_check_backup_dir(mirror_root) - mirror = mirror_root.new_index(index) - inc_rpath = datadir.append_path('increments', index) - restore_set_select(mirror_root, target) - restore_start_log(rpin, target, time) - restore.Restore(mirror, inc_rpath, target, time) - Log("Restore ended", 4) + if not restore_root_set: assert restore_get_root(src_rp) + restore_check_paths(src_rp, dest_rp, restore_as_of) + restore_set_fs_globals(Globals.rbdir) + src_rp = restore_init_quoting(src_rp) + restore_check_backup_dir(restore_root, src_rp, restore_as_of) + if restore_as_of: + try: time = Time.genstrtotime(restore_timestr) + except Time.TimeException, exc: Log.FatalError(str(exc)) + else: time = src_rp.getinctime() + inc_rpath = Globals.rbdir.append_path('increments', restore_index) + restore_set_select(restore_root, dest_rp) + restore_start_log(src_rp, dest_rp, time) + restore.Restore(restore_root.new_index(restore_index), + inc_rpath, dest_rp, time) + Log("Restore finished", 4) + +def restore_init_quoting(src_rp): + """Change rpaths into quoted versions of themselves if necessary""" + global restore_root + if not Globals.chars_to_quote: return src_rp + for conn in Globals.connections: conn.FilenameMapping.set_init_quote_vals() + restore_root = FilenameMapping.get_quotedrpath(restore_root) + SetConnections.UpdateGlobal( + 'rbdir', FilenameMapping.get_quotedrpath(Globals.rbdir)) + return FilenameMapping.get_quotedrpath(src_rp) + +def restore_set_fs_globals(target): + """Use fs_abilities to set the globals that depend on filesystem""" + target_fsa = fs_abilities.FSAbilities().init_readwrite(target, 0) + SetConnections.UpdateGlobal('read_acls', target_fsa.acls) + SetConnections.UpdateGlobal('write_acls', target_fsa.acls) + SetConnections.UpdateGlobal('read_eas', target_fsa.eas) + SetConnections.UpdateGlobal('write_eas', target_fsa.eas) + if Globals.read_eas: target.get_ea() + SetConnections.UpdateGlobal('preserve_hardlinks', target_fsa.hardlinks) + SetConnections.UpdateGlobal('change_ownership', target_fsa.ownership) + + mirror_fsa = fs_abilities.FSAbilities().init_readwrite(Globals.rbdir) + if Globals.chars_to_quote is None: # otherwise already overridden + if mirror_fsa.chars_to_quote: + SetConnections.UpdateGlobal('chars_to_quote', + mirror_fsa.chars_to_quote) + else: SetConnections.UpdateGlobal('chars_to_quote', "") def restore_set_select(mirror_rp, target): """Set the selection iterator on mirror side from command line args @@ -416,7 +431,7 @@ def restore_set_select(mirror_rp, target): def restore_start_log(rpin, target, time): """Open restore log file, log initial message""" - try: Log.open_logfile(datadir.append("restore.log")) + try: Log.open_logfile(Globals.rbdir.append("restore.log")) except LoggerError, e: Log("Warning, " + str(e), 2) # Log following message at file verbosity 3, but term verbosity 4 @@ -430,34 +445,29 @@ def restore_check_paths(rpin, rpout, restoreasof = None): if not restoreasof: if not rpin.lstat(): Log.FatalError("Source file %s does not exist" % rpin.path) - if Globals.quoting_enabled: - rpin = FilenameMapping.get_quotedrpath(rpin, 1) - if not rpin.isincfile(): - Log.FatalError("""File %s does not look like an increment file. - -Try restoring from an increment file (the filenames look like -"foobar.2001-09-01T04:49:04-07:00.diff").""" % rpin.path) - - if not rpout: rpout = rpath.RPath(Globals.local_connection, - rpin.getincbase_str()) - if rpout.lstat() and not force: + if not force and rpout.lstat() and (not rpout.isdir() or rpout.listdir()): Log.FatalError("Restore target %s already exists, " "specify --force to overwrite." % rpout.path) - return rpin, rpout -def restore_check_backup_dir(rpin): +def restore_check_backup_dir(mirror_root, src_rp, restore_as_of): """Make sure backup dir root rpin is in consistent state""" - result = checkdest_need_check(rpin) + if not restore_as_of and not src_rp.isincfile(): + Log.FatalError("""File %s does not look like an increment file. + +Try restoring from an increment file (the filenames look like +"foobar.2001-09-01T04:49:04-07:00.diff").""" % src_rp.path) + + result = checkdest_need_check(mirror_root) if result is None: Log.FatalError("%s does not appear to be an rdiff-backup directory." - % (rpin.path,)) + % (Globals.rbdir.path,)) elif result == 1: Log.FatalError( - "Previous backup to %s seems to have failed." - "Rerun rdiff-backup with --check-destination-dir option to revert" - "directory to state before unsuccessful session." % (rpin.path,)) + "Previous backup to %s seems to have failed.\nRerun rdiff-backup " + "rdiff-with --check-destination-dir option to revert directory " + "to state before unsuccessful session." % (mirror_root.path,)) def restore_get_root(rpin): - """Return (mirror root, index) and set the data dir + """Set data dir, restore_root and index, or return None if fail The idea here is to keep backing up on the path until we find a directory that contains "rdiff-backup-data". That is the @@ -470,7 +480,7 @@ def restore_get_root(rpin): funny way, using symlinks or somesuch. """ - global datadir + global restore_root, restore_index if rpin.isincfile(): relpath = rpin.getincbase().path else: relpath = rpin.path pathcomps = os.path.join(rpin.conn.os.getcwd(), relpath).split("/") @@ -482,23 +492,25 @@ def restore_get_root(rpin): if (parent_dir.isdir() and "rdiff-backup-data" in parent_dir.listdir()): break i = i-1 - else: Log.FatalError("Unable to find rdiff-backup-data directory") - - if not Globals.quoting_enabled: rootrp = parent_dir - else: rootrp = FilenameMapping.get_quotedrpath(parent_dir) - Log("Using mirror root directory %s" % rootrp.path, 6) + else: return None - datadir = rootrp.append_path("rdiff-backup-data") - SetConnections.UpdateGlobal('rbdir', datadir) - if not datadir.isdir(): + restore_root = parent_dir + Log("Using mirror root directory %s" % restore_root.path, 6) + SetConnections.UpdateGlobal('rbdir', + restore_root.append_path("rdiff-backup-data")) + if not Globals.rbdir.isdir(): Log.FatalError("Unable to read rdiff-backup-data directory %s" % - datadir.path) + Globals.rbdir.path) from_datadir = tuple(pathcomps[i:]) if not from_datadir or from_datadir[0] != "rdiff-backup-data": - return (rootrp, from_datadir) # in mirror, not increments - assert from_datadir[1] == "increments" - return (rootrp, from_datadir[2:]) + restore_index = from_datadir # in mirror, not increments + else: + assert (from_datadir[1] == "increments" or + (len(from_datadir) == 2 and + from_datadir[1].startswith('increments'))), from_datadir + restore_index = from_datadir[2:] + return 1 def ListIncrements(rp): @@ -555,7 +567,7 @@ def rom_check_dir(rootrp): rootrp.append_path("rdiff-backup-data")) if not Globals.rbdir.isdir(): Log.FatalError("Unable to open rdiff-backup-data dir %s" % - (datadir.path,)) + (Globals.rbdir.path,)) checkdest_if_necessary(rootrp) @@ -597,6 +609,9 @@ def CheckDest(dest_rp): def checkdest_need_check(dest_rp): """Return None if no dest dir found, 1 if dest dir needs check, 0 o/w""" if not dest_rp.isdir() or not Globals.rbdir.isdir(): return None + if Globals.rbdir.listdir() == ['chars_to_quote']: + # This may happen the first backup just after we test for quoting + return None curmirroot = Globals.rbdir.append("current_mirror") curmir_incs = restore.get_inclist(curmirroot) if not curmir_incs: diff --git a/rdiff-backup/rdiff_backup/Security.py b/rdiff-backup/rdiff_backup/Security.py index 83ddcf2..427f826 100644 --- a/rdiff-backup/rdiff_backup/Security.py +++ b/rdiff-backup/rdiff_backup/Security.py @@ -76,8 +76,9 @@ def set_security_level(action, cmdpairs): rdir = tempfile.gettempdir() elif islocal(cp1): sec_level = "read-only" - rdir = Main.restore_get_root(rpath.RPath(Globals.local_connection, - getpath(cp1)))[0].path + Main.restore_get_root(rpath.RPath(Globals.local_connection, + getpath(cp1))) + rdir = Main.restore_root.path else: assert islocal(cp2) sec_level = "all" diff --git a/rdiff-backup/rdiff_backup/SetConnections.py b/rdiff-backup/rdiff_backup/SetConnections.py index e5d081e..6846953 100644 --- a/rdiff-backup/rdiff_backup/SetConnections.py +++ b/rdiff-backup/rdiff_backup/SetConnections.py @@ -1,4 +1,4 @@ -# Copyright 2002 Ben Escoto +# Copyright 2002, 2003 Ben Escoto # # This file is part of rdiff-backup. # @@ -27,7 +27,7 @@ the related connections. import os from log import Log -import Globals, FilenameMapping, connection, rpath +import Globals, connection, rpath # This is the schema that determines how rdiff-backup will open a # pipe to the remote system. If the file is given as A::B, %s will @@ -178,7 +178,6 @@ def init_connection_settings(conn): conn.log.Log.setterm_verbosity(Log.term_verbosity) for setting_name in Globals.changed_settings: conn.Globals.set(setting_name, Globals.get(setting_name)) - FilenameMapping.set_init_quote_vals() def init_connection_remote(conn_number): """Run on server side to tell self that have given conn_number""" @@ -203,8 +202,6 @@ def BackupInitConnections(reading_conn, writing_conn): writing_conn.Globals.set("isbackup_writer", 1) UpdateGlobal("backup_reader", reading_conn) UpdateGlobal("backup_writer", writing_conn) - if writing_conn.os.getuid() == 0 and Globals.change_ownership != 0: - UpdateGlobal('change_ownership', 1) def CloseConnections(): """Close all connections. Run by client""" diff --git a/rdiff-backup/rdiff_backup/backup.py b/rdiff-backup/rdiff_backup/backup.py index 89d7bea..0be0702 100644 --- a/rdiff-backup/rdiff_backup/backup.py +++ b/rdiff-backup/rdiff_backup/backup.py @@ -1,4 +1,4 @@ -# Copyright 2002 Ben Escoto +# Copyright 2002, 2003 Ben Escoto # # This file is part of rdiff-backup. # @@ -22,7 +22,8 @@ from __future__ import generators import errno import Globals, metadata, rorpiter, TempFile, Hardlink, robust, increment, \ - rpath, static, log, selection, Time, Rdiff, statistics, iterfile + rpath, static, log, selection, Time, Rdiff, statistics, iterfile, \ + eas_acls def Mirror(src_rpath, dest_rpath): """Turn dest_rpath into a copy of src_rpath""" @@ -122,16 +123,27 @@ class DestinationStruct: destination except rdiff-backup-data directory. """ - if use_metadata: - metadata_iter = metadata.GetMetadata_at_time(Globals.rbdir, - Time.prevtime) + def get_basic_iter(): + """Returns iterator of basic metadata""" + metadata_iter = metadata.MetadataFile.get_objects_at_time( + Globals.rbdir, Time.prevtime) if metadata_iter: return metadata_iter log.Log("Warning: Metadata file not found.\n" "Metadata will be read from filesystem.", 2) - sel = selection.Select(rpath) - sel.parse_rbdir_exclude() - return sel.set_iter() + def get_iter_from_fs(): + """Get the combined iterator from the filesystem""" + sel = selection.Select(rpath) + sel.parse_rbdir_exclude() + return sel.set_iter() + + if use_metadata: + if Globals.read_eas: + rorp_iter = eas_acls.ExtendedAttributesFile.\ + get_combined_iter_at_time(Globals.rbdir, Time.prevtime) + else: rorp_iter = get_basic_iter() + if rorp_iter: return rorp_iter + return get_iter_from_fs() def set_rorp_cache(cls, baserp, source_iter, for_increment): """Initialize cls.CCPP, the destination rorp cache @@ -243,7 +255,8 @@ class CacheCollatedPostProcess: self.cache_size = cache_size self.statfileobj = statistics.init_statfileobj() if Globals.file_statistics: statistics.FileStats.init() - metadata.OpenMetadata() + metadata.MetadataFile.open_file() + if Globals.read_eas: eas_acls.ExtendedAttributesFile.open_file() # the following should map indicies to lists # [source_rorp, dest_rorp, changed_flag, success_flag, increment] @@ -317,7 +330,10 @@ class CacheCollatedPostProcess: metadata_rorp = source_rorp else: metadata_rorp = None if metadata_rorp and metadata_rorp.lstat(): - metadata.WriteMetadata(metadata_rorp) + metadata.MetadataFile.write_object(metadata_rorp) + if Globals.read_eas and not metadata_rorp.get_ea().empty(): + eas_acls.ExtendedAttributesFile.write_object( + metadata_rorp.get_ea()) if Globals.file_statistics: statistics.FileStats.update(source_rorp, dest_rorp, changed, inc) @@ -359,7 +375,8 @@ class CacheCollatedPostProcess: def close(self): """Process the remaining elements in the cache""" while self.cache_indicies: self.shorten_cache() - metadata.CloseMetadata() + metadata.MetadataFile.close_file() + if Globals.read_eas: eas_acls.ExtendedAttributesFile.close_file() if Globals.print_statistics: statistics.print_active_stats() if Globals.file_statistics: statistics.FileStats.close() statistics.write_active_statfileobj() diff --git a/rdiff-backup/rdiff_backup/connection.py b/rdiff-backup/rdiff_backup/connection.py index 05aef20..06d7e7a 100644 --- a/rdiff-backup/rdiff_backup/connection.py +++ b/rdiff-backup/rdiff_backup/connection.py @@ -22,6 +22,11 @@ from __future__ import generators import types, os, tempfile, cPickle, shutil, traceback, pickle, \ socket, sys, gzip +# The following EA and ACL modules may be used if available +try: import xattr +except ImportError: pass +try: import posix1e +except ImportError: pass class ConnectionError(Exception): pass @@ -513,7 +518,7 @@ class VirtualFile: import Globals, Time, Rdiff, Hardlink, FilenameMapping, C, Security, \ Main, rorpiter, selection, increment, statistics, manage, lazy, \ iterfile, rpath, robust, restore, manage, backup, connection, \ - TempFile, SetConnections, librsync, log, regress + TempFile, SetConnections, librsync, log, regress, fs_abilities Globals.local_connection = LocalConnection() Globals.connections.append(Globals.local_connection) diff --git a/rdiff-backup/rdiff_backup/eas_acls.py b/rdiff-backup/rdiff_backup/eas_acls.py new file mode 100644 index 0000000..748d8bb --- /dev/null +++ b/rdiff-backup/rdiff_backup/eas_acls.py @@ -0,0 +1,185 @@ +# Copyright 2003 Ben Escoto +# +# This file is part of rdiff-backup. +# +# rdiff-backup is free software; you can redistribute it and/or modify +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# rdiff-backup is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rdiff-backup; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 +# USA + +"""Store and retrieve extended attributes and access control lists + +Not all file systems will have EAs and ACLs, but if they do, store +this information in separate files in the rdiff-backup-data directory, +called extended_attributes.<time>.snapshot and +access_control_lists.<time>.snapshot. + +""" + +from __future__ import generators +import base64, errno, re +import static, Globals, metadata, connection, rorpiter, log + + +class ExtendedAttributes: + """Hold a file's extended attribute information""" + def __init__(self, index, attr_dict = None): + """Initialize EA object with no attributes""" + self.index = index + if attr_dict is None: self.attr_dict = {} + else: self.attr_dict = attr_dict + + def __eq__(self, ea): + """Equal if all attributes and index are equal""" + assert isinstance(ea, ExtendedAttributes) + return ea.index == self.index and ea.attr_dict == self.attr_dict + + def __ne__(self, ea): return not self.__eq__(ea) + + def get_indexpath(self): return self.index and '/'.join(self.index) or '.' + + def read_from_rp(self, rp): + """Set the extended attributes from an rpath""" + try: attr_list = rp.conn.xattr.listxattr(rp.path) + except IOError, exc: + if exc[0] == errno.EOPNOTSUPP: return # if not sup, consider empty + raise + for attr in attr_list: + try: self.attr_dict[attr] = rp.conn.xattr.getxattr(rp.path, attr) + except IOError, exc: + # File probably modified while reading, just continue + if exc[0] == errno.ENODATA: continue + elif exc[0] == errno.ENOENT: break + else: raise + + def clear_rp(self, rp): + """Delete all the extended attributes in rpath""" + for name in rp.conn.xattr.listxattr(rp.path): + rp.conn.xattr.removexattr(rp.path, name) + + def write_to_rp(self, rp): + """Write extended attributes to rpath rp""" + self.clear_rp(rp) + for (name, value) in self.attr_dict.iteritems(): + rp.conn.xattr.setxattr(rp.path, name, value) + + def get(self, name): + """Return attribute attached to given name""" + return self.attr_dict[name] + + def set(self, name, value = ""): + """Set given name to given value. Does not write to disk""" + self.attr_dict[name] = value + + def delete(self, name): + """Delete value associated with given name""" + del self.attr_dict[name] + + def empty(self): + """Return true if no extended attributes are set""" + return not self.attr_dict + +def compare_rps(rp1, rp2): + """Return true if rp1 and rp2 have same extended attributes""" + ea1 = ExtendedAttributes(rp1.index) + ea1.read_from_rp(rp1) + ea2 = ExtendedAttributes(rp2.index) + ea2.read_from_rp(rp2) + return ea1 == ea2 + + +def EA2Record(ea): + """Convert ExtendedAttributes object to text record""" + str_list = ['# file: %s' % ea.get_indexpath()] + for (name, val) in ea.attr_dict.iteritems(): + if not val: str_list.append(name) + else: + encoded_val = base64.encodestring(val).replace('\n', '') + str_list.append('%s=0s%s' % (name, encoded_val)) + return '\n'.join(str_list)+'\n' + +def Record2EA(record): + """Convert text record to ExtendedAttributes object""" + lines = record.split('\n') + first = lines.pop(0) + if not first[:8] == "# file: ": + raise metadata.ParsingError("Bad record beginning: " + first[:8]) + filename = first[8:] + if filename == '.': index = () + else: index = tuple(filename.split('/')) + ea = ExtendedAttributes(index) + + for line in lines: + line = line.strip() + if not line: continue + assert line[0] != '#', line + eq_pos = line.find('=') + if eq_pos == -1: ea.set(line) + else: + name = line[:eq_pos] + assert line[eq_pos+1:eq_pos+3] == '0s', \ + "Currently only base64 encoding supported" + encoded_val = line[eq_pos+3:] + ea.set(name, base64.decodestring(encoded_val)) + return ea + +def quote_path(path): + """Quote a path for use EA/ACL records. + + Right now no quoting!!! Change this to reflect the updated + quoting style of getfattr/setfattr when they are changed. + + """ + return path + + +class EAExtractor(metadata.FlatExtractor): + """Iterate ExtendedAttributes objects from the EA information file""" + record_boundary_regexp = re.compile("\\n# file:") + record_to_object = staticmethod(Record2EA) + def get_index_re(self, index): + """Find start of EA record with given index""" + if not index: indexpath = '.' + else: indexpath = '/'.join(index) + # Right now there is no quoting, due to a bug in + # getfacl/setfacl. Replace later when bug fixed. + return re.compile('(^|\\n)(# file: %s\\n)' % indexpath) + +class ExtendedAttributesFile(metadata.FlatFile): + """Store/retrieve EAs from extended_attributes file""" + _prefix = "extended_attributes" + _extractor = EAExtractor + _object_to_record = staticmethod(EA2Record) + + def get_combined_iter_at_time(cls, rbdir, rest_time, + restrict_index = None): + """Return an iter of rorps with extended attributes added""" + def join_eas(basic_iter, ea_iter): + """Join basic_iter with ea iter""" + collated = rorpiter.CollateIterators(basic_iter, ea_iter) + for rorp, ea in collated: + assert rorp, (rorp, (ea.index, ea.attr_dict), rest_time) + if not ea: ea = ExtendedAttributes(rorp.index) + rorp.set_ea(ea) + yield rorp + + basic_iter = metadata.MetadataFile.get_objects_at_time( + Globals.rbdir, rest_time, restrict_index) + if not basic_iter: return None + ea_iter = cls.get_objects_at_time(rbdir, rest_time, restrict_index) + if not ea_iter: + log.Log("Warning: Extended attributes file not found", 2) + ea_iter = iter([]) + return join_eas(basic_iter, ea_iter) + +static.MakeClass(ExtendedAttributesFile) diff --git a/rdiff-backup/rdiff_backup/fs_abilities.py b/rdiff-backup/rdiff_backup/fs_abilities.py index e6084c6..98c43f8 100644 --- a/rdiff-backup/rdiff_backup/fs_abilities.py +++ b/rdiff-backup/rdiff_backup/fs_abilities.py @@ -1,4 +1,4 @@ -# Copyright 2002 Ben Escoto +# Copyright 2003 Ben Escoto # # This file is part of rdiff-backup. # @@ -50,12 +50,14 @@ class FSAbilities: Only self.acls and self.eas are set. """ + self.root_rp = rp self.read_only = 1 self.set_eas(rp, 0) self.set_acls(rp) return self - def init_readwrite(self, rbdir, use_ctq_file = 1): + def init_readwrite(self, rbdir, use_ctq_file = 1, + override_chars_to_quote = None): """Set variables using fs tested at rp_base This method creates a temp directory in rp_base and writes to @@ -69,17 +71,21 @@ class FSAbilities: file in directory. """ - assert rbdir.isdir() + if not rbdir.isdir(): + assert not rbdir.lstat(), (rbdir.path, rbdir.lstat()) + rbdir.mkdir() + self.root_rp = rbdir self.read_only = 0 - subdir = TempFile.new_in_dir(rbdir) + subdir = rbdir.conn.TempFile.new_in_dir(rbdir) subdir.mkdir() self.set_ownership(subdir) self.set_hardlinks(subdir) self.set_fsync_dirs(subdir) self.set_eas(subdir, 1) self.set_acls(subdir) - self.set_chars_to_quote(subdir) + if override_chars_to_quote is None: self.set_chars_to_quote(subdir) + else: self.chars_to_quote = override_chars_to_quote if use_ctq_file: self.compare_chars_to_quote(rbdir) subdir.delete() return self @@ -95,19 +101,14 @@ class FSAbilities: fp.write(self.chars_to_quote) assert not fp.close() - def get_old_chars(): - fp = ctq_rp.open("rb") - old_chars = fp.read() - assert not fp.close() - return old_chars - if not ctq_rp.lstat(): write_new_chars() else: - old_chars = get_old_chars() + old_chars = ctq_rp.get_data() if old_chars != self.chars_to_quote: if self.chars_to_quote == "": log.Log("Warning: File system no longer needs quoting, " "but will retain for backwards compatibility.", 2) + self.chars_to_quote = old_chars else: log.FatalError("""New quoting requirements This may be caused when you copy an rdiff-backup directory from a @@ -127,7 +128,7 @@ rdiff-backup-data/chars_to_quote. except (IOError, OSError), exc: if exc[0] == errno.EPERM: log.Log("Warning: ownership cannot be changed on filesystem " - "at device %s" % (testdir.getdevloc(),), 2) + "at %s" % (self.root_rp.path,), 2) self.ownership = 0 else: raise else: self.ownership = 1 @@ -143,21 +144,15 @@ rdiff-backup-data/chars_to_quote. assert hl_source.getinode() == hl_dest.getinode() except (IOError, OSError), exc: if exc[0] in (errno.EOPNOTSUPP, errno.EPERM): - log.Log("Warning: hard linking not supported by filesystem %s" - % (testdir.getdevloc(),), 2) + log.Log("Warning: hard linking not supported by filesystem " + "at %s" % (self.root_rp.path,), 2) self.hardlinks = 0 else: raise else: self.hardlinks = 1 def set_fsync_dirs(self, testdir): """Set self.fsync_dirs if directories can be fsync'd""" - try: testdir.fsync() - except (IOError, OSError), exc: - log.Log("Warning: Directories on file system at %s are not " - "fsyncable.\nAssuming it's unnecessary." % - (testdir.getdevloc(),), 2) - self.fsync_dirs = 0 - else: self.fsync_dirs = 1 + self.fsync_dirs = testdir.conn.fs_abilities.test_fsync_local(testdir) def set_chars_to_quote(self, subdir): """Set self.chars_to_quote by trying to write various paths""" @@ -189,7 +184,7 @@ rdiff-backup-data/chars_to_quote. def sanity_check(): """Make sure basic filenames writable""" - for filename in ['5-_ a']: + for filename in ['5-_ a.']: rp = subdir.append(filename) rp.touch() assert rp.lstat() @@ -198,57 +193,70 @@ rdiff-backup-data/chars_to_quote. sanity_check() if is_case_sensitive(): if supports_unusual_chars(): self.chars_to_quote = "" - else: self.chars_to_quote = "^A-Za-z0-9_ -" + else: self.chars_to_quote = "^A-Za-z0-9_ -." else: if supports_unusual_chars(): self.chars_to_quote = "A-Z;" - else: self.chars_to_quote = "^a-z0-9_ -" + else: self.chars_to_quote = "^a-z0-9_ -." def set_acls(self, rp): """Set self.acls based on rp. Does not write. Needs to be local""" - assert Globals.local_connection is rp.conn - assert rp.lstat() - try: import posix1e - except ImportError: - log.Log("Warning: Unable to import module posix1e from pylibacl " - "package.\nACLs not supported on device %s" % - (rp.getdevloc(),), 2) - self.acls = 0 - return - - try: posix1e.ACL(file=rp.path) - except IOError, exc: - if exc[0] == errno.EOPNOTSUPP: - log.Log("Warning: ACLs appear not to be supported by " - "filesystem on device %s" % (rp.getdevloc(),), 2) - self.acls = 0 - else: raise - else: self.acls = 1 + self.acls = rp.conn.fs_abilities.test_acls_local(rp) def set_eas(self, rp, write): - """Set extended attributes from rp. Run locally. - - Tests writing if write is true. - - """ - assert Globals.local_connection is rp.conn - assert rp.lstat() - try: import xattr - except ImportError: - log.Log("Warning: Unable to import module xattr. ACLs not " - "supported on device %s" % (rp.getdevloc(),), 2) - self.eas = 0 - return - - try: - xattr.listxattr(rp.path) - if write: - xattr.setxattr(rp.path, "user.test", "test val") - assert xattr.getxattr(rp.path, "user.test") == "test val" - except IOError, exc: - if exc[0] == errno.EOPNOTSUPP: - log.Log("Warning: Extended attributes not supported by " - "filesystem on device %s" % (rp.getdevloc(),), 2) - self.eas = 0 - else: raise - else: self.eas = 1 + """Set extended attributes from rp. Tests writing if write is true.""" + self.eas = rp.conn.fs_abilities.test_eas_local(rp, write) + + +def test_eas_local(rp, write): + """Test ea support. Must be called locally. Usedy by set_eas above.""" + assert Globals.local_connection is rp.conn + assert rp.lstat() + try: import xattr + except ImportError: + log.Log("Warning: Unable to import module xattr. ACLs not " + "supported on filesystem at %s" % (rp.path,), 2) + return 0 + + try: + xattr.listxattr(rp.path) + if write: + xattr.setxattr(rp.path, "user.test", "test val") + assert xattr.getxattr(rp.path, "user.test") == "test val" + except IOError, exc: + if exc[0] == errno.EOPNOTSUPP: + log.Log("Warning: Extended attributes not supported by " + "filesystem at %s" % (rp.path,), 2) + return 0 + else: raise + else: return 1 + +def test_acls_local(rp): + """Test acl support. Call locally. Does not write.""" + assert Globals.local_connection is rp.conn + assert rp.lstat() + try: import posix1e + except ImportError: + log.Log("Warning: Unable to import module posix1e from pylibacl " + "package.\nACLs not supported on filesystem at %s" % + (rp.path,), 2) + return 0 + + try: posix1e.ACL(file=rp.path) + except IOError, exc: + if exc[0] == errno.EOPNOTSUPP: + log.Log("Warning: ACLs appear not to be supported by " + "filesystem at %s" % (rp.path,), 2) + return 0 + else: raise + else: return 1 + +def test_fsync_local(rp): + """Test fsyncing directories locally""" + assert rp.conn is Globals.local_connection + try: rp.fsync() + except (IOError, OSError), exc: + log.Log("Warning: Directories on file system at %s are not " + "fsyncable.\nAssuming it's unnecessary." % (rp.path,), 2) + return 0 + else: return 1 diff --git a/rdiff-backup/rdiff_backup/metadata.py b/rdiff-backup/rdiff_backup/metadata.py index f1f23ac..3d8ba60 100644 --- a/rdiff-backup/rdiff_backup/metadata.py +++ b/rdiff-backup/rdiff_backup/metadata.py @@ -56,7 +56,7 @@ field names and values. from __future__ import generators import re, gzip, os -import log, Globals, rpath, Time, robust, increment +import log, Globals, rpath, Time, robust, increment, static class ParsingError(Exception): """This is raised when bad or unparsable data is received""" @@ -165,16 +165,14 @@ def unquote_path(quoted_string): return re.sub("\\\\n|\\\\\\\\", replacement_func, quoted_string) -def write_rorp_iter_to_file(rorp_iter, file): - """Given iterator of RORPs, write records to (pre-opened) file object""" - for rorp in rorp_iter: file.write(RORP2Record(rorp)) - -class rorp_extractor: - """Controls iterating rorps from metadata file""" +class FlatExtractor: + """Controls iterating objects from flat file""" + # The following two should be set in subclasses + record_boundary_regexp = None # Matches beginning of next record + record_to_object = None # Function that converts text record to object def __init__(self, fileobj): self.fileobj = fileobj # holds file object we are reading from self.buf = "" # holds the next part of the file - self.record_boundary_regexp = re.compile("\\nFile") self.at_end = 0 # True if we are at the end of the file self.blocksize = 32 * 1024 @@ -191,12 +189,13 @@ class rorp_extractor: else: self.buf += newbuf def iterate(self): - """Return iterator over all records""" + """Return iterator that yields all objects with records""" while 1: next_pos = self.get_next_pos() - try: yield Record2RORP(self.buf[:next_pos]) + try: yield self.record_to_object(self.buf[:next_pos]) except ParsingError, e: - log.Log("Error parsing metadata file: %s" % (e,), 2) + if self.at_end: break # Ignore whitespace/bad records at end + log.Log("Error parsing flat file: %s" % (e,), 2) if self.at_end: break self.buf = self.buf[next_pos:] assert not self.close() @@ -209,15 +208,7 @@ class rorp_extractor: """ assert not self.buf or self.buf.endswith("\n") - if not index: indexpath = "." - else: indexpath = "/".join(index) - # Must double all backslashes, because they will be - # reinterpreted. For instance, to search for index \n - # (newline), it will be \\n (backslash n) in the file, so the - # regular expression is "File \\\\n\\n" (File two backslash n - # backslash n) - double_quote = re.sub("\\\\", "\\\\\\\\", indexpath) - begin_re = re.compile("(^|\\n)(File %s\\n)" % (double_quote,)) + begin_re = self.get_index_re(index) while 1: m = begin_re.search(self.buf) if m: @@ -229,18 +220,28 @@ class rorp_extractor: self.at_end = 1 return + def get_index_re(self, index): + """Return regular expression used to find index. + + Override this in sub classes. The regular expression's second + group needs to start at the beginning of the record that + contains information about the object with the given index. + + """ + assert 0, "Just a placeholder, must override this in subclasses" + def iterate_starting_with(self, index): - """Iterate records whose index starts with given index""" + """Iterate objects whose index starts with given index""" self.skip_to_index(index) if self.at_end: return while 1: next_pos = self.get_next_pos() - try: rorp = Record2RORP(self.buf[:next_pos]) + try: obj = self.record_to_object(self.buf[:next_pos]) except ParsingError, e: log.Log("Error parsing metadata file: %s" % (e,), 2) else: - if rorp.index[:len(index)] != index: break - yield rorp + if obj.index[:len(index)] != index: break + yield obj if self.at_end: break self.buf = self.buf[next_pos:] assert not self.close() @@ -249,73 +250,116 @@ class rorp_extractor: """Return value of closing associated file""" return self.fileobj.close() +class RorpExtractor(FlatExtractor): + """Iterate rorps from metadata file""" + record_boundary_regexp = re.compile("\\nFile") + record_to_object = staticmethod(Record2RORP) + def get_index_re(self, index): + """Find start of rorp record with given index""" + indexpath = index and '/'.join(index) or '.' + # Must double all backslashes, because they will be + # reinterpreted. For instance, to search for index \n + # (newline), it will be \\n (backslash n) in the file, so the + # regular expression is "File \\\\n\\n" (File two backslash n + # backslash n) + double_quote = re.sub("\\\\", "\\\\\\\\", indexpath) + return re.compile("(^|\\n)(File %s\\n)" % (double_quote,)) -metadata_rp = None -metadata_fileobj = None -metadata_record_buffer = [] # Use this because gzip writes are slow -def OpenMetadata(rp = None, compress = 1): - """Open the Metadata file for writing, return metadata fileobj""" - global metadata_rp, metadata_fileobj - assert not metadata_fileobj, "Metadata file already open" - if rp: metadata_rp = rp - else: - if compress: typestr = 'snapshot.gz' - else: typestr = 'snapshot' - metadata_rp = Globals.rbdir.append("mirror_metadata.%s.%s" % - (Time.curtimestr, typestr)) - metadata_fileobj = metadata_rp.open("wb", compress = compress) - -def WriteMetadata(rorp): - """Write metadata of rorp to file""" - global metadata_fileobj, metadata_record_buffer - metadata_record_buffer.append(RORP2Record(rorp)) - if len(metadata_record_buffer) >= 100: write_metadata_buffer() - -def write_metadata_buffer(): - global metadata_record_buffer - metadata_fileobj.write("".join(metadata_record_buffer)) - metadata_record_buffer = [] - -def CloseMetadata(): - """Close the metadata file""" - global metadata_rp, metadata_fileobj - assert metadata_fileobj, "Metadata file not open" - if metadata_record_buffer: write_metadata_buffer() - try: fileno = metadata_fileobj.fileno() # will not work if GzipFile - except AttributeError: fileno = metadata_fileobj.fileobj.fileno() - os.fsync(fileno) - result = metadata_fileobj.close() - metadata_fileobj = None - metadata_rp.setdata() - return result - -def GetMetadata(rp, restrict_index = None, compressed = None): - """Return iterator of metadata from given metadata file rp""" - if compressed is None: - if rp.isincfile(): - compressed = rp.inc_compressed - assert rp.inc_type == "data" or rp.inc_type == "snapshot" - else: compressed = rp.get_indexpath().endswith(".gz") - - fileobj = rp.open("rb", compress = compressed) - if restrict_index is None: return rorp_extractor(fileobj).iterate() - else: return rorp_extractor(fileobj).iterate_starting_with(restrict_index) - -def GetMetadata_at_time(rbdir, time, restrict_index = None, rblist = None): - """Scan through rbdir, finding metadata file at given time, iterate - - If rdlist is given, use that instead of listing rddir. Time here - is exact, we don't take the next one older or anything. Returns - None if no matching metadata found. + +class FlatFile: + """Manage a flat (probably text) file containing info on various files + + This is used for metadata information, and possibly EAs and ACLs. + The main read interface is as an iterator. The storage format is + a flat, probably compressed file, so random access is not + recommended. """ - if rblist is None: rblist = map(lambda x: rbdir.append(x), - robust.listrp(rbdir)) - for rp in rblist: - if (rp.isincfile() and - (rp.getinctype() == "data" or rp.getinctype() == "snapshot") and - rp.getincbase_str() == "mirror_metadata"): - if rp.getinctime() == time: return GetMetadata(rp, restrict_index) - return None + _prefix = None # Set this to real prefix when subclassing + _rp, _fileobj = None, None + # Buffering may be useful because gzip writes are slow + _buffering_on = 1 + _record_buffer, _max_buffer_size = None, 100 + _extractor = FlatExtractor # Set to class that iterates objects + + def open_file(cls, rp = None, compress = 1): + """Open file for writing. Use cls._rp if rp not given.""" + assert not cls._fileobj, "Flatfile already open" + cls._record_buffer = [] + if rp: cls._rp = rp + else: + if compress: typestr = 'snapshot.gz' + else: typestr = 'snapshot' + cls._rp = Globals.rbdir.append( + "%s.%s.%s" % (cls._prefix, Time.curtimestr, typestr)) + cls._fileobj = cls._rp.open("wb", compress = compress) + + def write_object(cls, object): + """Convert one object to record and write to file""" + record = cls._object_to_record(object) + if cls._buffering_on: + cls._record_buffer.append(record) + if len(cls._record_buffer) >= cls._max_buffer_size: + cls._fileobj.write("".join(cls._record_buffer)) + cls._record_buffer = [] + else: cls._fileobj.write(record) + + def close_file(cls): + """Close file, for when any writing is done""" + assert cls._fileobj, "File already closed" + if cls._buffering_on and cls._record_buffer: + cls._fileobj.write("".join(cls._record_buffer)) + cls._record_buffer = [] + try: fileno = cls._fileobj.fileno() # will not work if GzipFile + except AttributeError: fileno = cls._fileobj.fileobj.fileno() + os.fsync(fileno) + result = cls._fileobj.close() + cls._fileobj = None + cls._rp.setdata() + return result + + def get_objects(cls, restrict_index = None, compressed = None): + """Return iterator of objects records from file rp""" + assert cls._rp, "Must have rp set before get_objects can be used" + if compressed is None: + if cls._rp.isincfile(): + compressed = cls._rp.inc_compressed + assert (cls._rp.inc_type == 'data' or + cls._rp.inc_type == 'snapshot'), cls._rp.inc_type + else: compressed = cls._rp.get_indexpath().endswith('.gz') + + fileobj = cls._rp.open('rb', compress = compressed) + if restrict_index is None: return cls._extractor(fileobj).iterate() + else: + re = cls._extractor(fileobj) + return re.iterate_starting_with(restrict_index) + + def get_objects_at_time(cls, rbdir, time, restrict_index = None, + rblist = None): + """Scan through rbdir, finding data at given time, iterate + + If rblist is givenr, use that instead of listing rbdir. Time + here is exact, we don't take the next one older or anything. + Returns None if no file matching prefix is found. + """ + if rblist is None: + rblist = map(lambda x: rbdir.append(x), robust.listrp(rbdir)) + + for rp in rblist: + if (rp.isincfile() and + (rp.getinctype() == "data" or rp.getinctype() == "snapshot") + and rp.getincbase_str() == cls._prefix): + if rp.getinctime() == time: + cls._rp = rp + return cls.get_objects(restrict_index) + return None + +static.MakeClass(FlatFile) + +class MetadataFile(FlatFile): + """Store/retrieve metadata from mirror_metadata as rorps""" + _prefix = "mirror_metadata" + _extractor = RorpExtractor + _object_to_record = staticmethod(RORP2Record) diff --git a/rdiff-backup/rdiff_backup/regress.py b/rdiff-backup/rdiff_backup/regress.py index 4246431..35fb94e 100644 --- a/rdiff-backup/rdiff_backup/regress.py +++ b/rdiff-backup/rdiff_backup/regress.py @@ -124,7 +124,8 @@ def iterate_raw_rfs(mirror_rp, inc_rp): def yield_metadata(): """Iterate rorps from metadata file, if any are available""" - metadata_iter = metadata.GetMetadata_at_time(Globals.rbdir, regress_time) + metadata_iter = metadata.MetadataFile.get_objects_at_time(Globals.rbdir, + regress_time) if metadata_iter: return metadata_iter log.Log.FatalError("No metadata for time %s found, cannot regress" % Time.timetopretty(regress_time)) diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py index 8c0b253..9ae5df8 100644 --- a/rdiff-backup/rdiff_backup/restore.py +++ b/rdiff-backup/rdiff_backup/restore.py @@ -1,4 +1,4 @@ -# Copyright 2002 Ben Escoto +# Copyright 2002, 2003 Ben Escoto # # This file is part of rdiff-backup. # @@ -22,7 +22,7 @@ from __future__ import generators import tempfile, os, cStringIO import Globals, Time, Rdiff, Hardlink, rorpiter, selection, rpath, \ - log, static, robust, metadata, statistics, TempFile + log, static, robust, metadata, statistics, TempFile, eas_acls # This should be set to selection.Select objects over the source and @@ -154,8 +154,13 @@ class MirrorStruct: """ if rest_time is None: rest_time = _rest_time - metadata_iter = metadata.GetMetadata_at_time(Globals.rbdir, - rest_time, restrict_index = cls.mirror_base.index) + if Globals.write_eas: + metadata_iter = eas_acls.ExtendedAttributesFile.\ + get_combined_iter_at_time( + Globals.rbdir, rest_time, restrict_index = cls.mirror_base.index) + else: + metadata_iter = metadata.MetadataFile.get_objects_at_time( + Globals.rbdir, rest_time, restrict_index = cls.mirror_base.index) if metadata_iter: rorp_iter = metadata_iter elif require_metadata: log.Log.FatalError("Mirror metadata not found") else: diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py index 3fe54f5..51f49c6 100644 --- a/rdiff-backup/rdiff_backup/rpath.py +++ b/rdiff-backup/rdiff_backup/rpath.py @@ -1,4 +1,4 @@ -# Copyright 2002 Ben Escoto +# Copyright 2002, 2003 Ben Escoto # # This file is part of rdiff-backup. # @@ -156,6 +156,7 @@ def copy_attribs(rpin, rpout): if Globals.change_ownership: apply(rpout.chown, rpin.getuidgid()) if Globals.change_permissions: rpout.chmod(rpin.getperms()) if not rpin.isdev(): rpout.setmtime(rpin.getmtime()) + if Globals.write_eas: rpout.write_ea(rpin.get_ea()) def cmp_attribs(rp1, rp2): """True if rp1 has the same file attributes as rp2 @@ -301,7 +302,8 @@ class RORPath: return 1 def equal_verbose(self, other, check_index = 1, - compare_inodes = 0, compare_ownership = 0): + compare_inodes = 0, compare_ownership = 0, + compare_eas = 0): """Like __eq__, but log more information. Useful when testing""" if check_index and self.index != other.index: log.Log("Index %s != index %s" % (self.index, other.index), 2) @@ -318,6 +320,7 @@ class RORPath: elif key == 'size' and not self.isreg(): pass elif key == 'inode' and (not self.isreg() or not compare_inodes): pass + elif key == 'ea' and not compare_eas: pass elif (not other.data.has_key(key) or self.data[key] != other.data[key]): if not other.data.has_key(key): @@ -512,6 +515,14 @@ class RORPath: self.index) self.file_already_open = None + def set_ea(self, ea): + """Record extended attributes in dictionary. Does not write""" + self.data['ea'] = ea + + def get_ea(self): + """Return extended attributes object""" + return self.data['ea'] + class RPath(RORPath): """Remote Path class - wrapper around a possibly non-local pathname @@ -546,7 +557,7 @@ class RPath(RORPath): else: self.path = "/".join((base,) + index) self.file = None if data or base is None: self.data = data - else: self.data = self.conn.C.make_file_dict(self.path) + else: self.setdata() def __str__(self): return "Path: %s\nIndex: %s\nData: %s" % (self.path, self.index, @@ -571,6 +582,7 @@ class RPath(RORPath): def setdata(self): """Set data dictionary using C extension""" self.data = self.conn.C.make_file_dict(self.path) + if Globals.read_eas and self.lstat(): self.get_ea() def make_file_dict_old(self): """Create the data dictionary""" @@ -727,7 +739,7 @@ class RPath(RORPath): log.Log("Deleting %s" % self.path, 7) if self.isdir(): try: self.rmdir() - except os.error: shutil.rmtree(self.path) + except os.error: self.conn.shutil.rmtree(self.path) else: self.conn.os.unlink(self.path) self.setdata() @@ -929,6 +941,24 @@ class RPath(RORPath): assert not fp.close() return s + def get_ea(self): + """Return extended attributes object, setting if necessary""" + try: ea = self.data['ea'] + except KeyError: + ea = eas_acls.ExtendedAttributes(self.index) + if not self.issym(): + # Don't read from symlinks because they will be + # followed. Update this when llistxattr, + # etc. available + ea.read_from_rp(self) + self.data['ea'] = ea + return ea + + def write_ea(self, ea): + """Change extended attributes of rp""" + ea.write_to_rp(self) + self.data['ea'] = ea + class RPathFileHook: """Look like a file, but add closing hook""" @@ -945,3 +975,4 @@ class RPathFileHook: self.closing_thunk() return result +import eas_acls # Put at end to avoid regress diff --git a/rdiff-backup/testing/commontest.py b/rdiff-backup/testing/commontest.py index 8bd0f53..4cda300 100644 --- a/rdiff-backup/testing/commontest.py +++ b/rdiff-backup/testing/commontest.py @@ -3,7 +3,7 @@ import os, sys from rdiff_backup.log import Log from rdiff_backup.rpath import RPath from rdiff_backup import Globals, Hardlink, SetConnections, Main, \ - selection, lazy, Time, rpath + selection, lazy, Time, rpath, eas_acls RBBin = "../rdiff-backup" SourceDir = "../rdiff_backup" @@ -143,7 +143,7 @@ def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time): if inc: Main.Restore(get_increment_rp(mirror_rp, time), dest_rp) else: # use alternate syntax Main.restore_timestr = str(time) - Main.RestoreAsOf(mirror_rp, dest_rp) + Main.Restore(mirror_rp, dest_rp, restore_as_of = 1) Main.cleanup() def get_increment_rp(mirror_rp, time): @@ -166,7 +166,8 @@ def _reset_connections(src_rp, dest_rp): def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1, equality_func = None, exclude_rbdir = 1, - ignore_tmp_files = None, compare_ownership = 0): + ignore_tmp_files = None, compare_ownership = 0, + compare_eas = 0): """Compare src_rp and dest_rp, which can be directories This only compares file attributes, not the actual data. This @@ -178,8 +179,8 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1, src_rp.setdata() dest_rp.setdata() - Log("Comparing %s and %s, hardlinks %s" % (src_rp.path, dest_rp.path, - compare_hardlinks), 3) + Log("Comparing %s and %s, hardlinks %s, eas %s" % + (src_rp.path, dest_rp.path, compare_hardlinks, compare_eas), 3) src_select = selection.Select(src_rp) dest_select = selection.Select(dest_rp) @@ -214,11 +215,17 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1, if not src_rorp.equal_verbose(dest_rorp, compare_ownership = compare_ownership): return None - if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1 - Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3) - Log("%s: %s" % (dest_rorp.index, - Hardlink.get_indicies(dest_rorp, None)), 3) - return None + if not Hardlink.rorp_eq(src_rorp, dest_rorp): + Log("%s: %s" % (src_rorp.index, + Hardlink.get_indicies(src_rorp, 1)), 3) + Log("%s: %s" % (dest_rorp.index, + Hardlink.get_indicies(dest_rorp, None)), 3) + return None + if compare_eas and not eas_acls.compare_rps(src_rorp, dest_rorp): + Log("Different EAs in files %s and %s" % + (src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3) + return None + return 1 def rbdir_equal(src_rorp, dest_rorp): """Like hardlink_equal, but make allowances for data directories""" @@ -233,6 +240,10 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1, if dest_rorp.index[-1].endswith('gz'): return 1 # Don't compare .missing increments because they don't matter if dest_rorp.index[-1].endswith('.missing'): return 1 + if compare_eas and not eas_acls.compare_rps(src_rorp, dest_rorp): + Log("Different EAs in files %s and %s" % + (src_rorp.get_indexpath(), dest_rorp.get_indexpath())) + return None if compare_hardlinks: if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1 elif src_rorp.equal_verbose(dest_rorp, @@ -272,7 +283,8 @@ def BackupRestoreSeries(source_local, dest_local, list_of_dirnames, compare_hardlinks = 1, dest_dirname = "testfiles/output", restore_dirname = "testfiles/rest_out", - compare_backups = 1): + compare_backups = 1, + compare_eas = 0): """Test backing up/restoring of a series of directories The dirnames correspond to a single directory at different times. @@ -282,6 +294,8 @@ def BackupRestoreSeries(source_local, dest_local, list_of_dirnames, """ Globals.set('preserve_hardlinks', compare_hardlinks) + Globals.set('write_eas', compare_eas) + Globals.set('read_eas', compare_eas) time = 10000 dest_rp = rpath.RPath(Globals.local_connection, dest_dirname) restore_rp = rpath.RPath(Globals.local_connection, restore_dirname) @@ -296,7 +310,8 @@ def BackupRestoreSeries(source_local, dest_local, list_of_dirnames, time += 10000 _reset_connections(src_rp, dest_rp) if compare_backups: - assert CompareRecursive(src_rp, dest_rp, compare_hardlinks) + assert CompareRecursive(src_rp, dest_rp, compare_hardlinks, + compare_eas = compare_eas) time = 10000 for dirname in list_of_dirnames[:-1]: @@ -305,7 +320,7 @@ def BackupRestoreSeries(source_local, dest_local, list_of_dirnames, InternalRestore(dest_local, source_local, dest_dirname, restore_dirname, time) src_rp = rpath.RPath(Globals.local_connection, dirname) - assert CompareRecursive(src_rp, restore_rp) + assert CompareRecursive(src_rp, restore_rp, compare_eas = compare_eas) # Restore should default back to newest time older than it # with a backup then. diff --git a/rdiff-backup/testing/eas_aclstest.py b/rdiff-backup/testing/eas_aclstest.py new file mode 100644 index 0000000..a9a880b --- /dev/null +++ b/rdiff-backup/testing/eas_aclstest.py @@ -0,0 +1,135 @@ +import unittest, os, time +from commontest import * +from rdiff_backup.eas_acls import * +from rdiff_backup import Globals, rpath, Time + +tempdir = rpath.RPath(Globals.local_connection, "testfiles/output") + +class EATest(unittest.TestCase): + """Test extended attributes""" + sample_ea = ExtendedAttributes( + (), {'user.empty':'', 'user.not_empty':'foobar', 'user.third':'hello', + 'user.binary':chr(0)+chr(1)+chr(2)+chr(140)+'/="', + 'user.multiline':"""This is a fairly long extended attribute. + Encoding it will require several lines of + base64.""" + chr(177)*300}) + empty_ea = ExtendedAttributes(()) + ea1 = ExtendedAttributes(('1',), sample_ea.attr_dict.copy()) + ea1.delete('user.not_empty') + ea2 = ExtendedAttributes(('2',), sample_ea.attr_dict.copy()) + ea2.set('user.third', 'Another random attribute') + ea3 = ExtendedAttributes(('3',)) + ea4 = ExtendedAttributes(('4',), {'user.deleted': 'File to be deleted'}) + ea_testdir1 = rpath.RPath(Globals.local_connection, "testfiles/ea_test1") + ea_testdir2 = rpath.RPath(Globals.local_connection, "testfiles/ea_test2") + + def make_temp(self): + """Make temp directory testfiles/output""" + if tempdir.lstat(): tempdir.delete() + tempdir.mkdir() + + def testBasic(self): + """Test basic writing and reading of extended attributes""" + self.make_temp() + new_ea = ExtendedAttributes(()) + new_ea.read_from_rp(tempdir) + assert not new_ea.attr_dict + assert not new_ea == self.sample_ea + assert new_ea != self.sample_ea + assert new_ea == self.empty_ea + + self.sample_ea.write_to_rp(tempdir) + new_ea.read_from_rp(tempdir) + assert new_ea.attr_dict == self.sample_ea.attr_dict, \ + (new_ea.attr_dict, self.sample_ea.attr_dict) + assert new_ea == self.sample_ea + + def testRecord(self): + """Test writing a record and reading it back""" + record = EA2Record(self.sample_ea) + new_ea = Record2EA(record) + if not new_ea == self.sample_ea: + new_list = new_ea.attr_dict.keys() + sample_list = self.sample_ea.attr_dict.keys() + new_list.sort() + sample_list.sort() + assert new_list == sample_list, (new_list, sample_list) + for name in new_list: + assert self.sample_ea.get(name) == new_ea.get(name), \ + (self.sample_ea.get(name), new_ea.get(name)) + assert self.sample_ea.index == new_ea.index, \ + (self.sample_ea.index, new_ea.index) + assert 0, "We shouldn't have gotten this far" + + def make_backup_dirs(self): + """Create testfiles/ea_test[12] directories""" + if self.ea_testdir1.lstat(): self.ea_testdir1.delete() + if self.ea_testdir2.lstat(): self.ea_testdir2.delete() + self.ea_testdir1.mkdir() + rp1_1 = self.ea_testdir1.append('1') + rp1_2 = self.ea_testdir1.append('2') + rp1_3 = self.ea_testdir1.append('3') + rp1_4 = self.ea_testdir1.append('4') + map(rpath.RPath.touch, [rp1_1, rp1_2, rp1_3, rp1_4]) + self.sample_ea.write_to_rp(self.ea_testdir1) + self.ea1.write_to_rp(rp1_1) + self.ea2.write_to_rp(rp1_2) + self.ea4.write_to_rp(rp1_4) + + self.ea_testdir2.mkdir() + rp2_1 = self.ea_testdir2.append('1') + rp2_2 = self.ea_testdir2.append('2') + rp2_3 = self.ea_testdir2.append('3') + map(rpath.RPath.touch, [rp2_1, rp2_2, rp2_3]) + self.ea3.write_to_rp(self.ea_testdir2) + self.sample_ea.write_to_rp(rp2_1) + self.ea1.write_to_rp(rp2_2) + self.ea2.write_to_rp(rp2_3) + + def testIterate(self): + """Test writing several records and then reading them back""" + self.make_backup_dirs() + rp1 = self.ea_testdir1.append('1') + rp2 = self.ea_testdir1.append('2') + rp3 = self.ea_testdir1.append('3') + + # Now write records corresponding to above rps into file + Globals.rbdir = tempdir + Time.setcurtime(10000) + ExtendedAttributesFile.open_file() + for rp in [self.ea_testdir1, rp1, rp2, rp3]: + ea = ExtendedAttributes(rp.index) + ea.read_from_rp(rp) + ExtendedAttributesFile.write_object(ea) + ExtendedAttributesFile.close_file() + + # Read back records and compare + ea_iter = ExtendedAttributesFile.get_objects_at_time(tempdir, 10000) + assert ea_iter, "No extended_attributes.<time> file found" + sample_ea_reread = ea_iter.next() + assert sample_ea_reread == self.sample_ea + ea1_reread = ea_iter.next() + assert ea1_reread == self.ea1 + ea2_reread = ea_iter.next() + assert ea2_reread == self.ea2 + ea3_reread = ea_iter.next() + assert ea3_reread == self.ea3 + try: ea_iter.next() + except StopIteration: pass + else: assert 0, "Expected end to iterator" + + def testSeriesLocal(self): + """Test backing up and restoring directories with EAs locally""" + self.make_backup_dirs() + dirlist = ['testfiles/ea_test1', 'testfiles/empty', + 'testfiles/ea_test2', 'testfiles/ea_test1'] + BackupRestoreSeries(1, 1, dirlist, compare_eas = 1) + + def testSeriesRemote(self): + """Test backing up, restoring directories with EA remotely""" + self.make_backup_dirs() + dirlist = ['testfiles/ea_test1', 'testfiles/ea_test2', + 'testfiles/empty', 'testfiles/ea_test1'] + BackupRestoreSeries(None, None, dirlist, compare_eas = 1) + +if __name__ == "__main__": unittest.main() diff --git a/rdiff-backup/testing/finaltest.py b/rdiff-backup/testing/finaltest.py index 9837437..590b80b 100644 --- a/rdiff-backup/testing/finaltest.py +++ b/rdiff-backup/testing/finaltest.py @@ -35,7 +35,7 @@ class Local: vft2_in = get_local_rp('vft2_out') timbar_in = get_local_rp('increment1/timbar.pyc') - timbar_out = get_local_rp('../timbar.pyc') # in cur directory + timbar_out = get_local_rp('timbar.pyc') # in cur directory wininc2 = get_local_rp('win-increment2') wininc3 = get_local_rp('win-increment3') @@ -105,7 +105,7 @@ class PathSetter(unittest.TestCase): """Remove any temp directories created by previous tests""" assert not os.system(MiscDir + '/myrm testfiles/output* ' 'testfiles/restoretarget* testfiles/vft_out ' - 'timbar.pyc testfiles/vft2_out') + 'testfiles/timbar.pyc testfiles/vft2_out') def runtest(self): self.delete_tmpdirs() @@ -155,7 +155,7 @@ class PathSetter(unittest.TestCase): timbar_paths = self.getinc_paths("timbar.pyc.", "testfiles/output/rdiff-backup-data/increments") - self.exec_rb(None, timbar_paths[0]) + self.exec_rb(None, timbar_paths[0], 'testfiles/timbar.pyc') self.refresh(Local.timbar_in, Local.timbar_out) assert Local.timbar_in.equal_loose(Local.timbar_out) @@ -242,7 +242,7 @@ class Final(PathSetter): self.exec_rb(None, '../../../../../../proc', 'testfiles/procoutput') def testWindowsMode(self): - """Test backup with the --windows-mode option + """Test backup with quoting enabled We need to delete from the increment? directories long file names, because quoting adds too many extra letters. @@ -260,30 +260,35 @@ class Final(PathSetter): delete_long(Local.wininc3) old_schema = self.rb_schema - self.rb_schema = old_schema + " --windows-mode " + self.rb_schema = old_schema+" --override-chars-to-quote '^a-z0-9_ -.' " self.set_connections(None, None, None, None) self.delete_tmpdirs() - # Back up increment2, this contains a file with colons self.exec_rb(20000, 'testfiles/win-increment2', 'testfiles/output') + self.rb_schema = old_schema # Quoting setting should now be saved time.sleep(1) # Back up increment3 self.exec_rb(30000, 'testfiles/win-increment3', 'testfiles/output') - # Start restore - self.rb_schema = old_schema + ' --windows-restore ' - Globals.time_separator = "_" + # Start restore of increment 2 + Globals.chars_to_quote = '^a-z0-9_ -.' inc_paths = self.getinc_paths("increments.", "testfiles/output/rdiff-backup-data", 1) - Globals.time_separator = ":" + Globals.chars_to_quote = None assert len(inc_paths) == 1, inc_paths - # Restore increment2 self.exec_rb(None, inc_paths[0], 'testfiles/restoretarget2') assert CompareRecursive(Local.wininc2, Local.rpout2, compare_hardlinks = 0) + # Restore increment 3 again, using different syntax + self.rb_schema = old_schema + '-r 30000 ' + self.exec_rb(None, 'testfiles/output', 'testfiles/restoretarget3') + assert CompareRecursive(Local.wininc3, Local.rpout3, + compare_hardlinks = 0) + self.rb_schema = old_schema + # Now check to make sure no ":" in output directory popen_fp = os.popen("find testfiles/output -name '*:*' | wc") wc_output = popen_fp.read() diff --git a/rdiff-backup/testing/metadatatest.py b/rdiff-backup/testing/metadatatest.py index ed7e07c..b908dc2 100644 --- a/rdiff-backup/testing/metadatatest.py +++ b/rdiff-backup/testing/metadatatest.py @@ -40,13 +40,16 @@ class MetadataTest(unittest.TestCase): def testIterator(self): """Test writing RORPs to file and iterating them back""" + def write_rorp_iter_to_file(rorp_iter, file): + for rorp in rorp_iter: file.write(RORP2Record(rorp)) + l = self.get_rpaths() fp = cStringIO.StringIO() write_rorp_iter_to_file(iter(l), fp) fp.seek(0) cstring = fp.read() fp.seek(0) - outlist = list(rorp_extractor(fp).iterate()) + outlist = list(RorpExtractor(fp).iterate()) assert len(l) == len(outlist), (len(l), len(outlist)) for i in range(len(l)): if not l[i].equal_verbose(outlist[i]): @@ -65,18 +68,19 @@ class MetadataTest(unittest.TestCase): rpath_iter = selection.Select(rootrp).set_iter() start_time = time.time() - OpenMetadata(temprp) - for rp in rpath_iter: WriteMetadata(rp) - CloseMetadata() + MetadataFile.open_file(temprp) + for rp in rpath_iter: MetadataFile.write_object(rp) + MetadataFile.close_file() print "Writing metadata took %s seconds" % (time.time() - start_time) return temprp def testSpeed(self): """Test testIterator on 10000 files""" temprp = self.write_metadata_to_temp() + MetadataFile._rp = temprp start_time = time.time(); i = 0 - for rorp in GetMetadata(temprp): i += 1 + for rorp in MetadataFile.get_objects(): i += 1 print "Reading %s metadata entries took %s seconds." % \ (i, time.time() - start_time) @@ -98,11 +102,35 @@ class MetadataTest(unittest.TestCase): """ temprp = self.write_metadata_to_temp() + MetadataFile._rp = temprp start_time = time.time(); i = 0 - for rorp in GetMetadata(temprp, ("subdir3", "subdir10")): i += 1 + for rorp in MetadataFile.get_objects(("subdir3", "subdir10")): i += 1 print "Reading %s metadata entries took %s seconds." % \ (i, time.time() - start_time) assert i == 51 + def test_write(self): + """Test writing to metadata file, then reading back contents""" + global tempdir + temprp = tempdir.append("write_test.gz") + if temprp.lstat(): temprp.delete() + + self.make_temp() + rootrp = rpath.RPath(Globals.local_connection, + "testfiles/various_file_types") + dirlisting = rootrp.listdir() + dirlisting.sort() + rps = map(rootrp.append, dirlisting) + + assert not temprp.lstat() + MetadataFile.open_file(temprp) + for rp in rps: MetadataFile.write_object(rp) + MetadataFile.close_file() + assert temprp.lstat() + + reread_rps = list(MetadataFile.get_objects()) + assert len(reread_rps) == len(rps), (len(reread_rps), len(rps)) + for i in range(len(reread_rps)): + assert reread_rps[i] == rps[i], i if __name__ == "__main__": unittest.main() diff --git a/rdiff-backup/testing/regressiontest.py b/rdiff-backup/testing/regressiontest.py index 7ae60f5..c135387 100644 --- a/rdiff-backup/testing/regressiontest.py +++ b/rdiff-backup/testing/regressiontest.py @@ -12,7 +12,7 @@ testfiles Globals.set('change_source_perms', 1) Globals.counter = 0 -log.Log.setverbosity(3) +log.Log.setverbosity(7) def get_local_rp(extension): return rpath.RPath(Globals.local_connection, "testfiles/" + extension) @@ -172,16 +172,12 @@ class IncrementTest1(unittest.TestCase): hl2.hardlink(hl1.path) Myrm(Local.rpout.path) - old_settings = (Globals.quoting_enabled, Globals.chars_to_quote, - Globals.quoting_char) - Globals.quoting_enabled = 1 + old_chars = Globals.chars_to_quote Globals.chars_to_quote = 'A-Z' - Globals.quoting_char = ';' InternalBackup(1, 1, hldir.path, Local.rpout.path, current_time = 1) InternalBackup(1, 1, "testfiles/empty", Local.rpout.path, current_time = 10000) - (Globals.quoting_enabled, Globals.chars_to_quote, - Globals.quoting_char) = old_settings + Globals.chars_to_quote = old_chars def test_long_socket(self): """Test backing up a directory with long sockets in them @@ -389,8 +385,10 @@ class MirrorTest(PathSetter): Main.force = 1 assert not rpout.append("rdiff-backup-data").lstat() Main.misc_setup([rpin, rpout]) + Main.backup_check_dirs(rpin, rpout) + Main.backup_set_fs_globals(rpin, rpout) + Main.backup_set_rbdir(rpin, rpout) Main.backup_set_select(rpin) - Main.backup_init_dirs(rpin, rpout) backup.Mirror(rpin, rpout) log.ErrorLog.close() log.Log.close_logfile() diff --git a/rdiff-backup/testing/restoretest.py b/rdiff-backup/testing/restoretest.py index df99300..b401643 100644 --- a/rdiff-backup/testing/restoretest.py +++ b/rdiff-backup/testing/restoretest.py @@ -143,22 +143,6 @@ class RestoreTest(unittest.TestCase): "testfiles/output", 5000) assert CompareRecursive(inc1_rp, target_rp, compare_hardlinks = 0) -# def testRestoreCorrupt(self): -# """Test restoring a partially corrupt archive -# -# The problem here is that a directory is missing from what is -# to be restored, but because the previous backup was aborted in -# the middle, some of the files in that directory weren't marked -# as .missing. -# -# """ -# Myrm("testfiles/output") -# InternalRestore(1, 1, "testfiles/restoretest4", "testfiles/output", -# 10000) -# assert os.lstat("testfiles/output") -# self.assertRaises(OSError, os.lstat, "testfiles/output/tmp") -# self.assertRaises(OSError, os.lstat, "testfiles/output/rdiff-backup") - def testRestoreNoincs(self): """Test restoring a directory with no increments, just mirror""" Myrm("testfiles/output") |