From 62399345db69c91b455d27413536ef10ce2d847f Mon Sep 17 00:00:00 2001 From: ben Date: Wed, 15 May 2002 00:25:31 +0000 Subject: Belatedly added filename_mapping (should have been added with 0.7.4). Changed various files to fix --exclude-filelist problems when source remote. git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@85 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109 --- rdiff-backup/rdiff_backup/connection.py | 8 +++ rdiff-backup/rdiff_backup/selection.py | 14 +++-- rdiff-backup/src/connection.py | 8 +++ rdiff-backup/src/filename_mapping.py | 93 +++++++++++++++++++++++++++++++++ rdiff-backup/src/globals.py | 15 ++++-- rdiff-backup/src/main.py | 25 +++++---- rdiff-backup/src/selection.py | 14 +++-- 7 files changed, 154 insertions(+), 23 deletions(-) create mode 100644 rdiff-backup/src/filename_mapping.py diff --git a/rdiff-backup/rdiff_backup/connection.py b/rdiff-backup/rdiff_backup/connection.py index 94afea3..54cf527 100644 --- a/rdiff-backup/rdiff_backup/connection.py +++ b/rdiff-backup/rdiff_backup/connection.py @@ -1,3 +1,4 @@ +from __future__ import generators execfile("rdiff.py") import types, os, tempfile, cPickle, shutil, traceback @@ -487,3 +488,10 @@ class VirtualFile: def close(self): return self.connection.VirtualFile.closebyid(self.id) + + def __iter__(self): + """Iterates lines in file, like normal iter(file) behavior""" + while 1: + line = self.readline() + if not line: break + yield line diff --git a/rdiff-backup/rdiff_backup/selection.py b/rdiff-backup/rdiff_backup/selection.py index cae6db3..bd679de 100644 --- a/rdiff-backup/rdiff_backup/selection.py +++ b/rdiff-backup/rdiff_backup/selection.py @@ -157,7 +157,7 @@ class Select: if result is not None: return result return 1 - def ParseArgs(self, argtuples): + def ParseArgs(self, argtuples, filelists): """Create selection functions based on list of tuples The tuples have the form (option string, additional argument) @@ -169,6 +169,7 @@ class Select: information is sent over the link. """ + filelists_index = 0 try: for opt, arg in argtuples: if opt == "--exclude": @@ -176,19 +177,22 @@ class Select: elif opt == "--exclude-device-files": self.add_selection_func(self.devfiles_get_sf()) elif opt == "--exclude-filelist": - self.add_selection_func(self.filelist_get_sf(arg[1], - 0, arg[0])) + self.add_selection_func(self.filelist_get_sf( + filelists[filelists_index], 0, arg)) + filelists_index += 1 elif opt == "--exclude-regexp": self.add_selection_func(self.regexp_get_sf(arg, 0)) elif opt == "--include": self.add_selection_func(self.glob_get_sf(arg, 1)) elif opt == "--include-filelist": - self.add_selection_func(self.filelist_get_sf(arg[1], - 1, arg[0])) + self.add_selection_func(self.filelist_get_sf( + filelists[filelists_index], 1, arg)) + filelists_index += 1 elif opt == "--include-regexp": self.add_selection_func(self.regexp_get_sf(arg, 1)) else: assert 0, "Bad selection option %s" % opt except SelectError, e: self.parse_catch_error(e) + assert filelists_index == len(filelists) self.parse_last_excludes() self.parse_rbdir_exclude() diff --git a/rdiff-backup/src/connection.py b/rdiff-backup/src/connection.py index 94afea3..54cf527 100644 --- a/rdiff-backup/src/connection.py +++ b/rdiff-backup/src/connection.py @@ -1,3 +1,4 @@ +from __future__ import generators execfile("rdiff.py") import types, os, tempfile, cPickle, shutil, traceback @@ -487,3 +488,10 @@ class VirtualFile: def close(self): return self.connection.VirtualFile.closebyid(self.id) + + def __iter__(self): + """Iterates lines in file, like normal iter(file) behavior""" + while 1: + line = self.readline() + if not line: break + yield line diff --git a/rdiff-backup/src/filename_mapping.py b/rdiff-backup/src/filename_mapping.py new file mode 100644 index 0000000..78c37d9 --- /dev/null +++ b/rdiff-backup/src/filename_mapping.py @@ -0,0 +1,93 @@ +execfile("selection.py") +import re + +####################################################################### +# +# filename_mapping - used to coordinate related filenames +# +# For instance, some source filenames may contain characters not +# allowed on the mirror end. Also, if a source filename is very long +# (say 240 characters), the extra characters added to related +# increments may put them over the usual 255 character limit. +# + +class FilenameMapping: + """Contains class methods which coordinate related filenames""" + max_filename_length = 255 + + # If true, enable character quoting, and set characters making + # regex-style range. + chars_to_quote = None + + # These compiled regular expressions are used in quoting and unquoting + chars_to_quote_regexp = None + unquoting_regexp = None + + # Use given char to quote. Default is set in Globals. + quoting_char = None + + def set_init_quote_vals(cls): + """Set quoting value from Globals on all conns""" + for conn in Globals.connections: + conn.FilenameMapping.set_init_quote_vals_local() + + def set_init_quote_vals_local(cls): + """Set value on local connection, initialize regexps""" + cls.chars_to_quote = Globals.chars_to_quote + if len(Globals.quoting_char) != 1: + Log.FatalError("Expected single character for quoting char," + "got '%s' instead" % (Globals.quoting_char,)) + cls.quoting_char = Globals.quoting_char + cls.init_quoting_regexps() + + def init_quoting_regexps(cls): + """Compile quoting regular expressions""" + try: + cls.chars_to_quote_regexp = \ + re.compile("[%s%s]" % (cls.chars_to_quote, + cls.quoting_char), re.S) + cls.unquoting_regexp = \ + re.compile("%s[0-9]{3}" % cls.quoting_char, re.S) + except re.error: + Log.FatalError("Error '%s' when processing char quote list %s" % + (re.error, cls.chars_to_quote)) + + def quote(cls, path): + """Return quoted version of given path + + Any characters quoted will be replaced by the quoting char and + the ascii number of the character. For instance, "10:11:12" + would go to "10;05811;05812" if ":" were quoted and ";" were + the quoting character. + + """ + return cls.chars_to_quote_regexp.sub(cls.quote_single, path) + + def quote_single(cls, match): + """Return replacement for a single character""" + return "%s%03d" % (cls.quoting_char, ord(match.group())) + + def unquote(cls, path): + """Return original version of quoted filename""" + return cls.unquoting_regexp.sub(cls.unquote_single, path) + + def unquote_single(cls, match): + """Unquote a single quoted character""" + assert len(match.group()) == 4 + return chr(int(match.group()[1:])) + + def get_quoted_dir_children(cls, rpath): + """For rpath directory, return list of quoted children in dir""" + if not rpath.isdir(): return [] + dir_pairs = [(cls.unquote(filename), filename) + for filename in rpath.listdir()] + dir_pairs.sort() # sort by real index, not quoted part + child_list = [] + for unquoted, filename in dir_pairs: + childrp = rpath.append(unquoted) + childrp.quote_path() + child_list.append(childrp) + return child_list + +MakeClass(FilenameMapping) + diff --git a/rdiff-backup/src/globals.py b/rdiff-backup/src/globals.py index 2f0b0ca..58c43d4 100644 --- a/rdiff-backup/src/globals.py +++ b/rdiff-backup/src/globals.py @@ -192,12 +192,19 @@ class Globals: else: cls.__dict__[name] = re.compile(re_string) postset_regexp_local = classmethod(postset_regexp_local) - def set_select(cls, dsrpath, tuplelist, quote_mode = None): - """Initialize select object using tuplelist""" + def set_select(cls, dsrpath, tuplelist, quote_mode, *filelists): + """Initialize select object using tuplelist + + Note that each list in filelists must each be passed as + separate arguments, so each is recognized as a file by the + connection. Otherwise we will get an error because a list + containing files can't be pickled. + + """ if dsrpath.source: cls.select_source = Select(dsrpath, quote_mode) - cls.select_source.ParseArgs(tuplelist) + cls.select_source.ParseArgs(tuplelist, filelists) else: cls.select_mirror = Select(dsrpath, quote_mode) - cls.select_mirror.ParseArgs(tuplelist) + cls.select_mirror.ParseArgs(tuplelist, filelists) set_select = classmethod(set_select) diff --git a/rdiff-backup/src/main.py b/rdiff-backup/src/main.py index c130e56..72d93ce 100755 --- a/rdiff-backup/src/main.py +++ b/rdiff-backup/src/main.py @@ -14,6 +14,7 @@ class Main: self.remote_cmd, self.remote_schema = None, None self.force = None self.select_opts, self.select_mirror_opts = [], [] + self.select_files = [] def parse_cmdlineoptions(self): """Parse argument list and set global preferences""" @@ -37,8 +38,8 @@ class Main: "restore-as-of=", "resume", "resume-window=", "server", "terminal-verbosity=", "test-server", "verbosity", "version", "windows-mode", "windows-time-format"]) - except getopt.error: - self.commandline_error("Error parsing commandline options") + except getopt.error, e: + self.commandline_error("Bad commandline options: %s" % str(e)) for opt, arg in optlist: if opt == "-b" or opt == "--backup-mode": self.action = "backup" @@ -55,20 +56,24 @@ class Main: elif opt == "--exclude-device-files": self.select_opts.append((opt, arg)) elif opt == "--exclude-filelist": - self.select_opts.append((opt, (arg, sel_fl(arg)))) + self.select_opts.append((opt, arg)) + self.select_files.append(sel_fl(arg)) elif opt == "--exclude-filelist-stdin": self.select_opts.append(("--exclude-filelist", - ("standard input", sys.stdin))) + "standard input")) + self.select_files.append(sys.stdin) elif opt == "--exclude-mirror": self.select_mirror_opts.append(("--exclude", arg)) elif opt == "--exclude-regexp": self.select_opts.append((opt, arg)) elif opt == "--force": self.force = 1 elif opt == "--include": self.select_opts.append((opt, arg)) elif opt == "--include-filelist": - self.select_opts.append((opt, (arg, sel_fl(arg)))) + self.select_opts.append((opt, arg)) + self.select_files.append(sel_fl(arg)) elif opt == "--include-filelist-stdin": self.select_opts.append(("--include-filelist", - ("standard input", sys.stdin))) + "standard input")) + self.select_files.append(sys.stdin) elif opt == "--include-regexp": self.select_opts.append((opt, arg)) elif opt == "-l" or opt == "--list-increments": @@ -227,7 +232,8 @@ rdiff-backup with the --force option if you want to mirror anyway.""" % def backup_init_select(self, rpin, rpout): """Create Select objects on source and dest connections""" - rpin.conn.Globals.set_select(DSRPath(1, rpin), self.select_opts) + rpin.conn.Globals.set_select(DSRPath(1, rpin), self.select_opts, + None, *self.select_files) rpout.conn.Globals.set_select(DSRPath(None, rpout), self.select_mirror_opts, 1) @@ -378,8 +384,9 @@ Try restoring from an increment file (the filenames look like the restore operation isn't. """ - Globals.set_select(DSRPath(1, rpin), self.select_mirror_opts) - Globals.set_select(DSRPath(None, rpout), self.select_opts) + Globals.set_select(DSRPath(1, rpin), self.select_mirror_opts, None) + Globals.set_select(DSRPath(None, rpout), self.select_opts, None, + *self.select_files) def restore_get_root(self, rpin): """Return (mirror root, index) and set the data dir diff --git a/rdiff-backup/src/selection.py b/rdiff-backup/src/selection.py index cae6db3..bd679de 100644 --- a/rdiff-backup/src/selection.py +++ b/rdiff-backup/src/selection.py @@ -157,7 +157,7 @@ class Select: if result is not None: return result return 1 - def ParseArgs(self, argtuples): + def ParseArgs(self, argtuples, filelists): """Create selection functions based on list of tuples The tuples have the form (option string, additional argument) @@ -169,6 +169,7 @@ class Select: information is sent over the link. """ + filelists_index = 0 try: for opt, arg in argtuples: if opt == "--exclude": @@ -176,19 +177,22 @@ class Select: elif opt == "--exclude-device-files": self.add_selection_func(self.devfiles_get_sf()) elif opt == "--exclude-filelist": - self.add_selection_func(self.filelist_get_sf(arg[1], - 0, arg[0])) + self.add_selection_func(self.filelist_get_sf( + filelists[filelists_index], 0, arg)) + filelists_index += 1 elif opt == "--exclude-regexp": self.add_selection_func(self.regexp_get_sf(arg, 0)) elif opt == "--include": self.add_selection_func(self.glob_get_sf(arg, 1)) elif opt == "--include-filelist": - self.add_selection_func(self.filelist_get_sf(arg[1], - 1, arg[0])) + self.add_selection_func(self.filelist_get_sf( + filelists[filelists_index], 1, arg)) + filelists_index += 1 elif opt == "--include-regexp": self.add_selection_func(self.regexp_get_sf(arg, 1)) else: assert 0, "Bad selection option %s" % opt except SelectError, e: self.parse_catch_error(e) + assert filelists_index == len(filelists) self.parse_last_excludes() self.parse_rbdir_exclude() -- cgit v1.2.1