diff options
author | bescoto <bescoto@2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109> | 2002-12-21 06:35:08 +0000 |
---|---|---|
committer | bescoto <bescoto@2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109> | 2002-12-21 06:35:08 +0000 |
commit | e95a61773adb2f98499cf13ff543f4249ee38226 (patch) | |
tree | 835e2cc34386eb6c798026ebf5acd1ea30a9c537 /rdiff-backup/rdiff_backup | |
parent | 5d3974dcd81e009293afe2372364983ad3810568 (diff) | |
download | rdiff-backup-e95a61773adb2f98499cf13ff543f4249ee38226.tar.gz |
Removed most destructive stepping operations
git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@251 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109
Diffstat (limited to 'rdiff-backup/rdiff_backup')
-rw-r--r-- | rdiff-backup/rdiff_backup/Globals.py | 31 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/Main.py | 43 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/SetConnections.py | 5 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/connection.py | 1 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/highlevel.py | 47 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/increment.py | 12 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/restore.py | 7 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/rpath.py | 33 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/selection.py | 238 |
9 files changed, 189 insertions, 228 deletions
diff --git a/rdiff-backup/rdiff_backup/Globals.py b/rdiff-backup/rdiff_backup/Globals.py index 61991d4..1ba7490 100644 --- a/rdiff-backup/rdiff_backup/Globals.py +++ b/rdiff-backup/rdiff_backup/Globals.py @@ -55,10 +55,6 @@ change_ownership = None # permissions). change_mirror_perms = (process_uid != 0) -# If true, temporarily change permissions of unreadable files in -# the source directory to make sure we can read all files. -change_source_perms = None - # If true, try to reset the atimes of the source partition. preserve_atime = None @@ -103,24 +99,9 @@ client_conn = None # list. changed_settings = [] -# rdiff-backup will try to checkpoint its state every -# checkpoint_interval seconds. Then when resuming, at most this -# amount of time is lost. -checkpoint_interval = 20 - # The RPath of the rdiff-backup-data directory. rbdir = None -# Indicates if a resume or a lack of resume is forced. This -# should be None for the default. 0 means don't resume, and 1 -# means resume. -resume = None - -# If there has been an aborted backup fewer than this many seconds -# ago, attempt to resume it where it left off instead of starting -# a new one. -resume_window = 7200 - # This string is used when recognizing and creating time strings. # If the time_separator is ":", then W3 datetime strings like # 2001-12-07T04:22:01-07:00 are produced. It can be set to "_" to @@ -265,7 +246,7 @@ def postset_regexp_local(name, re_string, flags): if flags: globals()[name] = re.compile(re_string, flags) else: globals()[name] = re.compile(re_string) -def set_select(dsrpath, tuplelist, quote_mode, *filelists): +def set_select(source, rpath, tuplelist, quote_mode, *filelists): """Initialize select object using tuplelist Note that each list in filelists must each be passed as @@ -275,12 +256,10 @@ def set_select(dsrpath, tuplelist, quote_mode, *filelists): """ global select_source, select_mirror - if dsrpath.source: - select_source = Select(dsrpath, quote_mode) - select_source.ParseArgs(tuplelist, filelists) - else: - select_mirror = Select(dsrpath, quote_mode) - select_mirror.ParseArgs(tuplelist, filelists) + sel = Select(rpath, quote_mode) + sel.ParseArgs(tuplelist, filelists) + if source: select_source = sel + else: select_mirror = sel from rpath import * # kludge to avoid circularity - not needed in this module diff --git a/rdiff-backup/rdiff_backup/Main.py b/rdiff-backup/rdiff_backup/Main.py index 2b3d497..f130875 100644 --- a/rdiff-backup/rdiff_backup/Main.py +++ b/rdiff-backup/rdiff_backup/Main.py @@ -25,7 +25,6 @@ from log import * from lazy import * from connection import * from rpath import * -from destructive_stepping import * from robust import * from restore import * from highlevel import * @@ -49,21 +48,20 @@ def parse_cmdlineoptions(arglist): except IOError: Log.FatalError("Error opening file %s" % filename) try: optlist, args = getopt.getopt(arglist, "blr:sv:V", - ["backup-mode", "calculate-average", "change-source-perms", - "chars-to-quote=", "checkpoint-interval=", "current-time=", - "exclude=", "exclude-device-files", "exclude-filelist=", - "exclude-filelist-stdin", "exclude-globbing-filelist=", - "exclude-mirror=", "exclude-other-filesystems", - "exclude-regexp=", "exclude-special-files", "force", - "include=", "include-filelist=", "include-filelist-stdin", + ["backup-mode", "calculate-average", "chars-to-quote=", + "current-time=", "exclude=", "exclude-device-files", + "exclude-filelist=", "exclude-filelist-stdin", + "exclude-globbing-filelist=", "exclude-mirror=", + "exclude-other-filesystems", "exclude-regexp=", + "exclude-special-files", "force", "include=", + "include-filelist=", "include-filelist-stdin", "include-globbing-filelist=", "include-regexp=", "list-changed-since=", "list-increments", "no-compression", - "no-compression-regexp=", "no-hard-links", "no-resume", - "null-separator", "parsable-output", "print-statistics", - "quoting-char=", "remote-cmd=", "remote-schema=", - "remove-older-than=", "restore-as-of=", "restrict=", - "restrict-read-only=", "restrict-update-only=", "resume", - "resume-window=", "server", "sleep-ratio=", + "no-compression-regexp=", "no-hard-links", "null-separator", + "parsable-output", "print-statistics", "quoting-char=", + "remote-cmd=", "remote-schema=", "remove-older-than=", + "restore-as-of=", "restrict=", "restrict-read-only=", + "restrict-update-only=", "server", "sleep-ratio=", "ssh-no-compression", "terminal-verbosity=", "test-server", "verbosity=", "version", "windows-mode", "windows-time-format"]) @@ -73,13 +71,9 @@ def parse_cmdlineoptions(arglist): for opt, arg in optlist: if opt == "-b" or opt == "--backup-mode": action = "backup" elif opt == "--calculate-average": action = "calculate-average" - elif opt == "--change-source-perms": - Globals.set('change_source_perms', 1) elif opt == "--chars-to-quote": Globals.set('chars_to_quote', arg) Globals.set('quoting_enabled', 1) - elif opt == "--checkpoint-interval": - Globals.set_integer('checkpoint_interval', arg) elif opt == "--current-time": Globals.set_integer('current_time', arg) elif opt == "--exclude": select_opts.append((opt, arg)) @@ -118,7 +112,6 @@ def parse_cmdlineoptions(arglist): elif opt == "--no-compression-regexp": Globals.set("no_compression_regexp_string", arg) elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0) - elif opt == '--no-resume': Globals.resume = 0 elif opt == "--null-separator": Globals.set("null_separator", 1) elif opt == "--parsable-output": Globals.set('parsable_output', 1) elif opt == "--print-statistics": @@ -140,9 +133,6 @@ def parse_cmdlineoptions(arglist): elif opt == "--restrict-update-only": Globals.security_level = "update-only" Globals.restrict_path = arg - elif opt == '--resume': Globals.resume = 1 - elif opt == '--resume-window': - Globals.set_integer('resume_window', arg) elif opt == "-s" or opt == "--server": action = "server" Globals.server = 1 @@ -263,9 +253,8 @@ def Backup(rpin, rpout): def backup_init_select(rpin, rpout): """Create Select objects on source and dest connections""" - rpin.conn.Globals.set_select(DSRPath(1, rpin), select_opts, - None, *select_files) - rpout.conn.Globals.set_select(DSRPath(None, rpout), select_mirror_opts, 1) + rpin.conn.Globals.set_select(1, rpin, select_opts, None, *select_files) + rpout.conn.Globals.set_select(0, rpout, select_mirror_opts, 1) def backup_init_dirs(rpin, rpout): """Make sure rpin and rpout are valid, init data dir and logging""" @@ -424,8 +413,8 @@ def restore_init_select(rpin, rpout): the restore operation isn't. """ - Globals.set_select(DSRPath(1, rpin), select_mirror_opts, None) - Globals.set_select(DSRPath(None, rpout), select_opts, None, *select_files) + Globals.set_select(1, rpin, select_mirror_opts, None) + Globals.set_select(0, rpout, select_opts, None, *select_files) def restore_get_root(rpin): """Return (mirror root, index) and set the data dir diff --git a/rdiff-backup/rdiff_backup/SetConnections.py b/rdiff-backup/rdiff_backup/SetConnections.py index f0e413a..3bdc36f 100644 --- a/rdiff-backup/rdiff_backup/SetConnections.py +++ b/rdiff-backup/rdiff_backup/SetConnections.py @@ -197,11 +197,6 @@ def BackupInitConnections(reading_conn, writing_conn): writing_conn.Globals.set("isbackup_writer", 1) UpdateGlobal("backup_reader", reading_conn) UpdateGlobal("backup_writer", writing_conn) - if (Globals.change_source_perms and - reading_conn.Globals.get("process_uid") == 0): - Log("Warning: --change_source_perms should usually not be used when\n" - "the reading connection is running as root, because root can\n" - "read all files regardless of their permissions.", 2) def CloseConnections(): """Close all connections. Run by client""" diff --git a/rdiff-backup/rdiff_backup/connection.py b/rdiff-backup/rdiff_backup/connection.py index 9d139af..09e0a92 100644 --- a/rdiff-backup/rdiff_backup/connection.py +++ b/rdiff-backup/rdiff_backup/connection.py @@ -547,7 +547,6 @@ from connection import * from rpath import * from robust import * from rorpiter import * -from destructive_stepping import * from selection import * from statistics import * from increment import * diff --git a/rdiff-backup/rdiff_backup/highlevel.py b/rdiff-backup/rdiff_backup/highlevel.py index eb3628f..bcb07d6 100644 --- a/rdiff-backup/rdiff_backup/highlevel.py +++ b/rdiff-backup/rdiff_backup/highlevel.py @@ -87,27 +87,23 @@ class HLSourceStruct: """Return diffs and finalize any dsrp changes remaining Return a rorpiterator with files included of signatures of - dissimilar files. This is the last operation run on the local - filestream, so finalize dsrp writes. + dissimilar files. """ collated = RORPIter.CollateIterators(cls.initial_dsiter2, sigiter) - finalizer = IterTreeReducer(DestructiveSteppingFinalizer, []) - def error_handler(exc, dest_sig, dsrp): + def error_handler(exc, dest_sig, rp): Log("Error %s producing a diff of %s" % - (exc, dsrp and dsrp.path), 2) + (exc, rp and rp.path), 2) return None def diffs(): - for dsrp, dest_sig in collated: + for rp, dest_sig in collated: if dest_sig: if dest_sig.isplaceholder(): yield dest_sig else: diff = Robust.check_common_error( - error_handler, RORPIter.diffonce, [dest_sig, dsrp]) + error_handler, RORPIter.diffonce, [dest_sig, rp]) if diff: yield diff - if dsrp: finalizer(dsrp.index, dsrp) - finalizer.Finish() return diffs() MakeClass(HLSourceStruct) @@ -117,7 +113,7 @@ class HLDestinationStruct: """Hold info used by HL on the destination side""" _session_info = None # set to si if resuming def split_initial_dsiter(cls): - """Set initial_dsiters (iteration of all dsrps from rpath)""" + """Set initial_dsiters (iteration of all rps from rpath)""" result, cls.initial_dsiter2 = \ Iter.multiplex(Globals.select_mirror.set_iter(), 2) return result @@ -192,10 +188,10 @@ class HLDestinationStruct: return RORPIter.Signatures(dissimilars) def get_dsrp(cls, dest_rpath, index): - """Return initialized dsrp based on dest_rpath with given index""" - dsrp = DSRPath(None, dest_rpath.conn, dest_rpath.base, index) - if Globals.quoting_enabled: dsrp.quote_path() - return dsrp + """Return initialized rpath based on dest_rpath with given index""" + rp = RPath(dest_rpath.conn, dest_rpath.base, index) + if Globals.quoting_enabled: rp.quote_path() + return rp def get_finalizer(cls): """Return finalizer, starting from session info if necessary""" @@ -226,8 +222,8 @@ class HLDestinationStruct: def patch_and_finalize(cls, dest_rpath, diffs): """Apply diffs and finalize""" collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2) - finalizer = cls.get_finalizer() - diff_rorp, dsrp = None, None + #finalizer = cls.get_finalizer() + diff_rorp, rp = None, None def patch(diff_rorp, dsrp): if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index) @@ -244,13 +240,14 @@ class HLDestinationStruct: diff_rorp, dsrp = indexed_tuple dsrp = Robust.check_common_error(error_handler, patch, [diff_rorp, dsrp]) - finalizer(dsrp.index, dsrp) - finalizer.Finish() + #finalizer(dsrp.index, dsrp) + #finalizer.Finish() def patch_w_datadir_writes(cls, dest_rpath, diffs, inc_rpath): """Apply diffs and finalize, with checkpointing and statistics""" collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2) - finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath) + #finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath) + finalizer, ITR = None, cls.get_MirrorITR(inc_rpath) MiscStats.open_dir_stats_file() dsrp, finished_dsrp = None, None @@ -261,10 +258,10 @@ class HLDestinationStruct: if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index) if diff_rorp and diff_rorp.isplaceholder(): diff_rorp = None ITR(dsrp.index, diff_rorp, dsrp) - finalizer(dsrp.index, dsrp) + #finalizer(dsrp.index, dsrp) finished_dsrp = dsrp ITR.Finish() - finalizer.Finish() + #finalizer.Finish() except: cls.handle_last_error(finished_dsrp, finalizer, ITR) if Globals.preserve_hardlinks: Hardlink.final_writedata() @@ -274,7 +271,8 @@ class HLDestinationStruct: def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath): """Apply diffs, write increment if necessary, and finalize""" collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2) - finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath) + #finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath) + finalizer, ITR = None, cls.get_ITR(inc_rpath) MiscStats.open_dir_stats_file() dsrp, finished_dsrp = None, None @@ -286,10 +284,10 @@ class HLDestinationStruct: if not dsrp: dsrp = cls.get_dsrp(dest_rpath, index) if diff_rorp and diff_rorp.isplaceholder(): diff_rorp = None ITR(index, diff_rorp, dsrp) - finalizer(index, dsrp) + #finalizer(index, dsrp) finished_dsrp = dsrp ITR.Finish() - finalizer.Finish() + #finalizer.Finish() except: cls.handle_last_error(finished_dsrp, finalizer, ITR) if Globals.preserve_hardlinks: Hardlink.final_writedata() @@ -311,6 +309,5 @@ from log import * from rpath import * from robust import * from increment import * -from destructive_stepping import * from rorpiter import * import Globals, Hardlink, MiscStats, metadata diff --git a/rdiff-backup/rdiff_backup/increment.py b/rdiff-backup/rdiff_backup/increment.py index baba9e1..5040c40 100644 --- a/rdiff-backup/rdiff_backup/increment.py +++ b/rdiff-backup/rdiff_backup/increment.py @@ -145,8 +145,6 @@ class IncrementITRB(StatsITRB): 4. Directory -> Normal file: Wait until the end, so we can process all the files in the directory. - Remember this object needs to be pickable. - """ # Iff true, mirror file was a directory mirror_isdirectory = None @@ -299,7 +297,14 @@ class MirrorITRB(StatsITRB): def start_process(self, index, diff_rorp, mirror_dsrp): """Initialize statistics and do actual writing to mirror""" self.start_stats(mirror_dsrp) - if diff_rorp and not diff_rorp.isplaceholder(): + if (diff_rorp and diff_rorp.isdir() or + not diff_rorp and mirror_dsrp.isdir()): + # mirror_dsrp will end up as directory, update attribs later + if not diff_rorp: diff_rorp = mirror_dsrp.get_rorpath() + if not mirror_dsrp.isdir(): + mirror_dsrp.delete() + mirror_dsrp.mkdir() + elif diff_rorp and not diff_rorp.isplaceholder(): RORPIter.patchonce_action(None, mirror_dsrp, diff_rorp).execute() self.incpref = self.inc_rpath.new_index(index) @@ -309,6 +314,7 @@ class MirrorITRB(StatsITRB): """Update statistics when leaving""" self.end_stats(self.diff_rorp, self.mirror_dsrp) if self.mirror_dsrp.isdir(): + RPathStatic.copy_attribs(self.diff_rorp, self.mirror_dsrp) MiscStats.write_dir_stats_line(self, self.mirror_dsrp.index) def can_fast_process(self, index, diff_rorp, mirror_dsrp): diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py index e361512..9ca279e 100644 --- a/rdiff-backup/rdiff_backup/restore.py +++ b/rdiff-backup/rdiff_backup/restore.py @@ -41,7 +41,6 @@ class Restore: same index as mirror. """ - if not isinstance(mirror, DSRPath): mirror = DSRPath(1, mirror) if not isinstance(target, DSRPath): target = DSRPath(None, target) mirror_time = Restore.get_mirror_time() @@ -129,18 +128,16 @@ class Restore: "foo/bar". """ - assert isinstance(mirror, DSRPath) and isinstance(target, DSRPath) + assert isinstance(target, DSRPath) assert mirror.index == rid.index - mirror_finalizer = IterTreeReducer(DestructiveSteppingFinalizer, ()) target_finalizer = IterTreeReducer(DestructiveSteppingFinalizer, ()) for rcd in Restore.yield_rcds(rid.index, mirror, rid, target, time, mirror_time): rcd.RestoreFile() - if rcd.mirror: mirror_finalizer(rcd.index, rcd.mirror) + #if rcd.mirror: mirror_finalizer(rcd.index, rcd.mirror) target_finalizer(rcd.target.index, rcd.target) target_finalizer.Finish() - mirror_finalizer.Finish() def yield_rcds(index, mirrorrp, rid, target, rest_time, mirror_time): """Iterate RestoreCombinedData objects starting with given args diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py index 9d54872..5773dd7 100644 --- a/rdiff-backup/rdiff_backup/rpath.py +++ b/rdiff-backup/rdiff_backup/rpath.py @@ -288,6 +288,8 @@ class RORPath(RPathStatic): (not Globals.change_ownership or self.issym())): # Don't compare gid/uid for symlinks or if not change_ownership pass + elif key == 'mtime': + Log("%s differs only in mtime, skipping" % (self.path,), 2) elif key == 'atime' and not Globals.preserve_atime: pass elif key == 'devloc' or key == 'inode' or key == 'nlink': pass elif key == 'size' and not self.isreg(): pass @@ -319,6 +321,10 @@ class RORPath(RPathStatic): """Reproduce RORPath from __getstate__ output""" self.index, self.data = rorp_state + def get_rorpath(self): + """Return new rorpath based on self""" + return RORPath(self.index, self.data.copy()) + def make_placeholder(self): """Make rorp into a placeholder @@ -697,19 +703,15 @@ class RPath(RORPath): return self.conn.Globals.get('process_gid') == self.data['gid'] def delete(self): - """Delete file at self.path - - The destructive stepping allows this function to delete - directories even if they have files and we lack permissions. - - """ + """Delete file at self.path. Recursively deletes directories.""" Log("Deleting %s" % self.path, 7) self.setdata() - if not self.lstat(): return # must have been deleted in meantime + if not self.lstat(): + Log("Warning: %s does not exist---deleted in meantime?" + % (self.path,), 2) elif self.isdir(): itm = IterTreeReducer(RpathDeleter, []) - for dsrp in Select(DSRPath(None, self)).set_iter(): - itm(dsrp.index, dsrp) + for rp in Select(self).set_iter(): itm(rp.index, rp) itm.Finish() else: self.conn.os.unlink(self.path) self.setdata() @@ -891,18 +893,17 @@ class RPathFileHook: import FilenameMapping from lazy import * from selection import * -from destructive_stepping import * from highlevel import * class RpathDeleter(ITRBranch): """Delete a directory. Called by RPath.delete()""" - def start_process(self, index, dsrp): - self.dsrp = dsrp + def start_process(self, index, rp): + self.rp = rp def end_process(self): - if self.dsrp.isdir(): self.dsrp.rmdir() - else: self.dsrp.delete() + if self.rp.isdir(): self.rp.rmdir() + else: self.rp.delete() - def can_fast_process(self, index, dsrp): return not dsrp.isdir() - def fast_process(self, index, dsrp): dsrp.delete() + def can_fast_process(self, index, rp): return not rp.isdir() + def fast_process(self, index, rp): rp.delete() diff --git a/rdiff-backup/rdiff_backup/selection.py b/rdiff-backup/rdiff_backup/selection.py index 9fb43fc..70203f4 100644 --- a/rdiff-backup/rdiff_backup/selection.py +++ b/rdiff-backup/rdiff_backup/selection.py @@ -28,7 +28,6 @@ from __future__ import generators import re from log import * from robust import * -from destructive_stepping import * import FilenameMapping @@ -46,7 +45,7 @@ class GlobbingError(SelectError): class Select: - """Iterate appropriate DSRPaths in given directory + """Iterate appropriate RPaths in given directory This class acts as an iterator on account of its next() method. Basically, it just goes through all the files in a directory in @@ -66,7 +65,7 @@ class Select: file in the directory gets included, so does the directory. As mentioned above, each test takes the form of a selection - function. The selection function takes a dsrp, and returns: + function. The selection function takes an rpath, and returns: None - means the test has nothing to say about the related file 0 - the file is excluded by the test @@ -82,18 +81,18 @@ class Select: # This re should not match normal filenames, but usually just globs glob_re = re.compile("(.*[*?[]|ignorecase\\:)", re.I | re.S) - def __init__(self, dsrpath, quoted_filenames = None): - """DSRPIterator initializer. dsrp is the root directory + def __init__(self, rpath, quoted_filenames = None): + """Select initializer. rpath is the root directory When files have quoted characters in them, quoted_filenames should be true. Then RPath's index will be the unquoted version. """ - assert isinstance(dsrpath, DSRPath) + assert isinstance(rpath, RPath) self.selection_functions = [] - self.dsrpath = dsrpath - self.prefix = self.dsrpath.path + self.rpath = rpath + self.prefix = self.rpath.path self.quoting_on = Globals.quoting_enabled and quoted_filenames def set_iter(self, starting_index = None, iterate_parents = None, @@ -103,19 +102,19 @@ class Select: Will iterate indicies greater than starting_index. If iterate_parents is true, will also include parents of starting_index in iteration. Selection function sel_func is - called on each dsrp and is usually self.Select. Returns self + called on each rpath and is usually self.Select. Returns self just for convenience. """ if not sel_func: sel_func = self.Select - self.dsrpath.setdata() # this may have changed since Select init + self.rpath.setdata() # this may have changed since Select init if starting_index is not None: self.starting_index = starting_index - self.iter = self.iterate_starting_from(self.dsrpath, + self.iter = self.iterate_starting_from(self.rpath, self.iterate_starting_from, sel_func) elif self.quoting_on: - self.iter = self.Iterate(self.dsrpath, self.Iterate, sel_func) - else: self.iter = self.Iterate_fast(self.dsrpath, sel_func) + self.iter = self.Iterate(self.rpath, self.Iterate, sel_func) + else: self.iter = self.Iterate_fast(self.rpath, sel_func) # only iterate parents if we are not starting from beginning self.iterate_parents = starting_index is not None and iterate_parents @@ -123,7 +122,7 @@ class Select: self.__iter__ = lambda: self return self - def Iterate_fast(self, dsrpath, sel_func): + def Iterate_fast(self, rpath, sel_func): """Like Iterate, but don't recur, saving time Only handles standard case (quoting off, starting from @@ -131,116 +130,115 @@ class Select: """ def error_handler(exc, filename): - Log("Error initializing file %s/%s" % (dsrpath.path, filename), 2) + Log("Error initializing file %s/%s" % (rpath.path, filename), 2) return None - def diryield(dsrpath): - """Generate relevant files in directory dsrpath + def diryield(rpath): + """Generate relevant files in directory rpath - Returns (dsrp, num) where num == 0 means dsrp should be - generated normally, num == 1 means the dsrp is a directory + Returns (rpath, num) where num == 0 means rpath should be + generated normally, num == 1 means the rpath is a directory and should be included iff something inside is included. """ - for filename in Robust.listrp(dsrpath): - new_dsrp = Robust.check_common_error(error_handler, - dsrpath.append, (filename,)) - if new_dsrp: - s = sel_func(new_dsrp) - if s == 1: yield (new_dsrp, 0) - elif s == 2 and new_dsrp.isdir(): yield (new_dsrp, 1) - - yield dsrpath - diryield_stack = [diryield(dsrpath)] - delayed_dsrp_stack = [] + for filename in Robust.listrp(rpath): + new_rpath = Robust.check_common_error(error_handler, + rpath.append, (filename,)) + if new_rpath: + s = sel_func(new_rpath) + if s == 1: yield (new_rpath, 0) + elif s == 2 and new_rpath.isdir(): yield (new_rpath, 1) + + yield rpath + diryield_stack = [diryield(rpath)] + delayed_rp_stack = [] while diryield_stack: - try: dsrp, val = diryield_stack[-1].next() + try: rpath, val = diryield_stack[-1].next() except StopIteration: diryield_stack.pop() - if delayed_dsrp_stack: delayed_dsrp_stack.pop() + if delayed_rp_stack: delayed_rp_stack.pop() continue if val == 0: - if delayed_dsrp_stack: - for delayed_dsrp in delayed_dsrp_stack: yield delayed_dsrp - del delayed_dsrp_stack[:] - yield dsrp - if dsrp.isdir(): diryield_stack.append(diryield(dsrp)) + if delayed_rp_stack: + for delayed_rp in delayed_rp_stack: yield delayed_rp + del delayed_rp_stack[:] + yield rpath + if rpath.isdir(): diryield_stack.append(diryield(rpath)) elif val == 1: - delayed_dsrp_stack.append(dsrp) - diryield_stack.append(diryield(dsrp)) + delayed_rp_stack.append(rpath) + diryield_stack.append(diryield(rpath)) - def Iterate(self, dsrpath, rec_func, sel_func): - """Return iterator yielding dsrps in dsrpath + def Iterate(self, rpath, rec_func, sel_func): + """Return iterator yielding rpaths in rpath rec_func is usually the same as this function and is what Iterate uses to find files in subdirectories. It is used in iterate_starting_from. - sel_func is the selection function to use on the dsrps. It is - usually self.Select. + sel_func is the selection function to use on the rpaths. It + is usually self.Select. """ - s = sel_func(dsrpath) + s = sel_func(rpath) if s == 0: return elif s == 1: # File is included - yield dsrpath - if dsrpath.isdir(): - for dsrp in self.iterate_in_dir(dsrpath, rec_func, sel_func): - yield dsrp + yield rpath + if rpath.isdir(): + for rp in self.iterate_in_dir(rpath, rec_func, sel_func): + yield rp elif s == 2: - if dsrpath.isdir(): # Directory is merely scanned - iid = self.iterate_in_dir(dsrpath, rec_func, sel_func) + if rpath.isdir(): # Directory is merely scanned + iid = self.iterate_in_dir(rpath, rec_func, sel_func) try: first = iid.next() - except StopIteration: return # no files inside; skip dsrp - yield dsrpath + except StopIteration: return # no files inside; skip rp + yield rpath yield first - for dsrp in iid: yield dsrp + for rp in iid: yield rp else: assert 0, "Invalid selection result %s" % (str(s),) - def iterate_in_dir(self, dsrpath, rec_func, sel_func): - """Iterate the dsrps in directory dsrpath.""" + def iterate_in_dir(self, rpath, rec_func, sel_func): + """Iterate the rpaths in directory rpath.""" def error_handler(exc, filename): - Log("Error initializing file %s/%s" % (dsrpath.path, filename), 2) + Log("Error initializing file %s/%s" % (rpath.path, filename), 2) return None if self.quoting_on: - for subdir in FilenameMapping.get_quoted_dir_children(dsrpath): - for dsrp in rec_func(subdir, rec_func, sel_func): - yield dsrp + for subdir in FilenameMapping.get_quoted_dir_children(rpath): + for rp in rec_func(subdir, rec_func, sel_func): + yield rp else: - for filename in Robust.listrp(dsrpath): - new_dsrp = Robust.check_common_error( - error_handler, dsrpath.append, [filename]) - if new_dsrp: - for dsrp in rec_func(new_dsrp, rec_func, sel_func): - yield dsrp - - def iterate_starting_from(self, dsrpath, rec_func, sel_func): + for filename in Robust.listrp(rpath): + new_rp = Robust.check_common_error( + error_handler, rpath.append, [filename]) + if new_rp: + for rp in rec_func(new_rp, rec_func, sel_func): + yield rp + + def iterate_starting_from(self, rpath, rec_func, sel_func): """Like Iterate, but only yield indicies > self.starting_index""" - if dsrpath.index > self.starting_index: # past starting_index - for dsrp in self.Iterate(dsrpath, self.Iterate, sel_func): - yield dsrp - elif (dsrpath.index == self.starting_index[:len(dsrpath.index)] - and dsrpath.isdir()): + if rpath.index > self.starting_index: # past starting_index + for rp in self.Iterate(rpath, self.Iterate, sel_func): + yield rp + elif (rpath.index == self.starting_index[:len(rpath.index)] + and rpath.isdir()): # May encounter starting index on this branch - if self.iterate_parents: yield dsrpath - for dsrp in self.iterate_in_dir(dsrpath, - self.iterate_starting_from, - sel_func): yield dsrp - - def iterate_with_finalizer(self): - """Like Iterate, but missing some options, and add finalizer""" - finalize = IterTreeReducer(DestructiveSteppingFinalizer, ()) - for dsrp in self: - yield dsrp - finalize(dsrp.index, dsrp) - finalize.Finish() - - def Select(self, dsrp): + if self.iterate_parents: yield rpath + for rp in self.iterate_in_dir(rpath, self.iterate_starting_from, + sel_func): yield rp + +# def iterate_with_finalizer(self): +# """Like Iterate, but missing some options, and add finalizer""" +# finalize = IterTreeReducer(DestructiveSteppingFinalizer, ()) +# for rp in self: +# yield rp +# finalize(rp.index, rp)) +# finalize.Finish() + + def Select(self, rp): """Run through the selection functions and return dominant val 0/1/2""" for sf in self.selection_functions: - result = sf(dsrp) + result = sf(rp) if result is not None: return result return 1 @@ -353,11 +351,11 @@ probably isn't what you meant.""" % tuple_list.sort() i = [0] # We have to put index in list because of stupid scoping rules - def selection_function(dsrp): + def selection_function(rp): while 1: if i[0] >= len(tuple_list): return None include, move_on = \ - self.filelist_pair_match(dsrp, tuple_list[i[0]]) + self.filelist_pair_match(rp, tuple_list[i[0]]) if move_on: i[0] += 1 if include is None: continue # later line may match @@ -415,12 +413,12 @@ probably isn't what you meant.""" % index = tuple(filter(lambda x: x, line.split("/"))) # remove empties return (index, include) - def filelist_pair_match(self, dsrp, pair): - """Matches a filelist tuple against a dsrp + def filelist_pair_match(self, rp, pair): + """Matches a filelist tuple against a rpath Returns a pair (include, move_on). include is None if the tuple doesn't match either way, and 0/1 if the tuple excludes - or includes the dsrp. + or includes the rpath. move_on is true if the tuple cannot match a later index, and so we should move on to the next tuple in the index. @@ -428,16 +426,16 @@ probably isn't what you meant.""" % """ index, include = pair if include == 1: - if index < dsrp.index: return (None, 1) - if index == dsrp.index: return (1, 1) - elif index[:len(dsrp.index)] == dsrp.index: + if index < rp.index: return (None, 1) + if index == rp.index: return (1, 1) + elif index[:len(rp.index)] == rp.index: return (1, None) # /foo/bar implicitly includes /foo - else: return (None, None) # dsrp greater, not initial sequence + else: return (None, None) # rp greater, not initial sequence elif include == 0: - if dsrp.index[:len(index)] == index: + if rp.index[:len(index)] == index: return (0, None) # /foo implicitly excludes /foo/bar - elif index < dsrp.index: return (None, 1) - else: return (None, None) # dsrp greater, not initial sequence + elif index < rp.index: return (None, 1) + else: return (None, None) # rp greater, not initial sequence else: assert 0, "Include is %s, should be 0 or 1" % (include,) def filelist_globbing_get_sfs(self, filelist_fp, inc_default, list_name): @@ -460,9 +458,9 @@ probably isn't what you meant.""" % def other_filesystems_get_sf(self, include): """Return selection function matching files on other filesystems""" assert include == 0 or include == 1 - root_devloc = self.dsrpath.getdevloc() - def sel_func(dsrp): - if dsrp.getdevloc() == root_devloc: return None + root_devloc = self.rpath.getdevloc() + def sel_func(rp): + if rp.getdevloc() == root_devloc: return None else: return include sel_func.exclude = not include sel_func.name = "Match other filesystems" @@ -476,8 +474,8 @@ probably isn't what you meant.""" % Log("Error compiling regular expression %s" % regexp_string, 1) raise - def sel_func(dsrp): - if regexp.search(dsrp.path): return include + def sel_func(rp): + if regexp.search(rp.path): return include else: return None sel_func.exclude = not include @@ -489,8 +487,8 @@ probably isn't what you meant.""" % if self.selection_functions: Log("Warning: exclude-device-files is not the first " "selector.\nThis may not be what you intended", 3) - def sel_func(dsrp): - if dsrp.isdev(): return include + def sel_func(rp): + if rp.isdev(): return include else: return None sel_func.exclude = not include sel_func.name = (include and "include" or "exclude") + " device files" @@ -501,8 +499,8 @@ probably isn't what you meant.""" % if self.selection_functions: Log("Warning: exclude-special-files is not the first " "selector.\nThis may not be what you intended", 3) - def sel_func(dsrp): - if dsrp.issym() or dsrp.issock() or dsrp.isfifo() or dsrp.isdev(): + def sel_func(rp): + if rp.issym() or rp.issock() or rp.isfifo() or rp.isdev(): return include else: return None sel_func.exclude = not include @@ -512,7 +510,7 @@ probably isn't what you meant.""" % def glob_get_sf(self, glob_str, include): """Return selection function given by glob string""" assert include == 0 or include == 1 - if glob_str == "**": sel_func = lambda dsrp: include + if glob_str == "**": sel_func = lambda rp: include elif not self.glob_re.match(glob_str): # normal file sel_func = self.glob_get_filename_sf(glob_str, include) else: sel_func = self.glob_get_normal_sf(glob_str, include) @@ -539,14 +537,14 @@ probably isn't what you meant.""" % def glob_get_tuple_sf(self, tuple, include): """Return selection function based on tuple""" - def include_sel_func(dsrp): - if (dsrp.index == tuple[:len(dsrp.index)] or - dsrp.index[:len(tuple)] == tuple): + def include_sel_func(rp): + if (rp.index == tuple[:len(rp.index)] or + rp.index[:len(tuple)] == tuple): return 1 # /foo/bar implicitly matches /foo, vice-versa else: return None - def exclude_sel_func(dsrp): - if dsrp.index[:len(tuple)] == tuple: + def exclude_sel_func(rp): + if rp.index[:len(tuple)] == tuple: return 0 # /foo excludes /foo/bar, not vice-versa else: return None @@ -585,17 +583,17 @@ probably isn't what you meant.""" % scan_comp_re = re_comp("^(%s)$" % "|".join(self.glob_get_prefix_res(glob_str))) - def include_sel_func(dsrp): - if glob_comp_re.match(dsrp.path): return 1 - elif scan_comp_re.match(dsrp.path): return 2 + def include_sel_func(rp): + if glob_comp_re.match(rp.path): return 1 + elif scan_comp_re.match(rp.path): return 2 else: return None - def exclude_sel_func(dsrp): - if glob_comp_re.match(dsrp.path): return 0 + def exclude_sel_func(rp): + if glob_comp_re.match(rp.path): return 0 else: return None # Check to make sure prefix is ok - if not include_sel_func(self.dsrpath): raise FilePrefixError(glob_str) + if not include_sel_func(self.rpath): raise FilePrefixError(glob_str) if include: return include_sel_func else: return exclude_sel_func |