diff options
49 files changed, 2269 insertions, 2274 deletions
diff --git a/CHANGES.txt b/CHANGES.txt index 6e68a86..87763cd 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -53,7 +53,7 @@ * OSFSWatchMixin improvements: * watch_inotify: allow more than one watcher on a single path. * watch_win32: don't create immortal reference cycles. - * watch_win32: report errors if the filesystem does't support + * watch_win32: report errors if the filesystem doesn't support ReadDirectoryChangesW. * MountFS: added support for mounting at the root directory, and for mounting over an existing mount. @@ -76,7 +76,7 @@ * Added a concept of a writeable FS to MultiFS * Added ilistdir() and ilistdirinfo() methods, which are generator-based variants of listdir() and listdirinfo(). - * Removed obsolete module fs.objectree; use fs.path.PathMap instead. + * Removed obsolete module fs.objecttree; use fs.path.PathMap instead. * Added setcontents_async method to base * Added `appdirfs` module to abstract per-user application directories diff --git a/MANIFEST.in b/MANIFEST.in index 2720cd5..c7a8b83 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include AUTHORS -include README.txt +include README.md include LICENSE.txt include CHANGES.txt
\ No newline at end of file @@ -5,7 +5,7 @@ PyFilesystem is an abstraction layer for *filesystems*. In the same way that Pyt Pyfilesystem works with Linux, Windows and Mac. -Suported Filesystems +Supported Filesystems --------------------- Here are a few of the filesystems that can be accessed with Pyfilesystem: @@ -16,7 +16,7 @@ Here are a few of the filesystems that can be accessed with Pyfilesystem: * **MountFS** creates a virtual directory structure built from other filesystems * **MultiFS** a virtual filesystem that combines a list of filesystems into one, and checks them in order when opening files * **OSFS** the native filesystem -* **SFTPFS** access files & directores stored on a Secure FTP server +* **SFTPFS** access files & directories stored on a Secure FTP server * **S3FS** access files & directories stored on Amazon S3 storage * **TahoeLAFS** access files & directories stored on a Tahoe distributed filesystem * **ZipFS** access files and directories contained in a zip file @@ -49,6 +49,11 @@ An alternative to explicitly importing the filesystem class you want, is to use You could change ``C:/projects`` to ``zip://source.zip`` to open the zip file, or even ``ftp://ftp.example.org/code/projects/`` to sum up the bytes of Python stored on an ftp server. +Documentation +------------- + +http://docs.pyfilesystem.org + Screencast ---------- diff --git a/docs/commands.rst b/docs/commands.rst index 6ca1e84..9209ab8 100644 --- a/docs/commands.rst +++ b/docs/commands.rst @@ -42,7 +42,7 @@ Lists the contents of a directory, similar to the ``ls`` command, e.g.:: fstree ------ -Displays an ASCII tree of a directory. e.g:: +Displays an ASCII tree of a directory. e.g.:: fstree fstree -g @@ -52,7 +52,7 @@ Displays an ASCII tree of a directory. e.g:: fscat ----- -Writes a file to stdout, e.g:: +Writes a file to stdout, e.g.:: fscat ~/.bashrc fscat http://www.willmcgugan.com @@ -61,7 +61,7 @@ Writes a file to stdout, e.g:: fsinfo ------ -Displays information regarding a file / directory, e.g:: +Displays information regarding a file / directory, e.g.:: fsinfo C:\autoexec.bat fsinfo ftp://ftp.mozilla.org/pub/README @@ -69,7 +69,7 @@ Displays information regarding a file / directory, e.g:: fsmv ---- -Moves a file from one location to another, e.g:: +Moves a file from one location to another, e.g.:: fsmv foo bar fsmv *.jpg zip://photos.zip @@ -77,7 +77,7 @@ Moves a file from one location to another, e.g:: fsmkdir ------- -Makes a directory on a filesystem, e.g:: +Makes a directory on a filesystem, e.g.:: fsmkdir foo fsmkdir ftp://ftp.mozilla.org/foo @@ -86,7 +86,7 @@ Makes a directory on a filesystem, e.g:: fscp ---- -Copies a file from one location to another, e.g:: +Copies a file from one location to another, e.g.:: fscp foo bar fscp ftp://ftp.mozilla.org/pub/README readme.txt @@ -94,7 +94,7 @@ Copies a file from one location to another, e.g:: fsrm ---- -Removes (deletes) a file from a filesystem, e.g:: +Removes (deletes) a file from a filesystem, e.g.:: fsrm foo fsrm -r mydir @@ -102,7 +102,7 @@ Removes (deletes) a file from a filesystem, e.g:: fsserve ------- -Serves the contents of a filesystem over a network with one of a number of methods; HTTP, RPC or SFTP, e.g:: +Serves the contents of a filesystem over a network with one of a number of methods; HTTP, RPC or SFTP, e.g.:: fsserve fsserve --type rpc @@ -111,8 +111,8 @@ Serves the contents of a filesystem over a network with one of a number of metho fsmount ------- -Mounts a filesystem with FUSE (on Linux) and Dokan (on Windows), e.g:: +Mounts a filesystem with FUSE (on Linux) and Dokan (on Windows), e.g.:: fsmount mem:// ram fsserve mem:// M - fsserve ftp://ftp.mozilla.org/pub ftpgateway
\ No newline at end of file + fsserve ftp://ftp.mozilla.org/pub ftpgateway diff --git a/docs/concepts.rst b/docs/concepts.rst index 313ed04..dceaacd 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -45,7 +45,7 @@ PyFilesystem will catch any attempts to read outside of the root directory. For Paths ----- -Paths used within an FS object use the same common format, regardless of the underlaying file system it represents (or the platform it resides on). +Paths used within an FS object use the same common format, regardless of the underlying file system it represents (or the platform it resides on). When working with paths in FS objects, keep in mind the following: diff --git a/docs/getting_started.rst b/docs/getting_started.rst index df2eff3..e05a13e 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -14,10 +14,10 @@ Or to upgrade to the most recent version:: pip install fs --upgrade -You can also install the cutting edge release by checking out the source via SVN:: +You can also install the cutting edge release by cloning the source from GIT:: - svn checkout http://pyfilesystem.googlecode.com/svn/trunk/ pyfilesystem-read-only - cd pyfilesystem-read-only + git clone https://github.com/PyFilesystem/pyfilesystem.git + cd pyfilesystem python setup.py install Whichever method you use, you should now have the `fs` module on your path (version number may vary):: @@ -41,8 +41,8 @@ Prerequisites PyFilesystem requires at least **Python 2.6**. There are a few other dependencies if you want to use some of the more advanced filesystem interfaces, but for basic use all that is needed is the Python standard library. - * Boto (required for :mod:`fs.s3fs`) http://code.google.com/p/boto/ - * Paramiko (required for :class:`fs.ftpfs.FTPFS`) http://www.lag.net/paramiko/ + * Boto (required for :mod:`fs.s3fs`) https://github.com/boto/boto + * Paramiko (required for :mod:`fs.sftpfs`) https://github.com/paramiko/paramiko * wxPython (required for :mod:`fs.browsewin`) http://www.wxpython.org/ diff --git a/docs/implementersguide.rst b/docs/implementersguide.rst index ceac3a8..debdf71 100644 --- a/docs/implementersguide.rst +++ b/docs/implementersguide.rst @@ -62,7 +62,7 @@ Thread Safety All PyFilesystem methods, other than the constructor, should be thread-safe where-ever possible. One way to do this is to pass ``threads_synchronize=True`` to the base constructor and use the :func:`~fs.base.synchronize` decorator to lock the FS object when a method is called. -If the implementation can not be made thread-safe for technical reasons, ensure that ``getmeta("thread_safe")`` returns ``False``. +If the implementation cannot be made thread-safe for technical reasons, ensure that ``getmeta("thread_safe")`` returns ``False``. Meta Values diff --git a/docs/introduction.rst b/docs/introduction.rst index 6d6c665..affa880 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -29,7 +29,7 @@ If you have any problems or questions, please contact the developers through one Bugs #### -If you find a bug in PyFilesystem, please file an issue: http://code.google.com/p/pyfilesystem/issues/list +If you find a bug in PyFilesystem, please file an issue: https://github.com/PyFilesystem/pyfilesystem/issues Discussion Group ################ diff --git a/fs/browsewin.py b/fs/browsewin.py index 9ce44fe..32f9b9c 100644 --- a/fs/browsewin.py +++ b/fs/browsewin.py @@ -37,7 +37,7 @@ class InfoFrame(wx.Frame): for key in sorted(keys, key=lambda k:k.lower()): self.list_ctrl.Append((key, unicode(info.get(key)))) - + self.Center() @@ -109,10 +109,10 @@ class BrowseFrame(wx.Frame): msg = "Failed to get directory listing for %s\n\nThe following error was reported:\n\n%s" % (path, e) wx.MessageDialog(self, msg, "Error listing directory", wx.OK).ShowModal() paths = [] - - + + #paths = [(self.fs.isdir(p), p) for p in self.fs.listdir(path, absolute=True)] - + if self.hide_dotfiles: paths = [p for p in paths if not isdotfile(p[1])] @@ -173,7 +173,7 @@ class BrowseFrame(wx.Frame): path = item_data["path"] info = self.fs.getinfo(path) - info_frame = InfoFrame(self, path, self.fs.desc(path), info) + info_frame = InfoFrame(self, path, self.fs.desc(path), info) info_frame.Show() info_frame.CenterOnParent() diff --git a/fs/commands/fscat.py b/fs/commands/fscat.py index 2849ce1..077a0e2 100644 --- a/fs/commands/fscat.py +++ b/fs/commands/fscat.py @@ -3,26 +3,26 @@ from fs.commands.runner import Command import sys class FSCat(Command): - + usage = """fscat [OPTION]... [FILE]... Concetanate FILE(s)""" version = "1.0" - + def do_run(self, options, args): - count = 0 - for fs, path, is_dir in self.get_resources(args): + count = 0 + for fs, path, is_dir in self.get_resources(args): if is_dir: self.error('%s is a directory\n' % path) - return 1 + return 1 self.output(fs.getcontents(path)) - count += 1 + count += 1 if self.is_terminal() and count: self.output('\n') - + def run(): return FSCat().run() - + if __name__ == "__main__": sys.exit(run()) -
\ No newline at end of file + diff --git a/fs/commands/fscp.py b/fs/commands/fscp.py index 58d6c1a..6d61c46 100644 --- a/fs/commands/fscp.py +++ b/fs/commands/fscp.py @@ -8,52 +8,52 @@ import time import threading -class FileOpThread(threading.Thread): - +class FileOpThread(threading.Thread): + def __init__(self, action, name, dest_fs, queue, on_done, on_error): - self.action = action + self.action = action self.dest_fs = dest_fs self.queue = queue self.on_done = on_done self.on_error = on_error self.finish_event = threading.Event() - super(FileOpThread, self).__init__() - - def run(self): - - while not self.finish_event.isSet(): + super(FileOpThread, self).__init__() + + def run(self): + + while not self.finish_event.isSet(): try: path_type, fs, path, dest_path = self.queue.get(timeout=0.1) except queue.Empty: continue - try: - if path_type == FScp.DIR: + try: + if path_type == FScp.DIR: self.dest_fs.makedir(path, recursive=True, allow_recreate=True) - else: - self.action(fs, path, self.dest_fs, dest_path, overwrite=True) - except Exception, e: - self.on_error(e) - self.queue.task_done() - break + else: + self.action(fs, path, self.dest_fs, dest_path, overwrite=True) + except Exception, e: + self.on_error(e) + self.queue.task_done() + break else: - self.queue.task_done() + self.queue.task_done() self.on_done(path_type, fs, path, self.dest_fs, dest_path) - - + + class FScp(Command): - + DIR, FILE = 0, 1 - + usage = """fscp [OPTION]... [SOURCE]... [DESTINATION] Copy SOURCE to DESTINATION""" - + def get_action(self): if self.options.threads > 1: return copyfile_non_atomic else: return copyfile - + def get_verb(self): return 'copying...' @@ -62,40 +62,40 @@ Copy SOURCE to DESTINATION""" optparse.add_option('-p', '--progress', dest='progress', action="store_true", default=False, help="show progress", metavar="PROGRESS") optparse.add_option('-t', '--threads', dest='threads', action="store", default=1, - help="number of threads to use", type="int", metavar="THREAD_COUNT") + help="number of threads to use", type="int", metavar="THREAD_COUNT") return optparse - + def do_run(self, options, args): - - self.options = options + + self.options = options if len(args) < 2: self.error("at least two filesystems required\n") return 1 - + srcs = args[:-1] - dst = args[-1] - + dst = args[-1] + dst_fs, dst_path = self.open_fs(dst, writeable=True, create_dir=True) - + if dst_path is not None and dst_fs.isfile(dst_path): self.error('Destination must be a directory\n') return 1 - + if dst_path: dst_fs = dst_fs.makeopendir(dst_path) - dst_path = None - + dst_path = None + copy_fs_paths = [] - - progress = options.progress - + + progress = options.progress + if progress: sys.stdout.write(self.progress_bar(len(srcs), 0, 'scanning...')) sys.stdout.flush() - - self.root_dirs = [] + + self.root_dirs = [] for i, fs_url in enumerate(srcs): - src_fs, src_path = self.open_fs(fs_url) + src_fs, src_path = self.open_fs(fs_url) if src_path is None: src_path = '/' @@ -103,44 +103,44 @@ Copy SOURCE to DESTINATION""" if iswildcard(src_path): for file_path in src_fs.listdir(wildcard=src_path, full=True): copy_fs_paths.append((self.FILE, src_fs, file_path, file_path)) - - else: - if src_fs.isdir(src_path): - self.root_dirs.append((src_fs, src_path)) + + else: + if src_fs.isdir(src_path): + self.root_dirs.append((src_fs, src_path)) src_sub_fs = src_fs.opendir(src_path) for dir_path, file_paths in src_sub_fs.walk(): - if dir_path not in ('', '/'): + if dir_path not in ('', '/'): copy_fs_paths.append((self.DIR, src_sub_fs, dir_path, dir_path)) sub_fs = src_sub_fs.opendir(dir_path) - for file_path in file_paths: + for file_path in file_paths: copy_fs_paths.append((self.FILE, sub_fs, file_path, pathjoin(dir_path, file_path))) else: if src_fs.exists(src_path): copy_fs_paths.append((self.FILE, src_fs, src_path, src_path)) else: self.error('%s is not a file or directory\n' % src_path) - return 1 - + return 1 + if progress: sys.stdout.write(self.progress_bar(len(srcs), i + 1, 'scanning...')) sys.stdout.flush() - + if progress: sys.stdout.write(self.progress_bar(len(copy_fs_paths), 0, self.get_verb())) sys.stdout.flush() - - if self.options.threads > 1: + + if self.options.threads > 1: copy_fs_dirs = [r for r in copy_fs_paths if r[0] == self.DIR] - copy_fs_paths = [r for r in copy_fs_paths if r[0] == self.FILE] - for path_type, fs, path, dest_path in copy_fs_dirs: - dst_fs.makedir(path, allow_recreate=True, recursive=True) - + copy_fs_paths = [r for r in copy_fs_paths if r[0] == self.FILE] + for path_type, fs, path, dest_path in copy_fs_dirs: + dst_fs.makedir(path, allow_recreate=True, recursive=True) + self.lock = threading.RLock() - + self.total_files = len(copy_fs_paths) self.done_files = 0 - - file_queue = queue.Queue() + + file_queue = queue.Queue() threads = [FileOpThread(self.get_action(), 'T%i' % i, dst_fs, @@ -148,59 +148,59 @@ Copy SOURCE to DESTINATION""" self.on_done, self.on_error) for i in xrange(options.threads)] - + for thread in threads: thread.start() - + self.action_errors = [] complete = False - try: - enqueue = file_queue.put + try: + enqueue = file_queue.put for resource in copy_fs_paths: enqueue(resource) - + while not file_queue.empty(): time.sleep(0) if self.any_error(): raise SystemExit # Can't use queue.join here, or KeyboardInterrupt will not be - # caught until the queue is finished + # caught until the queue is finished #file_queue.join() - - except KeyboardInterrupt: - options.progress = False + + except KeyboardInterrupt: + options.progress = False self.output("\nCancelling...\n") - + except SystemExit: - options.progress = False - + options.progress = False + finally: - sys.stdout.flush() + sys.stdout.flush() for thread in threads: - thread.finish_event.set() + thread.finish_event.set() for thread in threads: thread.join() complete = True if not self.any_error(): self.post_actions() - + dst_fs.close() - + if self.action_errors: for error in self.action_errors: - self.error(self.wrap_error(unicode(error)) + '\n') + self.error(self.wrap_error(unicode(error)) + '\n') sys.stdout.flush() else: if complete and options.progress: sys.stdout.write(self.progress_bar(self.total_files, self.done_files, '')) sys.stdout.write('\n') sys.stdout.flush() - + def post_actions(self): pass - - def on_done(self, path_type, src_fs, src_path, dst_fs, dst_path): - self.lock.acquire() + + def on_done(self, path_type, src_fs, src_path, dst_fs, dst_path): + self.lock.acquire() try: if self.options.verbose: if path_type == self.DIR: @@ -208,44 +208,44 @@ Copy SOURCE to DESTINATION""" else: print "%s -> %s" % (src_fs.desc(src_path), dst_fs.desc(dst_path)) elif self.options.progress: - self.done_files += 1 + self.done_files += 1 sys.stdout.write(self.progress_bar(self.total_files, self.done_files, self.get_verb())) sys.stdout.flush() finally: self.lock.release() - - def on_error(self, e): + + def on_error(self, e): self.lock.acquire() try: self.action_errors.append(e) finally: self.lock.release() - - def any_error(self): + + def any_error(self): self.lock.acquire() try: return bool(self.action_errors) finally: self.lock.release() - + def progress_bar(self, total, remaining, msg=''): bar_width = 20 throbber = '|/-\\' throb = throbber[remaining % len(throbber)] done = float(remaining) / total - + done_steps = int(done * bar_width) - bar_steps = ('#' * done_steps).ljust(bar_width) - - msg = '%s %i%%' % (msg, int(done * 100.0)) + bar_steps = ('#' * done_steps).ljust(bar_width) + + msg = '%s %i%%' % (msg, int(done * 100.0)) msg = msg.ljust(20) - + if total == remaining: throb = '' - + bar = '\r%s[%s] %s\r' % (throb, bar_steps, msg.lstrip()) return bar - + def run(): return FScp().run() diff --git a/fs/commands/fsinfo.py b/fs/commands/fsinfo.py index 943a28f..5cdbbac 100644 --- a/fs/commands/fsinfo.py +++ b/fs/commands/fsinfo.py @@ -4,10 +4,10 @@ import sys from datetime import datetime class FSInfo(Command): - + usage = """fsinfo [OPTION]... [PATH] Display information regarding an FS resource""" - + def get_optparse(self): optparse = super(FSInfo, self).get_optparse() optparse.add_option('-k', '--key', dest='keys', action='append', default=[], @@ -20,12 +20,12 @@ Display information regarding an FS resource""" help="list directories only", metavar="DIRSONLY") optparse.add_option('-f', '--filesonly', dest='filesonly', action="store_true", default=False, help="list files only", metavar="FILESONLY") - return optparse - - + return optparse + + def do_run(self, options, args): - - def wrap_value(val): + + def wrap_value(val): if val.rstrip() == '\0': return self.wrap_error('... missing ...') return val @@ -40,41 +40,41 @@ Display information regarding an FS resource""" except: text = repr(text) return text - - + + keys = options.keys or None for fs, path, is_dir in self.get_resources(args, files_only=options.filesonly, - dirs_only=options.dirsonly): + dirs_only=options.dirsonly): if not options.omit: - if options.simple: + if options.simple: file_line = u'%s\n' % self.wrap_filename(path) else: file_line = u'[%s] %s\n' % (self.wrap_filename(path), self.wrap_faded(fs.desc(path))) - self.output(file_line) + self.output(file_line) info = fs.getinfo(path) - + for k, v in info.items(): if k.startswith('_'): del info[k] elif not isinstance(v, (basestring, int, long, float, bool, datetime)): - del info[k] - - if keys: + del info[k] + + if keys: table = [(k, make_printable(info.get(k, '\0'))) for k in keys] else: keys = sorted(info.keys()) table = [(k, make_printable(info[k])) for k in sorted(info.keys())] - + if options.simple: for row in table: self.output(row[-1] + '\n') - else: - self.output_table(table, {0:self.wrap_table_header, 1:wrap_value}) + else: + self.output_table(table, {0:self.wrap_table_header, 1:wrap_value}) + - def run(): - return FSInfo().run() - + return FSInfo().run() + if __name__ == "__main__": - sys.exit(run()) + sys.exit(run()) diff --git a/fs/commands/fsmkdir.py b/fs/commands/fsmkdir.py index ad4324e..73c0a02 100644 --- a/fs/commands/fsmkdir.py +++ b/fs/commands/fsmkdir.py @@ -3,20 +3,20 @@ from fs.commands.runner import Command import sys class FSMkdir(Command): - + usage = """fsmkdir [PATH] Make a directory""" version = "1.0" - + def do_run(self, options, args): - - for fs_url in args: - self.open_fs(fs_url, create_dir=True) - + + for fs_url in args: + self.open_fs(fs_url, create_dir=True) + def run(): return FSMkdir().run() - + if __name__ == "__main__": sys.exit(run()) -
\ No newline at end of file + diff --git a/fs/commands/fsmv.py b/fs/commands/fsmv.py index 9e96de6..5867a4c 100644 --- a/fs/commands/fsmv.py +++ b/fs/commands/fsmv.py @@ -5,26 +5,26 @@ from fs.commands import fscp import sys class FSmv(fscp.FScp): - + usage = """fsmv [OPTION]... [SOURCE] [DESTINATION] Move files from SOURCE to DESTINATION""" - + def get_verb(self): return 'moving...' - - def get_action(self): - if self.options.threads > 1: + + def get_action(self): + if self.options.threads > 1: return movefile_non_atomic else: return movefile - + def post_actions(self): for fs, dirpath in self.root_dirs: - if not contains_files(fs, dirpath): + if not contains_files(fs, dirpath): fs.removedir(dirpath, force=True) - + def run(): return FSmv().run() - + if __name__ == "__main__": sys.exit(run()) diff --git a/fs/commands/fsrm.py b/fs/commands/fsrm.py index 5e2bd09..8985e64 100644 --- a/fs/commands/fsrm.py +++ b/fs/commands/fsrm.py @@ -6,10 +6,10 @@ from fs.commands.runner import Command import sys class FSrm(Command): - + usage = """fsrm [OPTION]... [PATH] Remove a file or directory at PATH""" - + def get_optparse(self): optparse = super(FSrm, self).get_optparse() optparse.add_option('-f', '--force', dest='force', action='store_true', default=False, @@ -19,13 +19,13 @@ Remove a file or directory at PATH""" optparse.add_option('-r', '--recursive', dest='recursive', action='store_true', default=False, help='remove directories and their contents recursively') return optparse - + def do_run(self, options, args): - + interactive = options.interactive verbose = options.verbose - - for fs, path, is_dir in self.get_resources(args): + + for fs, path, is_dir in self.get_resources(args): if interactive: if is_dir: msg = "remove directory '%s'?" % path @@ -44,11 +44,11 @@ Remove a file or directory at PATH""" else: if verbose: self.output("removed '%s'\n" % path) - - -def run(): + + +def run(): return FSrm().run() - + if __name__ == "__main__": - sys.exit(run()) -
\ No newline at end of file + sys.exit(run()) + diff --git a/fs/commands/fstree.py b/fs/commands/fstree.py index 3d2a7b9..bbb88b5 100644 --- a/fs/commands/fstree.py +++ b/fs/commands/fstree.py @@ -7,18 +7,18 @@ from fs.commands.runner import Command from fs.utils import print_fs class FSTree(Command): - + usage = """fstree [OPTION]... [PATH] Recursively display the contents of PATH in an ascii tree""" - + def get_optparse(self): - optparse = super(FSTree, self).get_optparse() + optparse = super(FSTree, self).get_optparse() optparse.add_option('-l', '--level', dest='depth', type="int", default=5, help="Descend only LEVEL directories deep (-1 for infinite)", metavar="LEVEL") optparse.add_option('-g', '--gui', dest='gui', action='store_true', default=False, help="browse the tree with a gui") optparse.add_option('-a', '--all', dest='all', action='store_true', default=False, - help="do not hide dot files") + help="do not hide dot files") optparse.add_option('--dirsfirst', dest='dirsfirst', action='store_true', default=False, help="List directories before files") optparse.add_option('-P', dest="pattern", default=None, @@ -26,13 +26,13 @@ Recursively display the contents of PATH in an ascii tree""" optparse.add_option('-d', dest="dirsonly", default=False, action='store_true', help="List directories only") return optparse - - def do_run(self, options, args): - + + def do_run(self, options, args): + if not args: args = ['.'] - - for fs, path, is_dir in self.get_resources(args, single=True): + + for fs, path, is_dir in self.get_resources(args, single=True): if not is_dir: self.error(u"'%s' is not a dir\n" % path) return 1 @@ -62,13 +62,13 @@ Recursively display the contents of PATH in an ascii tree""" return '%i %s' % (count, one) else: return '%i %s' % (count, many) - + self.output("%s, %s\n" % (pluralize('directory', 'directories', dircount), pluralize('file', 'files', filecount))) - + def run(): - return FSTree().run() - + return FSTree().run() + if __name__ == "__main__": sys.exit(run()) - + diff --git a/fs/contrib/bigfs/__init__.py b/fs/contrib/bigfs/__init__.py index 27d05be..fe47350 100644 --- a/fs/contrib/bigfs/__init__.py +++ b/fs/contrib/bigfs/__init__.py @@ -2,7 +2,7 @@ fs.contrib.bigfs ================ -A FS object that represents the contents of a BIG file +A FS object that represents the contents of a BIG file (C&C Generals, BfME C&C3, C&C Red Alert 3, C&C4 file format) Written by Koen van de Sande @@ -34,14 +34,14 @@ class BIGEntry: return f else: return self.decompress(f, wrapAsFile=True) - + def getcontents(self, baseFile): f = SubrangeFile(baseFile, self.offset, self.storedSize) if not self.isCompressed: return f.read() else: return self.decompress(f, wrapAsFile=False) - + def decompress(self, g, wrapAsFile=True): buf = g.read(2) magic = unpack(">H", buf)[0] @@ -55,7 +55,7 @@ class BIGEntry: outputSize = unpack(">I", "\0" + g.read(3))[0] if magic & 0x100: unknown1 = unpack(">I", "\0" + g.read(3))[0] - + output = [] while True: opcode = unpack("B", g.read(1))[0] @@ -63,36 +63,36 @@ class BIGEntry: # read second opcode opcode2 = unpack("B", g.read(1))[0] #print "0x80", toBits(opcode), toBits(opcode2), opcode & 0x03, (((opcode & 0x60) << 3) | opcode2) + Q, ((opcode & 0x1C) >> 2) + 2 + R - + # copy at most 3 bytes to output stream (lowest 2 bits of opcode) count = opcode & 0x03 for i in range(count): output.append(g.read(1)) - + # you always have to look at least one byte, hence the +1 # use bit6 and bit5 (bit7=0 to trigger the if-statement) of opcode, and 8 bits of opcode2 (10-bits) lookback = (((opcode & 0x60) << 3) | opcode2) + 1 - + # use bit4..2 of opcode count = ((opcode & 0x1C) >> 2) + 3 - + for i in range(count): output.append(output[-lookback]) elif not (opcode & 0x40): # opcode: bit7..6==10 to get here opcode2 = unpack("B", g.read(1))[0] opcode3 = unpack("B", g.read(1))[0] #print "0x40", toBits(opcode), toBits(opcode2), toBits(opcode3) - + # copy count bytes (upper 2 bits of opcode2) count = opcode2 >> 6 for i in range(count): output.append(g.read(1)) - + # look back again (lower 6 bits of opcode2, all 8 bits of opcode3, total 14-bits) lookback = (((opcode2 & 0x3F) << 8) | opcode3) + 1 # lower 6 bits of opcode are the count to copy count = (opcode & 0x3F) + 4 - + for i in range(count): output.append(output[-lookback]) elif not (opcode & 0x20): # opcode: bit7..5=110 to get here @@ -104,7 +104,7 @@ class BIGEntry: count = opcode & 0x03 for i in range(count): output.append(g.read(1)) - + # look back: bit4 of opcode, all bits of opcode2 and opcode3, total 17-bits lookback = (((opcode & 0x10) >> 4) << 16) | (opcode2 << 8) | (opcode3) + 1 # bit3..2 of opcode and the whole of opcode4 @@ -129,15 +129,15 @@ class BIGEntry: for i in range(count): output.append(g.read(1)) #print "0xLO", toBits(opcode), count - + if wrapAsFile: return StringIO("".join(output)) else: return "".join(output) - + def __str__(self): return "<BIGEntry %s offset=%d storedSize=%d isCompressed=%s realSize=%d in %s" % (self.filename, self.offset, self.storedSize, str(self.isCompressed), self.realSize, self.filenameBIG) - + class _ExceptionProxy(object): @@ -156,12 +156,12 @@ class _ExceptionProxy(object): class BigFS(FS): """A FileSystem that represents a BIG file.""" - + _meta = { 'virtual' : False, 'read_only' : True, 'unicode_paths' : True, 'case_insensitive_paths' : False, - 'network' : False, + 'network' : False, } def __init__(self, filename, mode="r", thread_synchronize=True): @@ -260,7 +260,7 @@ class BigFS(FS): @synchronize def open(self, path, mode="r", **kwargs): - path = normpath(relpath(path)) + path = normpath(relpath(path)) if 'r' in mode: if self.file_mode not in 'ra': diff --git a/fs/contrib/bigfs/subrangefile.py b/fs/contrib/bigfs/subrangefile.py index e0ca1cb..3a99c55 100644 --- a/fs/contrib/bigfs/subrangefile.py +++ b/fs/contrib/bigfs/subrangefile.py @@ -28,7 +28,7 @@ class SubrangeFile: self.startOffset = startOffset self.fileSize = fileSize self.seek(0) - + def __str__(self): return "<SubrangeFile: %s@%d size=%d>" % (self.name, self.startOffset, self.fileSize) @@ -48,7 +48,7 @@ class SubrangeFile: offset = 0 offset = self.startOffset + self.fileSize + offset self.f.seek(offset) - + def tell(self): return self.f.tell() - self.startOffset @@ -60,7 +60,7 @@ class SubrangeFile: if self.tell() + iSize > self.fileSize: iSize = self.fileSize - self.tell() return iSize - + def readline(self,size=None): toRead = self.__maxSize(size) return self.f.readline(toRead) diff --git a/fs/contrib/davfs/util.py b/fs/contrib/davfs/util.py index 0fd587a..1dd9e7d 100644 --- a/fs/contrib/davfs/util.py +++ b/fs/contrib/davfs/util.py @@ -42,7 +42,7 @@ def get_filesize(file): raise AttributeError return file.size - + def file_chunks(f,chunk_size=1024*64): """Generator yielding chunks of a file. @@ -108,7 +108,7 @@ class FakeReq: def add_unredirected_header(self,header,value): self.connection.putheader(header,value) - + class FakeResp: """Compatability interface to use cookielib with raw httplib objects.""" @@ -178,5 +178,5 @@ if len(cookielib.parse_ns_headers([_test_cookie])) != 2: result.append(pairs) return result cookielib.parse_ns_headers = parse_ns_headers - assert len(cookielib.parse_ns_headers([_test_cookie])) == 2 + assert len(cookielib.parse_ns_headers([_test_cookie])) == 2 diff --git a/fs/contrib/sqlitefs.py b/fs/contrib/sqlitefs.py index b334d32..d6248e0 100644 --- a/fs/contrib/sqlitefs.py +++ b/fs/contrib/sqlitefs.py @@ -48,14 +48,14 @@ class SqliteFsFileBase(object): self.closed = False #real file like object. Most of the methods are passed to this object self.real_stream= real_file - + def close(self): if not self.closed and self.real_stream is not None: self._do_close() self.fs._on_close(self) self.real_stream.close() self.closed = True - + def __str__(self): return "<SqliteFS File in %s %s>" % (self.fs, self.path) @@ -70,18 +70,18 @@ class SqliteFsFileBase(object): def flush(self): self.real_stream.flush() - + def __iter__(self): raise OperationFailedError('__iter__', self.path) - + def next(self): raise OperationFailedError('next', self.path) - + def readline(self, *args, **kwargs): - raise OperationFailedError('readline', self.path) - + raise OperationFailedError('readline', self.path) + def read(self, size=None): - raise OperationFailedError('read', self.path) + raise OperationFailedError('read', self.path) def seek(self, *args, **kwargs): return self.real_stream.seek(*args, **kwargs) @@ -91,20 +91,20 @@ class SqliteFsFileBase(object): def truncate(self, *args, **kwargs): raise OperationFailedError('truncate', self.path) - + def write(self, data): raise OperationFailedError('write', self.path) - + def writelines(self, *args, **kwargs): raise OperationFailedError('writelines', self.path) - + def __enter__(self): return self def __exit__(self,exc_type,exc_value,traceback): self.close() return False - + class SqliteWritableFile(SqliteFsFileBase): ''' represents an sqlite file. Usually used for 'writing'. OnClose will @@ -114,28 +114,28 @@ class SqliteWritableFile(SqliteFsFileBase): super(SqliteWritableFile, self).__init__(fs, path, id) #open a temp file and return that. self.real_stream = tempfile.SpooledTemporaryFile(max_size='128*1000') - + def _do_close(self): #push the contents of the file to blob self.fs._writeblob(self.id, self.real_stream) - + def truncate(self, *args, **kwargs): return self.real_stream.truncate(*args, **kwargs) - + def write(self, data): return self.real_stream.write(data) def writelines(self, *args, **kwargs): return self.real_stream.writelines(*args, **kwargs) - + class SqliteReadableFile(SqliteFsFileBase): def __init__(self,fs, path, id, real_file): super(SqliteReadableFile, self).__init__(fs, path, id, real_file) - assert(self.real_stream != None) - + assert(self.real_stream != None) + def _do_close(self): pass - + def __iter__(self): return iter(self.real_stream) @@ -144,13 +144,13 @@ class SqliteReadableFile(SqliteFsFileBase): def readline(self, *args, **kwargs): return self.real_stream.readline(*args, **kwargs) - + def read(self, size=None): if( size==None): size=-1 return self.real_stream.read(size) - + class SqliteFS(FS): ''' sqlite based file system to store the files in sqlite database as 'blobs' @@ -161,12 +161,12 @@ class SqliteFS(FS): id : file id name : name of file parent : id of parent directory for the file. - + FsDirMetaData table: name : name of the directory (wihtout parent directory names) fullpath : full path of the directory including the parent directory name parent_id : id of the parent directory - + FsFileTable: size : file size in bytes (this is actual file size). Blob size may be different if compressed @@ -176,49 +176,49 @@ class SqliteFS(FS): last_modified : timestamp of last modification author : who changed it last content : blob where actual file contents are stored. - + TODO : Need an open files table or a flag in sqlite database. To avoid opening the file twice. (even from the different process or thread) ''' - + def __init__(self, sqlite_filename): super(SqliteFS, self).__init__() self.dbpath =sqlite_filename - self.dbcon =None + self.dbcon =None self.__actual_query_cur = None self.__actual_update_cur =None self.open_files = [] - + def close(self): ''' unlock all files. and close all open connections. ''' - self.close_all() + self.close_all() self._closedb() super(SqliteFS,self).close() - + def _initdb(self): if( self.dbcon is None): - self.dbcon = apsw.Connection(self.dbpath) + self.dbcon = apsw.Connection(self.dbpath) self._create_tables() - + @property def _querycur(self): assert(self.dbcon != None) if( self.__actual_query_cur == None): self.__actual_query_cur = self.dbcon.cursor() return(self.__actual_query_cur) - + @property def _updatecur(self): assert(self.dbcon != None) if( self.__actual_update_cur == None): self.__actual_update_cur = self.dbcon.cursor() return(self.__actual_update_cur) - + def _closedb(self): self.dbcon.close() - + def close_all(self): ''' close all open files @@ -226,7 +226,7 @@ class SqliteFS(FS): openfiles = list(self.open_files) for fileobj in openfiles: fileobj.close() - + def _create_tables(self): cur = self._updatecur cur.execute("CREATE TABLE IF NOT EXISTS FsFileMetaData(name text, fileid INTEGER, parent INTEGER)") @@ -235,12 +235,12 @@ class SqliteFS(FS): cur.execute("CREATE TABLE IF NOT EXISTS FsFileTable(type text, compression text, author TEXT, \ created timestamp, last_modified timestamp, last_accessed timestamp, \ locked BOOL, size INTEGER, contents BLOB)") - + #if the root directory name is created rootid = self._get_dir_id('/') if( rootid is None): cur.execute("INSERT INTO FsDirMetaData (name, fullpath) VALUES ('/','/')") - + def _get_dir_id(self, dirpath): ''' get the id for given directory path. @@ -248,15 +248,15 @@ class SqliteFS(FS): dirpath = remove_end_slash(dirpath) if( dirpath== None or len(dirpath)==0): dirpath = '/' - + self._querycur.execute("SELECT rowid from FsDirMetaData where fullpath=?",(dirpath,)) dirid = None dirrow = fetchone(self._querycur) if( dirrow): dirid = dirrow[0] - + return(dirid) - + def _get_file_id(self, dir_id, filename): ''' get the file id from the path @@ -269,10 +269,10 @@ class SqliteFS(FS): if( row ): file_id = row[0] return(file_id) - + def _get_file_contentid(self, file_id): ''' - return the file content id from the 'content' table (i.e. FsFileTable) + return the file content id from the 'content' table (i.e. FsFileTable) ''' assert(file_id != None) content_id = None @@ -281,7 +281,7 @@ class SqliteFS(FS): assert(row != None) content_id = row[0] return(content_id) - + def _create_file_entry(self, dirid, filename, **kwargs): ''' create file entry in the file table @@ -302,7 +302,7 @@ class SqliteFS(FS): #self.dbcon.commit() fileid = self.dbcon.last_insert_rowid() return(fileid) - + def _writeblob(self, fileid, stream): ''' extract the data from stream and write it as blob. @@ -314,15 +314,15 @@ class SqliteFS(FS): blob_stream=self.dbcon.blobopen("main", "FsFileTable", "contents", fileid, True) # 1 is for read/write stream.seek(0) blob_stream.write(stream.read()) - blob_stream.close() - - def _on_close(self, fileobj): + blob_stream.close() + + def _on_close(self, fileobj): #Unlock file on close. assert(fileobj != None and fileobj.id != None) self._lockfileentry(fileobj.id, lock=False) #Now remove it from openfile list. self.open_files.remove(fileobj) - + def _islocked(self, fileid): ''' check if the file is locked. @@ -336,7 +336,7 @@ class SqliteFS(FS): assert(row != None) locked = row[0] return(locked) - + def _lockfileentry(self, contentid, lock=True): ''' lock the file entry in the database. @@ -345,18 +345,18 @@ class SqliteFS(FS): last_accessed=datetime.datetime.now().isoformat() self._updatecur.execute('UPDATE FsFileTable SET locked=?, last_accessed=? where rowid=?', (lock, last_accessed, contentid)) - - def _makedir(self, parent_id, dname): + + def _makedir(self, parent_id, dname): self._querycur.execute("SELECT fullpath from FsDirMetaData where rowid=?",(parent_id,)) row = fetchone(self._querycur) assert(row != None) - parentpath = row[0] + parentpath = row[0] fullpath= pathjoin(parentpath, dname) - fullpath= remove_end_slash(fullpath) + fullpath= remove_end_slash(fullpath) created = datetime.datetime.now().isoformat() self._updatecur.execute('INSERT INTO FsDirMetaData(name, fullpath, parentid,created) \ VALUES(?,?,?,?)', (dname, fullpath, parent_id,created)) - + def _rename_file(self, src, dst): ''' rename source file 'src' to destination file 'dst' @@ -374,8 +374,8 @@ class SqliteFS(FS): if( dstfile_id != None): raise DestinationExistsError(dst) #All checks are done. Delete the entry for the source file. - #Create an entry for the destination file. - + #Create an entry for the destination file. + srcdir_id = self._get_dir_id(srcdir) assert(srcdir_id != None) srcfile_id = self._get_file_id(srcdir_id, srcfname) @@ -384,7 +384,7 @@ class SqliteFS(FS): self._updatecur.execute('DELETE FROM FsFileMetaData where ROWID=?',(srcfile_id,)) self._updatecur.execute("INSERT INTO FsFileMetaData(name, parent, fileid) \ VALUES(?,?,?)",(dstfname, dstdirid, srccontent_id)) - + def _rename_dir(self, src, dst): src = remove_end_slash(src) dst = remove_end_slash(dst) @@ -397,27 +397,27 @@ class SqliteFS(FS): raise ParentDirectoryMissingError(dst) srcdirid = self._get_dir_id(src) assert(srcdirid != None) - dstdname = basename(dst) + dstdname = basename(dst) self._updatecur.execute('UPDATE FsDirMetaData SET name=?, fullpath=?, \ parentid=? where ROWID=?',(dstdname, dst, dstparentid, srcdirid,)) - + def _get_dir_list(self, dirid, path, full): assert(dirid != None) assert(path != None) if( full==True): dirsearchpath = path + r'%' self._querycur.execute('SELECT fullpath FROM FsDirMetaData where fullpath LIKE ?', - (dirsearchpath,)) + (dirsearchpath,)) else: #search inside this directory only self._querycur.execute('SELECT fullpath FROM FsDirMetaData where parentid=?', (dirid,)) - dirlist = [row[0] for row in self._querycur] + dirlist = [row[0] for row in self._querycur] return dirlist - + def _get_file_list(self, dirpath, full): assert(dirpath != None) - + if( full==True): searchpath = dirpath + r"%" self._querycur.execute('SELECT FsFileMetaData.name, FsDirMetaData.fullpath \ @@ -429,10 +429,10 @@ class SqliteFS(FS): self._querycur.execute('SELECT FsFileMetaData.name, FsDirMetaData.fullpath \ FROM FsFileMetaData, FsDirMetaData where FsFileMetaData.parent=FsDirMetaData.ROWID \ and FsFileMetaData.parent =?',(parentid,)) - - filelist = [pathjoin(row[1],row[0]) for row in self._querycur] + + filelist = [pathjoin(row[1],row[0]) for row in self._querycur] return(filelist) - + def _get_dir_info(self, path): ''' get the directory information dictionary. @@ -440,7 +440,7 @@ class SqliteFS(FS): info = dict() info['st_mode'] = 0755 return info - + def _get_file_info(self, path): filedir = dirname(path) filename = basename(path) @@ -462,36 +462,36 @@ class SqliteFS(FS): info['last_accessed'] = row[4] info['st_mode'] = 0666 return(info) - + def _isfile(self,path): path = normpath(path) filedir = dirname(path) filename = basename(path) - dirid = self._get_dir_id(filedir) + dirid = self._get_dir_id(filedir) return(dirid is not None and self._get_file_id(dirid, filename) is not None) - + def _isdir(self,path): - path = normpath(path) + path = normpath(path) return(self._get_dir_id(path) is not None) - + def _isexist(self,path): return self._isfile(path) or self._isdir(path) - + @synchronize def open(self, path, mode='r', **kwargs): self._initdb() path = normpath(path) filedir = dirname(path) filename = basename(path) - + dir_id = self._get_dir_id(filedir) if( dir_id == None): raise ResourceNotFoundError(filedir) - - file_id = self._get_file_id(dir_id, filename) + + file_id = self._get_file_id(dir_id, filename) if( self._islocked(file_id)): - raise ResourceLockedError(path) - + raise ResourceLockedError(path) + sqfsfile=None if 'r' in mode: if file_id is None: @@ -500,74 +500,74 @@ class SqliteFS(FS): #make sure lock status is updated before the blob is opened self._lockfileentry(content_id, lock=True) blob_stream=self.dbcon.blobopen("main", "FsFileTable", "contents", file_id, False) # 1 is for read/write - sqfsfile = SqliteReadableFile(self, path, content_id, blob_stream) - + sqfsfile = SqliteReadableFile(self, path, content_id, blob_stream) + elif 'w' in mode or 'a' in mode: if( file_id is None): file_id= self._create_file_entry(dir_id, filename) assert(file_id != None) - + content_id = self._get_file_contentid(file_id) - #file_dir_entry.accessed_time = datetime.datetime.now() + #file_dir_entry.accessed_time = datetime.datetime.now() self._lockfileentry(content_id, lock=True) - sqfsfile = SqliteWritableFile(self, path, content_id) - + sqfsfile = SqliteWritableFile(self, path, content_id) + if( sqfsfile): self.open_files.append(sqfsfile) return sqfsfile - - raise ResourceNotFoundError(path) - + + raise ResourceNotFoundError(path) + @synchronize def isfile(self, path): self._initdb() return self._isfile(path) - + @synchronize def isdir(self, path): self._initdb() return self._isdir(path) - + @synchronize def listdir(self, path='/', wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False): - path = normpath(path) + path = normpath(path) dirid = self._get_dir_id(path) if( dirid == None): raise ResourceInvalidError(path) - + dirlist = self._get_dir_list(dirid, path,full) if( dirs_only): pathlist = dirlist - else: + else: filelist = self._get_file_list(path, full) - + if( files_only == True): pathlist = filelist else: pathlist = filelist + dirlist - - + + if( wildcard and dirs_only == False): pass - + if( absolute == False): pathlist = map(lambda dpath:frombase(path,dpath), pathlist) - + return(pathlist) - - + + @synchronize def makedir(self, path, recursive=False, allow_recreate=False): self._initdb() path = remove_end_slash(normpath(path)) - + if(self._isexist(path)==False): parentdir = dirname(path) dname = basename(path) - + parent_id = self._get_dir_id(parentdir) if( parent_id ==None): - if( recursive == False): + if( recursive == False): raise ParentDirectoryMissingError(path) else: self.makedir(parentdir, recursive,allow_recreate) @@ -575,7 +575,7 @@ class SqliteFS(FS): self._makedir(parent_id,dname) else: raise DestinationExistsError(path) - + @synchronize def remove(self, path): self._initdb() @@ -583,16 +583,16 @@ class SqliteFS(FS): if( self.isdir(path)==True): #path is actually a directory raise ResourceInvalidError(path) - + filedir = dirname(path) filename = basename(path) dirid = self._get_dir_id(filedir) fileid = self._get_file_id(dirid, filename) if( fileid == None): raise ResourceNotFoundError(path) - + content_id = self._get_file_contentid(fileid) - + self._updatecur.execute("DELETE FROM FsFileMetaData where ROWID=?",(fileid,)) #check there is any other file pointing to same location. If not #delete the content as well. @@ -600,8 +600,8 @@ class SqliteFS(FS): (content_id,)) row = fetchone(self._querycur) if( row == None or row[0] == 0): - self._updatecur.execute("DELETE FROM FsFileTable where ROWID=?",(content_id,)) - + self._updatecur.execute("DELETE FROM FsFileTable where ROWID=?",(content_id,)) + @synchronize def removedir(self,path, recursive=False, force=False): self._initdb() @@ -617,36 +617,36 @@ class SqliteFS(FS): row = fetchone(self._qurycur) if( row[0] > 0): raise DirectoryNotEmptyError(path) - self._updatecur.execute("DELETE FROM FsDirMetaData where ROWID=?",(dirid,)) - + self._updatecur.execute("DELETE FROM FsDirMetaData where ROWID=?",(dirid,)) + @synchronize def rename(self,src, dst): self._initdb() src = normpath(src) dst = normpath(dst) if self._isexist(dst)== False: - #first check if this is a directory rename or a file rename + #first check if this is a directory rename or a file rename if( self.isfile(src)): self._rename_file(src, dst) elif self.isdir(src): self._rename_dir(src, dst) else: raise ResourceNotFoundError(path) - else: + else: raise DestinationExistsError(dst) - + @synchronize def getinfo(self, path): - self._initdb() + self._initdb() path = normpath(path) isfile = False isdir = self.isdir(path) if( isdir == False): isfile=self.isfile(path) - + if( not isfile and not isdir): raise ResourceNotFoundError(path) - + if isdir: info= self._get_dir_info(path) else: @@ -664,7 +664,7 @@ class SqliteFS(FS): # #mp = dokan.mount(sqfs,driveletter,foreground=True) # #mp.unmount() # sqfs.close() -# +# #def run_tests(sqlfilename): # fs = SqliteFS(sqlfilename) # fs.makedir('/test') @@ -691,15 +691,15 @@ class SqliteFS(FS): # flist = fs.listdir('/', full=True,absolute=True,files_only=True) # print flist # fs.close() -# +# #if __name__ == '__main__': # run_tests("sqfs.sqlite") # mount_windows("sqfs.sqlite", 'm') -# +# # #fs.remove('/test1/test1.txt') # #try: # # f = fs.open('/test1/test1.txt', "r") # #except ResourceNotFoundError: # # print "Success : file doesnot exist" # #fs.browse() -#
\ No newline at end of file +# diff --git a/fs/contrib/tahoelafs/__init__.py b/fs/contrib/tahoelafs/__init__.py index c01dd5e..91858bd 100644 --- a/fs/contrib/tahoelafs/__init__.py +++ b/fs/contrib/tahoelafs/__init__.py @@ -1,415 +1,415 @@ -'''
-fs.contrib.tahoelafs
-====================
-
-This modules provides a PyFilesystem interface to the Tahoe Least Authority
-File System. Tahoe-LAFS is a distributed, encrypted, fault-tolerant storage
-system:
-
- http://tahoe-lafs.org/
-
-You will need access to a Tahoe-LAFS "web api" service.
-
-Example (it will use publicly available (but slow) Tahoe-LAFS cloud)::
-
- from fs.contrib.tahoelafs import TahoeLAFS, Connection
- dircap = TahoeLAFS.createdircap(webapi='http://insecure.tahoe-lafs.org')
- print "Your dircap (unique key to your storage directory) is", dircap
- print "Keep it safe!"
- fs = TahoeLAFS(dircap, autorun=False, webapi='http://insecure.tahoe-lafs.org')
- f = fs.open("foo.txt", "a")
- f.write('bar!')
- f.close()
- print "Now visit %s and enjoy :-)" % fs.getpathurl('foo.txt')
-
-When any problem occurred, you can turn on internal debugging messages::
-
- import logging
- l = logging.getLogger()
- l.setLevel(logging.DEBUG)
- l.addHandler(logging.StreamHandler(sys.stdout))
-
- ... your Python code using TahoeLAFS ...
-
-TODO:
-
- * unicode support
- * try network errors / bad happiness
- * exceptions
- * tests
- * sanitize all path types (., /)
- * support for extra large file uploads (poster module)
- * Possibility to block write until upload done (Tahoe mailing list)
- * Report something sane when Tahoe crashed/unavailable
- * solve failed unit tests (makedir_winner, ...)
- * file times
- * docs & author
- * python3 support
- * remove creating blank files (depends on FileUploadManager)
-
-TODO (Not TahoeLAFS specific tasks):
- * RemoteFileBuffer on the fly buffering support
- * RemoteFileBuffer unit tests
- * RemoteFileBuffer submit to trunk
- * Implement FileUploadManager + faking isfile/exists of just processing file
- * pyfilesystem docs is outdated (rename, movedir, ...)
-
-'''
-
-
-import stat as statinfo
-
-import logging
-from logging import DEBUG, INFO, ERROR, CRITICAL
-
-import fs
-import fs.errors as errors
-from fs.path import abspath, relpath, normpath, dirname, pathjoin
-from fs.base import FS, NullFile
-from fs import _thread_synchronize_default, SEEK_END
-from fs.remote import CacheFSMixin, RemoteFileBuffer
-from fs.base import fnmatch, NoDefaultMeta
-
-from util import TahoeUtil
-from connection import Connection
-
-from six import b
-
-logger = fs.getLogger('fs.tahoelafs')
-
-def _fix_path(func):
- """Method decorator for automatically normalising paths."""
- def wrapper(self, *args, **kwds):
- if len(args):
- args = list(args)
- args[0] = _fixpath(args[0])
- return func(self, *args, **kwds)
- return wrapper
-
-
-def _fixpath(path):
- """Normalize the given path."""
- return abspath(normpath(path))
-
-
-
-class _TahoeLAFS(FS):
- """FS providing raw access to a Tahoe-LAFS Filesystem.
-
- This class implements all the details of interacting with a Tahoe-backed
- filesystem, but you probably don't want to use it in practice. Use the
- TahoeLAFS class instead, which has some internal caching to improve
- performance.
- """
-
- _meta = { 'virtual' : False,
- 'read_only' : False,
- 'unicode_paths' : True,
- 'case_insensitive_paths' : False,
- 'network' : True
- }
-
-
- def __init__(self, dircap, largefilesize=10*1024*1024, webapi='http://127.0.0.1:3456'):
- '''Creates instance of TahoeLAFS.
-
- :param dircap: special hash allowing user to work with TahoeLAFS directory.
- :param largefilesize: - Create placeholder file for files larger than this treshold.
- Uploading and processing of large files can last extremely long (many hours),
- so placing this placeholder can help you to remember that upload is processing.
- Setting this to None will skip creating placeholder files for any uploads.
- '''
- self.dircap = dircap if not dircap.endswith('/') else dircap[:-1]
- self.largefilesize = largefilesize
- self.connection = Connection(webapi)
- self.tahoeutil = TahoeUtil(webapi)
- super(_TahoeLAFS, self).__init__(thread_synchronize=_thread_synchronize_default)
-
- def __str__(self):
- return "<TahoeLAFS: %s>" % self.dircap
-
- @classmethod
- def createdircap(cls, webapi='http://127.0.0.1:3456'):
- return TahoeUtil(webapi).createdircap()
-
- def getmeta(self,meta_name,default=NoDefaultMeta):
- if meta_name == "read_only":
- return self.dircap.startswith('URI:DIR2-RO')
- return super(_TahoeLAFS,self).getmeta(meta_name,default)
-
- @_fix_path
- def open(self, path, mode='r', **kwargs):
- self._log(INFO, 'Opening file %s in mode %s' % (path, mode))
- newfile = False
- if not self.exists(path):
- if 'w' in mode or 'a' in mode:
- newfile = True
- else:
- self._log(DEBUG, "File %s not found while opening for reads" % path)
- raise errors.ResourceNotFoundError(path)
- elif self.isdir(path):
- self._log(DEBUG, "Path %s is directory, not a file" % path)
- raise errors.ResourceInvalidError(path)
- elif 'w' in mode:
- newfile = True
-
- if newfile:
- self._log(DEBUG, 'Creating empty file %s' % path)
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
- self.setcontents(path, b(''))
- handler = NullFile()
- else:
- self._log(DEBUG, 'Opening existing file %s for reading' % path)
- handler = self.getrange(path,0)
-
- return RemoteFileBuffer(self, path, mode, handler,
- write_on_flush=False)
-
- @_fix_path
- def desc(self, path):
- try:
- return self.getinfo(path)
- except:
- return ''
-
- @_fix_path
- def exists(self, path):
- try:
- self.getinfo(path)
- self._log(DEBUG, "Path %s exists" % path)
- return True
- except errors.ResourceNotFoundError:
- self._log(DEBUG, "Path %s does not exists" % path)
- return False
- except errors.ResourceInvalidError:
- self._log(DEBUG, "Path %s does not exists, probably misspelled URI" % path)
- return False
-
- @_fix_path
- def getsize(self, path):
- try:
- size = self.getinfo(path)['size']
- self._log(DEBUG, "Size of %s is %d" % (path, size))
- return size
- except errors.ResourceNotFoundError:
- return 0
-
- @_fix_path
- def isfile(self, path):
- try:
- isfile = (self.getinfo(path)['type'] == 'filenode')
- except errors.ResourceNotFoundError:
- #isfile = not path.endswith('/')
- isfile = False
- self._log(DEBUG, "Path %s is file: %d" % (path, isfile))
- return isfile
-
- @_fix_path
- def isdir(self, path):
- try:
- isdir = (self.getinfo(path)['type'] == 'dirnode')
- except errors.ResourceNotFoundError:
- isdir = False
- self._log(DEBUG, "Path %s is directory: %d" % (path, isdir))
- return isdir
-
-
- def listdir(self, *args, **kwargs):
- return [ item[0] for item in self.listdirinfo(*args, **kwargs) ]
-
- def listdirinfo(self, *args, **kwds):
- return list(self.ilistdirinfo(*args,**kwds))
-
- def ilistdir(self, *args, **kwds):
- for item in self.ilistdirinfo(*args,**kwds):
- yield item[0]
-
- @_fix_path
- def ilistdirinfo(self, path="/", wildcard=None, full=False, absolute=False,
- dirs_only=False, files_only=False):
- self._log(DEBUG, "Listing directory (listdirinfo) %s" % path)
-
- if dirs_only and files_only:
- raise ValueError("dirs_only and files_only can not both be True")
-
- for item in self.tahoeutil.list(self.dircap, path):
- if dirs_only and item['type'] == 'filenode':
- continue
- elif files_only and item['type'] == 'dirnode':
- continue
-
- if wildcard is not None:
- if isinstance(wildcard,basestring):
- if not fnmatch.fnmatch(item['name'], wildcard):
- continue
- else:
- if not wildcard(item['name']):
- continue
-
- if full:
- item_path = relpath(pathjoin(path, item['name']))
- elif absolute:
- item_path = abspath(pathjoin(path, item['name']))
- else:
- item_path = item['name']
-
- yield (item_path, item)
-
- @_fix_path
- def remove(self, path):
- self._log(INFO, 'Removing file %s' % path)
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
-
- if not self.isfile(path):
- if not self.isdir(path):
- raise errors.ResourceNotFoundError(path)
- raise errors.ResourceInvalidError(path)
-
- try:
- self.tahoeutil.unlink(self.dircap, path)
- except Exception, e:
- raise errors.ResourceInvalidError(path)
-
- @_fix_path
- def removedir(self, path, recursive=False, force=False):
- self._log(INFO, "Removing directory %s" % path)
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
- if not self.isdir(path):
- if not self.isfile(path):
- raise errors.ResourceNotFoundError(path)
- raise errors.ResourceInvalidError(path)
- if not force and self.listdir(path):
- raise errors.DirectoryNotEmptyError(path)
-
- self.tahoeutil.unlink(self.dircap, path)
-
- if recursive and path != '/':
- try:
- self.removedir(dirname(path), recursive=True)
- except errors.DirectoryNotEmptyError:
- pass
-
- @_fix_path
- def makedir(self, path, recursive=False, allow_recreate=False):
- self._log(INFO, "Creating directory %s" % path)
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
- if self.exists(path):
- if not self.isdir(path):
- raise errors.ResourceInvalidError(path)
- if not allow_recreate:
- raise errors.DestinationExistsError(path)
- if not recursive and not self.exists(dirname(path)):
- raise errors.ParentDirectoryMissingError(path)
- self.tahoeutil.mkdir(self.dircap, path)
-
- def movedir(self, src, dst, overwrite=False):
- self.move(src, dst, overwrite=overwrite)
-
- def move(self, src, dst, overwrite=False):
- self._log(INFO, "Moving file from %s to %s" % (src, dst))
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
- src = _fixpath(src)
- dst = _fixpath(dst)
- if not self.exists(dirname(dst)):
- raise errors.ParentDirectoryMissingError(dst)
- if not overwrite and self.exists(dst):
- raise errors.DestinationExistsError(dst)
- self.tahoeutil.move(self.dircap, src, dst)
-
- def rename(self, src, dst):
- self.move(src, dst)
-
- def copy(self, src, dst, overwrite=False, chunk_size=16384):
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
- # FIXME: this is out of date; how to do native tahoe copy?
- # FIXME: Workaround because isfile() not exists on _TahoeLAFS
- FS.copy(self, src, dst, overwrite, chunk_size)
-
- def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
- # FIXME: this is out of date; how to do native tahoe copy?
- # FIXME: Workaround because isfile() not exists on _TahoeLAFS
- FS.copydir(self, src, dst, overwrite, ignore_errors, chunk_size)
-
-
- def _log(self, level, message):
- if not logger.isEnabledFor(level): return
- logger.log(level, u'(%d) %s' % (id(self),
- unicode(message).encode('ASCII', 'replace')))
-
- @_fix_path
- def getpathurl(self, path, allow_none=False, webapi=None):
- '''
- Retrieve URL where the file/directory is stored
- '''
- if webapi == None:
- webapi = self.connection.webapi
- self._log(DEBUG, "Retrieving URL for %s over %s" % (path, webapi))
- path = self.tahoeutil.fixwinpath(path, False)
- return u"%s/uri/%s%s" % (webapi, self.dircap, path)
-
- @_fix_path
- def getrange(self, path, offset, length=None):
- return self.connection.get(u'/uri/%s%s' % (self.dircap, path),
- offset=offset, length=length)
-
- @_fix_path
- def setcontents(self, path, file, chunk_size=64*1024):
- self._log(INFO, 'Uploading file %s' % path)
- size=None
-
- if self.getmeta("read_only"):
- raise errors.UnsupportedError('read only filesystem')
-
- # Workaround for large files:
- # First create zero file placeholder, then
- # upload final content.
- if self.largefilesize != None and getattr(file, 'read', None):
- # As 'file' can be also a string, need to check,
- # if 'file' looks like duck. Sorry, file.
- file.seek(0, SEEK_END)
- size = file.tell()
- file.seek(0)
-
- if size > self.largefilesize:
- self.connection.put(u'/uri/%s%s' % (self.dircap, path),
- "PyFilesystem.TahoeLAFS: Upload started, final size %d" % size)
-
- self.connection.put(u'/uri/%s%s' % (self.dircap, path), file, size=size)
-
- @_fix_path
- def getinfo(self, path):
- self._log(INFO, 'Reading meta for %s' % path)
- info = self.tahoeutil.info(self.dircap, path)
- #import datetime
- #info['created_time'] = datetime.datetime.now()
- #info['modified_time'] = datetime.datetime.now()
- #info['accessed_time'] = datetime.datetime.now()
- if info['type'] == 'filenode':
- info["st_mode"] = 0x700 | statinfo.S_IFREG
- elif info['type'] == 'dirnode':
- info["st_mode"] = 0x700 | statinfo.S_IFDIR
- return info
-
-
-
-class TahoeLAFS(CacheFSMixin,_TahoeLAFS):
- """FS providing cached access to a Tahoe Filesystem.
-
- This class is the preferred means to access a Tahoe filesystem. It
- maintains an internal cache of recently-accessed metadata to speed
- up operations.
- """
-
- def __init__(self, *args, **kwds):
- kwds.setdefault("cache_timeout",60)
- super(TahoeLAFS,self).__init__(*args,**kwds)
-
-
+''' +fs.contrib.tahoelafs +==================== + +This modules provides a PyFilesystem interface to the Tahoe Least Authority +File System. Tahoe-LAFS is a distributed, encrypted, fault-tolerant storage +system: + + http://tahoe-lafs.org/ + +You will need access to a Tahoe-LAFS "web api" service. + +Example (it will use publicly available (but slow) Tahoe-LAFS cloud):: + + from fs.contrib.tahoelafs import TahoeLAFS, Connection + dircap = TahoeLAFS.createdircap(webapi='http://insecure.tahoe-lafs.org') + print "Your dircap (unique key to your storage directory) is", dircap + print "Keep it safe!" + fs = TahoeLAFS(dircap, autorun=False, webapi='http://insecure.tahoe-lafs.org') + f = fs.open("foo.txt", "a") + f.write('bar!') + f.close() + print "Now visit %s and enjoy :-)" % fs.getpathurl('foo.txt') + +When any problem occurred, you can turn on internal debugging messages:: + + import logging + l = logging.getLogger() + l.setLevel(logging.DEBUG) + l.addHandler(logging.StreamHandler(sys.stdout)) + + ... your Python code using TahoeLAFS ... + +TODO: + + * unicode support + * try network errors / bad happiness + * exceptions + * tests + * sanitize all path types (., /) + * support for extra large file uploads (poster module) + * Possibility to block write until upload done (Tahoe mailing list) + * Report something sane when Tahoe crashed/unavailable + * solve failed unit tests (makedir_winner, ...) + * file times + * docs & author + * python3 support + * remove creating blank files (depends on FileUploadManager) + +TODO (Not TahoeLAFS specific tasks): + * RemoteFileBuffer on the fly buffering support + * RemoteFileBuffer unit tests + * RemoteFileBuffer submit to trunk + * Implement FileUploadManager + faking isfile/exists of just processing file + * pyfilesystem docs is outdated (rename, movedir, ...) + +''' + + +import stat as statinfo + +import logging +from logging import DEBUG, INFO, ERROR, CRITICAL + +import fs +import fs.errors as errors +from fs.path import abspath, relpath, normpath, dirname, pathjoin +from fs.base import FS, NullFile +from fs import _thread_synchronize_default, SEEK_END +from fs.remote import CacheFSMixin, RemoteFileBuffer +from fs.base import fnmatch, NoDefaultMeta + +from util import TahoeUtil +from connection import Connection + +from six import b + +logger = fs.getLogger('fs.tahoelafs') + +def _fix_path(func): + """Method decorator for automatically normalising paths.""" + def wrapper(self, *args, **kwds): + if len(args): + args = list(args) + args[0] = _fixpath(args[0]) + return func(self, *args, **kwds) + return wrapper + + +def _fixpath(path): + """Normalize the given path.""" + return abspath(normpath(path)) + + + +class _TahoeLAFS(FS): + """FS providing raw access to a Tahoe-LAFS Filesystem. + + This class implements all the details of interacting with a Tahoe-backed + filesystem, but you probably don't want to use it in practice. Use the + TahoeLAFS class instead, which has some internal caching to improve + performance. + """ + + _meta = { 'virtual' : False, + 'read_only' : False, + 'unicode_paths' : True, + 'case_insensitive_paths' : False, + 'network' : True + } + + + def __init__(self, dircap, largefilesize=10*1024*1024, webapi='http://127.0.0.1:3456'): + '''Creates instance of TahoeLAFS. + + :param dircap: special hash allowing user to work with TahoeLAFS directory. + :param largefilesize: - Create placeholder file for files larger than this treshold. + Uploading and processing of large files can last extremely long (many hours), + so placing this placeholder can help you to remember that upload is processing. + Setting this to None will skip creating placeholder files for any uploads. + ''' + self.dircap = dircap if not dircap.endswith('/') else dircap[:-1] + self.largefilesize = largefilesize + self.connection = Connection(webapi) + self.tahoeutil = TahoeUtil(webapi) + super(_TahoeLAFS, self).__init__(thread_synchronize=_thread_synchronize_default) + + def __str__(self): + return "<TahoeLAFS: %s>" % self.dircap + + @classmethod + def createdircap(cls, webapi='http://127.0.0.1:3456'): + return TahoeUtil(webapi).createdircap() + + def getmeta(self,meta_name,default=NoDefaultMeta): + if meta_name == "read_only": + return self.dircap.startswith('URI:DIR2-RO') + return super(_TahoeLAFS,self).getmeta(meta_name,default) + + @_fix_path + def open(self, path, mode='r', **kwargs): + self._log(INFO, 'Opening file %s in mode %s' % (path, mode)) + newfile = False + if not self.exists(path): + if 'w' in mode or 'a' in mode: + newfile = True + else: + self._log(DEBUG, "File %s not found while opening for reads" % path) + raise errors.ResourceNotFoundError(path) + elif self.isdir(path): + self._log(DEBUG, "Path %s is directory, not a file" % path) + raise errors.ResourceInvalidError(path) + elif 'w' in mode: + newfile = True + + if newfile: + self._log(DEBUG, 'Creating empty file %s' % path) + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + self.setcontents(path, b('')) + handler = NullFile() + else: + self._log(DEBUG, 'Opening existing file %s for reading' % path) + handler = self.getrange(path,0) + + return RemoteFileBuffer(self, path, mode, handler, + write_on_flush=False) + + @_fix_path + def desc(self, path): + try: + return self.getinfo(path) + except: + return '' + + @_fix_path + def exists(self, path): + try: + self.getinfo(path) + self._log(DEBUG, "Path %s exists" % path) + return True + except errors.ResourceNotFoundError: + self._log(DEBUG, "Path %s does not exists" % path) + return False + except errors.ResourceInvalidError: + self._log(DEBUG, "Path %s does not exists, probably misspelled URI" % path) + return False + + @_fix_path + def getsize(self, path): + try: + size = self.getinfo(path)['size'] + self._log(DEBUG, "Size of %s is %d" % (path, size)) + return size + except errors.ResourceNotFoundError: + return 0 + + @_fix_path + def isfile(self, path): + try: + isfile = (self.getinfo(path)['type'] == 'filenode') + except errors.ResourceNotFoundError: + #isfile = not path.endswith('/') + isfile = False + self._log(DEBUG, "Path %s is file: %d" % (path, isfile)) + return isfile + + @_fix_path + def isdir(self, path): + try: + isdir = (self.getinfo(path)['type'] == 'dirnode') + except errors.ResourceNotFoundError: + isdir = False + self._log(DEBUG, "Path %s is directory: %d" % (path, isdir)) + return isdir + + + def listdir(self, *args, **kwargs): + return [ item[0] for item in self.listdirinfo(*args, **kwargs) ] + + def listdirinfo(self, *args, **kwds): + return list(self.ilistdirinfo(*args,**kwds)) + + def ilistdir(self, *args, **kwds): + for item in self.ilistdirinfo(*args,**kwds): + yield item[0] + + @_fix_path + def ilistdirinfo(self, path="/", wildcard=None, full=False, absolute=False, + dirs_only=False, files_only=False): + self._log(DEBUG, "Listing directory (listdirinfo) %s" % path) + + if dirs_only and files_only: + raise ValueError("dirs_only and files_only can not both be True") + + for item in self.tahoeutil.list(self.dircap, path): + if dirs_only and item['type'] == 'filenode': + continue + elif files_only and item['type'] == 'dirnode': + continue + + if wildcard is not None: + if isinstance(wildcard,basestring): + if not fnmatch.fnmatch(item['name'], wildcard): + continue + else: + if not wildcard(item['name']): + continue + + if full: + item_path = relpath(pathjoin(path, item['name'])) + elif absolute: + item_path = abspath(pathjoin(path, item['name'])) + else: + item_path = item['name'] + + yield (item_path, item) + + @_fix_path + def remove(self, path): + self._log(INFO, 'Removing file %s' % path) + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + + if not self.isfile(path): + if not self.isdir(path): + raise errors.ResourceNotFoundError(path) + raise errors.ResourceInvalidError(path) + + try: + self.tahoeutil.unlink(self.dircap, path) + except Exception, e: + raise errors.ResourceInvalidError(path) + + @_fix_path + def removedir(self, path, recursive=False, force=False): + self._log(INFO, "Removing directory %s" % path) + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + if not self.isdir(path): + if not self.isfile(path): + raise errors.ResourceNotFoundError(path) + raise errors.ResourceInvalidError(path) + if not force and self.listdir(path): + raise errors.DirectoryNotEmptyError(path) + + self.tahoeutil.unlink(self.dircap, path) + + if recursive and path != '/': + try: + self.removedir(dirname(path), recursive=True) + except errors.DirectoryNotEmptyError: + pass + + @_fix_path + def makedir(self, path, recursive=False, allow_recreate=False): + self._log(INFO, "Creating directory %s" % path) + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + if self.exists(path): + if not self.isdir(path): + raise errors.ResourceInvalidError(path) + if not allow_recreate: + raise errors.DestinationExistsError(path) + if not recursive and not self.exists(dirname(path)): + raise errors.ParentDirectoryMissingError(path) + self.tahoeutil.mkdir(self.dircap, path) + + def movedir(self, src, dst, overwrite=False): + self.move(src, dst, overwrite=overwrite) + + def move(self, src, dst, overwrite=False): + self._log(INFO, "Moving file from %s to %s" % (src, dst)) + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + src = _fixpath(src) + dst = _fixpath(dst) + if not self.exists(dirname(dst)): + raise errors.ParentDirectoryMissingError(dst) + if not overwrite and self.exists(dst): + raise errors.DestinationExistsError(dst) + self.tahoeutil.move(self.dircap, src, dst) + + def rename(self, src, dst): + self.move(src, dst) + + def copy(self, src, dst, overwrite=False, chunk_size=16384): + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + # FIXME: this is out of date; how to do native tahoe copy? + # FIXME: Workaround because isfile() not exists on _TahoeLAFS + FS.copy(self, src, dst, overwrite, chunk_size) + + def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384): + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + # FIXME: this is out of date; how to do native tahoe copy? + # FIXME: Workaround because isfile() not exists on _TahoeLAFS + FS.copydir(self, src, dst, overwrite, ignore_errors, chunk_size) + + + def _log(self, level, message): + if not logger.isEnabledFor(level): return + logger.log(level, u'(%d) %s' % (id(self), + unicode(message).encode('ASCII', 'replace'))) + + @_fix_path + def getpathurl(self, path, allow_none=False, webapi=None): + ''' + Retrieve URL where the file/directory is stored + ''' + if webapi == None: + webapi = self.connection.webapi + self._log(DEBUG, "Retrieving URL for %s over %s" % (path, webapi)) + path = self.tahoeutil.fixwinpath(path, False) + return u"%s/uri/%s%s" % (webapi, self.dircap, path) + + @_fix_path + def getrange(self, path, offset, length=None): + return self.connection.get(u'/uri/%s%s' % (self.dircap, path), + offset=offset, length=length) + + @_fix_path + def setcontents(self, path, file, chunk_size=64*1024): + self._log(INFO, 'Uploading file %s' % path) + size=None + + if self.getmeta("read_only"): + raise errors.UnsupportedError('read only filesystem') + + # Workaround for large files: + # First create zero file placeholder, then + # upload final content. + if self.largefilesize != None and getattr(file, 'read', None): + # As 'file' can be also a string, need to check, + # if 'file' looks like duck. Sorry, file. + file.seek(0, SEEK_END) + size = file.tell() + file.seek(0) + + if size > self.largefilesize: + self.connection.put(u'/uri/%s%s' % (self.dircap, path), + "PyFilesystem.TahoeLAFS: Upload started, final size %d" % size) + + self.connection.put(u'/uri/%s%s' % (self.dircap, path), file, size=size) + + @_fix_path + def getinfo(self, path): + self._log(INFO, 'Reading meta for %s' % path) + info = self.tahoeutil.info(self.dircap, path) + #import datetime + #info['created_time'] = datetime.datetime.now() + #info['modified_time'] = datetime.datetime.now() + #info['accessed_time'] = datetime.datetime.now() + if info['type'] == 'filenode': + info["st_mode"] = 0x700 | statinfo.S_IFREG + elif info['type'] == 'dirnode': + info["st_mode"] = 0x700 | statinfo.S_IFDIR + return info + + + +class TahoeLAFS(CacheFSMixin,_TahoeLAFS): + """FS providing cached access to a Tahoe Filesystem. + + This class is the preferred means to access a Tahoe filesystem. It + maintains an internal cache of recently-accessed metadata to speed + up operations. + """ + + def __init__(self, *args, **kwds): + kwds.setdefault("cache_timeout",60) + super(TahoeLAFS,self).__init__(*args,**kwds) + + diff --git a/fs/contrib/tahoelafs/connection.py b/fs/contrib/tahoelafs/connection.py index e3df28a..448919b 100644 --- a/fs/contrib/tahoelafs/connection.py +++ b/fs/contrib/tahoelafs/connection.py @@ -17,12 +17,12 @@ class PutRequest(Request): def __init__(self, *args, **kwargs): self.get_method = lambda: u'PUT' Request.__init__(self, *args, **kwargs) - + class DeleteRequest(Request): def __init__(self, *args, **kwargs): self.get_method = lambda: u'DELETE' Request.__init__(self, *args, **kwargs) - + class Connection: def __init__(self, webapi): self.webapi = webapi @@ -37,13 +37,13 @@ class Connection: size = len(f) elif getattr(f, 'read', None): if size == None: - # When size is already known, skip this + # When size is already known, skip this f.seek(0, SEEK_END) size = f.tell() f.seek(0) else: raise errors.UnsupportedError("Cannot handle type %s" % type(f)) - + headers = {'Content-Length': size} headers.update(self.headers) return headers @@ -59,7 +59,7 @@ class Connection: if params: return u"%s?%s" % (q, self._urlencode(params)) return q - + def _urlopen(self, req): try: return urlopen(req) @@ -74,17 +74,17 @@ class Connection: # Standard not found raise errors.ResourceNotFoundError(e.fp.read()) raise errors.ResourceInvalidError(e.fp.read()) - + def post(self, path, data={}, params={}): data = self._urlencode(data) path = self._quotepath(path, params) req = Request(''.join([self.webapi, path]), data, headers=self.headers) return self._urlopen(req) - + def get(self, path, data={}, offset=None, length=None): data = self._urlencode(data) path = self._quotepath(path) - if data: + if data: path = u'?'.join([path, data]) headers = {} @@ -95,17 +95,17 @@ class Connection: (int(offset), int(offset+length)) else: headers['Range'] = 'bytes=%d-' % int(offset) - + req = Request(''.join([self.webapi, path]), headers=headers) return self._urlopen(req) def put(self, path, data, size=None, params={}): path = self._quotepath(path, params) headers = self._get_headers(data, size=size) - req = PutRequest(''.join([self.webapi, path]), data, headers=headers) + req = PutRequest(''.join([self.webapi, path]), data, headers=headers) return self._urlopen(req) - - def delete(self, path, data={}): + + def delete(self, path, data={}): path = self._quotepath(path) req = DeleteRequest(''.join([self.webapi, path]), data, headers=self.headers) return self._urlopen(req) diff --git a/fs/contrib/tahoelafs/test_tahoelafs.py b/fs/contrib/tahoelafs/test_tahoelafs.py index 61a28e5..5cb6ca7 100644 --- a/fs/contrib/tahoelafs/test_tahoelafs.py +++ b/fs/contrib/tahoelafs/test_tahoelafs.py @@ -1,52 +1,52 @@ -#!/usr/bin/python
-"""
- Test the TahoeLAFS
-
- @author: Marek Palatinus <marek@palatinus.cz>
-"""
-
-import sys
-import logging
-import unittest
-
-from fs.base import FS
-import fs.errors as errors
-from fs.tests import FSTestCases, ThreadingTestCases
-from fs.contrib.tahoelafs import TahoeLAFS, Connection
-
-logging.getLogger().setLevel(logging.DEBUG)
-logging.getLogger('fs.tahoelafs').addHandler(logging.StreamHandler(sys.stdout))
-
-WEBAPI = 'http://insecure.tahoe-lafs.org'
-
-
-# The public grid is too slow for threading testcases, disabling for now...
-class TestTahoeLAFS(unittest.TestCase,FSTestCases):#,ThreadingTestCases):
-
- # Disabled by default because it takes a *really* long time.
- __test__ = False
-
- def setUp(self):
- self.dircap = TahoeLAFS.createdircap(WEBAPI)
- self.fs = TahoeLAFS(self.dircap, cache_timeout=0, webapi=WEBAPI)
-
- def tearDown(self):
- self.fs.close()
-
- def test_dircap(self):
- # Is dircap in correct format?
- self.assert_(self.dircap.startswith('URI:DIR2:') and len(self.dircap) > 50)
-
- def test_concurrent_copydir(self):
- # makedir() on TahoeLAFS is currently not atomic
- pass
-
- def test_makedir_winner(self):
- # makedir() on TahoeLAFS is currently not atomic
- pass
-
- def test_big_file(self):
- pass
-
-if __name__ == '__main__':
- unittest.main()
+#!/usr/bin/python +""" + Test the TahoeLAFS + + @author: Marek Palatinus <marek@palatinus.cz> +""" + +import sys +import logging +import unittest + +from fs.base import FS +import fs.errors as errors +from fs.tests import FSTestCases, ThreadingTestCases +from fs.contrib.tahoelafs import TahoeLAFS, Connection + +logging.getLogger().setLevel(logging.DEBUG) +logging.getLogger('fs.tahoelafs').addHandler(logging.StreamHandler(sys.stdout)) + +WEBAPI = 'http://insecure.tahoe-lafs.org' + + +# The public grid is too slow for threading testcases, disabling for now... +class TestTahoeLAFS(unittest.TestCase,FSTestCases):#,ThreadingTestCases): + + # Disabled by default because it takes a *really* long time. + __test__ = False + + def setUp(self): + self.dircap = TahoeLAFS.createdircap(WEBAPI) + self.fs = TahoeLAFS(self.dircap, cache_timeout=0, webapi=WEBAPI) + + def tearDown(self): + self.fs.close() + + def test_dircap(self): + # Is dircap in correct format? + self.assert_(self.dircap.startswith('URI:DIR2:') and len(self.dircap) > 50) + + def test_concurrent_copydir(self): + # makedir() on TahoeLAFS is currently not atomic + pass + + def test_makedir_winner(self): + # makedir() on TahoeLAFS is currently not atomic + pass + + def test_big_file(self): + pass + +if __name__ == '__main__': + unittest.main() diff --git a/fs/contrib/tahoelafs/util.py b/fs/contrib/tahoelafs/util.py index fda8a2b..9525f6d 100644 --- a/fs/contrib/tahoelafs/util.py +++ b/fs/contrib/tahoelafs/util.py @@ -1,140 +1,140 @@ -'''
-Created on 25.9.2010
-
-@author: marekp
-'''
-
-import sys
-import platform
-import stat as statinfo
-
-import fs.errors as errors
-from fs.path import pathsplit
-try:
- # For non-CPython or older CPython versions.
- # Simplejson also comes with C speedup module which
- # is not in standard CPython >=2.6 library.
- import simplejson as json
-except ImportError:
- try:
- import json
- except ImportError:
- print "simplejson (http://pypi.python.org/pypi/simplejson/) required"
- raise
-
-from .connection import Connection
-
-python3 = int(platform.python_version_tuple()[0]) > 2
-
-if python3:
- from urllib.error import HTTPError
-else:
- from urllib2 import HTTPError
-
-class TahoeUtil:
- def __init__(self, webapi):
- self.connection = Connection(webapi)
-
- def createdircap(self):
- return self.connection.post(u'/uri', params={u't': u'mkdir'}).read()
-
- def unlink(self, dircap, path=None):
- path = self.fixwinpath(path, False)
- self.connection.delete(u'/uri/%s%s' % (dircap, path))
-
- def info(self, dircap, path):
- path = self.fixwinpath(path, False)
- meta = json.load(self.connection.get(u'/uri/%s%s' % (dircap, path), {u't': u'json'}))
- return self._info(path, meta)
-
- def fixwinpath(self, path, direction=True):
- '''
- No, Tahoe really does not support file streams...
- This is ugly hack, because it is not Tahoe-specific.
- Should be move to middleware if will be any.
- '''
- if platform.system() != 'Windows':
- return path
-
- if direction and ':' in path:
- path = path.replace(':', '__colon__')
- elif not direction and '__colon__' in path:
- path = path.replace('__colon__', ':')
- return path
-
- def _info(self, path, data):
- if isinstance(data, list):
- type = data[0]
- data = data[1]
- elif isinstance(data, dict):
- type = data['type']
- else:
- raise errors.ResourceInvalidError('Metadata in unknown format!')
-
- if type == 'unknown':
- raise errors.ResourceNotFoundError(path)
-
- info = {'name': unicode(self.fixwinpath(path, True)),
- 'type': type,
- 'size': data.get('size', 0),
- 'ctime': None,
- 'uri': data.get('rw_uri', data.get('ro_uri'))}
- if 'metadata' in data:
- info['ctime'] = data['metadata'].get('ctime')
-
- if info['type'] == 'dirnode':
- info['st_mode'] = 0777 | statinfo.S_IFDIR
- else:
- info['st_mode'] = 0644
-
- return info
-
- def list(self, dircap, path=None):
- path = self.fixwinpath(path, False)
-
- data = json.load(self.connection.get(u'/uri/%s%s' % (dircap, path), {u't': u'json'}))
-
- if len(data) < 2 or data[0] != 'dirnode':
- raise errors.ResourceInvalidError('Metadata in unknown format!')
-
- data = data[1]['children']
- for i in data.keys():
- x = self._info(i, data[i])
- yield x
-
- def mkdir(self, dircap, path):
- path = self.fixwinpath(path, False)
- path = pathsplit(path)
-
- self.connection.post(u"/uri/%s%s" % (dircap, path[0]), data={u't': u'mkdir', u'name': path[1]})
-
- def move(self, dircap, src, dst):
- if src == '/' or dst == '/':
- raise errors.UnsupportedError("Too dangerous operation, aborting")
-
- src = self.fixwinpath(src, False)
- dst = self.fixwinpath(dst, False)
-
- src_tuple = pathsplit(src)
- dst_tuple = pathsplit(dst)
-
- if src_tuple[0] == dst_tuple[0]:
- # Move inside one directory
- self.connection.post(u"/uri/%s%s" % (dircap, src_tuple[0]), data={u't': u'rename',
- u'from_name': src_tuple[1], u'to_name': dst_tuple[1]})
- return
-
- # Move to different directory. Firstly create link on dst, then remove from src
- try:
- self.info(dircap, dst)
- except errors.ResourceNotFoundError:
- pass
- else:
- self.unlink(dircap, dst)
-
- uri = self.info(dircap, src)['uri']
- self.connection.put(u"/uri/%s%s" % (dircap, dst), data=uri, params={u't': u'uri'})
- if uri != self.info(dircap, dst)['uri']:
- raise errors.OperationFailedError('Move failed')
-
- self.unlink(dircap, src)
+''' +Created on 25.9.2010 + +@author: marekp +''' + +import sys +import platform +import stat as statinfo + +import fs.errors as errors +from fs.path import pathsplit +try: + # For non-CPython or older CPython versions. + # Simplejson also comes with C speedup module which + # is not in standard CPython >=2.6 library. + import simplejson as json +except ImportError: + try: + import json + except ImportError: + print "simplejson (http://pypi.python.org/pypi/simplejson/) required" + raise + +from .connection import Connection + +python3 = int(platform.python_version_tuple()[0]) > 2 + +if python3: + from urllib.error import HTTPError +else: + from urllib2 import HTTPError + +class TahoeUtil: + def __init__(self, webapi): + self.connection = Connection(webapi) + + def createdircap(self): + return self.connection.post(u'/uri', params={u't': u'mkdir'}).read() + + def unlink(self, dircap, path=None): + path = self.fixwinpath(path, False) + self.connection.delete(u'/uri/%s%s' % (dircap, path)) + + def info(self, dircap, path): + path = self.fixwinpath(path, False) + meta = json.load(self.connection.get(u'/uri/%s%s' % (dircap, path), {u't': u'json'})) + return self._info(path, meta) + + def fixwinpath(self, path, direction=True): + ''' + No, Tahoe really does not support file streams... + This is ugly hack, because it is not Tahoe-specific. + Should be move to middleware if will be any. + ''' + if platform.system() != 'Windows': + return path + + if direction and ':' in path: + path = path.replace(':', '__colon__') + elif not direction and '__colon__' in path: + path = path.replace('__colon__', ':') + return path + + def _info(self, path, data): + if isinstance(data, list): + type = data[0] + data = data[1] + elif isinstance(data, dict): + type = data['type'] + else: + raise errors.ResourceInvalidError('Metadata in unknown format!') + + if type == 'unknown': + raise errors.ResourceNotFoundError(path) + + info = {'name': unicode(self.fixwinpath(path, True)), + 'type': type, + 'size': data.get('size', 0), + 'ctime': None, + 'uri': data.get('rw_uri', data.get('ro_uri'))} + if 'metadata' in data: + info['ctime'] = data['metadata'].get('ctime') + + if info['type'] == 'dirnode': + info['st_mode'] = 0777 | statinfo.S_IFDIR + else: + info['st_mode'] = 0644 + + return info + + def list(self, dircap, path=None): + path = self.fixwinpath(path, False) + + data = json.load(self.connection.get(u'/uri/%s%s' % (dircap, path), {u't': u'json'})) + + if len(data) < 2 or data[0] != 'dirnode': + raise errors.ResourceInvalidError('Metadata in unknown format!') + + data = data[1]['children'] + for i in data.keys(): + x = self._info(i, data[i]) + yield x + + def mkdir(self, dircap, path): + path = self.fixwinpath(path, False) + path = pathsplit(path) + + self.connection.post(u"/uri/%s%s" % (dircap, path[0]), data={u't': u'mkdir', u'name': path[1]}) + + def move(self, dircap, src, dst): + if src == '/' or dst == '/': + raise errors.UnsupportedError("Too dangerous operation, aborting") + + src = self.fixwinpath(src, False) + dst = self.fixwinpath(dst, False) + + src_tuple = pathsplit(src) + dst_tuple = pathsplit(dst) + + if src_tuple[0] == dst_tuple[0]: + # Move inside one directory + self.connection.post(u"/uri/%s%s" % (dircap, src_tuple[0]), data={u't': u'rename', + u'from_name': src_tuple[1], u'to_name': dst_tuple[1]}) + return + + # Move to different directory. Firstly create link on dst, then remove from src + try: + self.info(dircap, dst) + except errors.ResourceNotFoundError: + pass + else: + self.unlink(dircap, dst) + + uri = self.info(dircap, src)['uri'] + self.connection.put(u"/uri/%s%s" % (dircap, dst), data=uri, params={u't': u'uri'}) + if uri != self.info(dircap, dst)['uri']: + raise errors.OperationFailedError('Move failed') + + self.unlink(dircap, src) diff --git a/fs/expose/django_storage.py b/fs/expose/django_storage.py index 9a84b2a..636c3d5 100644 --- a/fs/expose/django_storage.py +++ b/fs/expose/django_storage.py @@ -28,7 +28,7 @@ class FSStorage(Storage): """ :param fs: an FS object :param base_url: The url to prepend to the path - + """ if fs is None: fs = settings.DEFAULT_FILE_STORAGE_FS diff --git a/fs/expose/fuse/__init__.py b/fs/expose/fuse/__init__.py index ed75033..60614be 100644 --- a/fs/expose/fuse/__init__.py +++ b/fs/expose/fuse/__init__.py @@ -41,7 +41,7 @@ to subprocess.Popen:: The binding to FUSE is created via ctypes, using a custom version of the fuse.py code from Giorgos Verigakis: - http://code.google.com/p/fusepy/ + https://github.com/terencehonles/fusepy """ diff --git a/fs/expose/fuse/fuse.py b/fs/expose/fuse/fuse.py index e15a5d5..ed1fbc4 100644 --- a/fs/expose/fuse/fuse.py +++ b/fs/expose/fuse/fuse.py @@ -113,7 +113,7 @@ elif _system == 'Linux': c_uid_t = c_uint setxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), c_size_t, c_int) getxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), c_size_t) - + if _machine == 'x86_64': c_stat._fields_ = [ ('st_dev', c_dev_t), @@ -294,12 +294,12 @@ class FUSE(object): """This class is the lower level interface and should not be subclassed under normal use. Its methods are called by fuse. Assumes API version 2.6 or later.""" - + def __init__(self, operations, mountpoint, raw_fi=False, **kwargs): """Setting raw_fi to True will cause FUSE to pass the fuse_file_info class as is to Operations, instead of just the fh field. This gives you access to direct_io, keep_cache, etc.""" - + self.operations = operations self.raw_fi = raw_fi args = ['fuse'] @@ -315,7 +315,7 @@ class FUSE(object): for key, val in kwargs.items())) args.append(mountpoint) argv = (c_char_p * len(args))(*args) - + fuse_ops = fuse_operations() for name, prototype in fuse_operations._fields_: if prototype != c_voidp and getattr(operations, name, None): @@ -326,7 +326,7 @@ class FUSE(object): del self.operations # Invoke the destructor if err: raise RuntimeError(err) - + def _wrapper_(self, func, *args, **kwargs): """Decorator for the methods that follow""" try: @@ -336,40 +336,40 @@ class FUSE(object): except: print_exc() return -EFAULT - + def getattr(self, path, buf): return self.fgetattr(path, buf, None) - + def readlink(self, path, buf, bufsize): ret = self.operations('readlink', path) data = create_string_buffer(ret[:bufsize - 1]) memmove(buf, data, len(data)) return 0 - + def mknod(self, path, mode, dev): return self.operations('mknod', path, mode, dev) - + def mkdir(self, path, mode): return self.operations('mkdir', path, mode) - + def unlink(self, path): return self.operations('unlink', path) - + def rmdir(self, path): return self.operations('rmdir', path) - + def symlink(self, source, target): return self.operations('symlink', target, source) - + def rename(self, old, new): return self.operations('rename', old, new) - + def link(self, source, target): return self.operations('link', target, source) - + def chmod(self, path, mode): return self.operations('chmod', path, mode) - + def chown(self, path, uid, gid): # Check if any of the arguments is a -1 that has overflowed if c_uid_t(uid + 1).value == 0: @@ -377,10 +377,10 @@ class FUSE(object): if c_gid_t(gid + 1).value == 0: gid = -1 return self.operations('chown', path, uid, gid) - + def truncate(self, path, length): return self.operations('truncate', path, length) - + def open(self, path, fip): fi = fip.contents if self.raw_fi: @@ -388,7 +388,7 @@ class FUSE(object): else: fi.fh = self.operations('open', path, fi.flags) return 0 - + def read(self, path, buf, size, offset, fip): fh = fip.contents if self.raw_fi else fip.contents.fh ret = self.operations('read', path, size, offset, fh) @@ -397,12 +397,12 @@ class FUSE(object): data = create_string_buffer(ret[:size], size) memmove(buf, data, size) return size - + def write(self, path, buf, size, offset, fip): data = string_at(buf, size) fh = fip.contents if self.raw_fi else fip.contents.fh return self.operations('write', path, data, offset, fh) - + def statfs(self, path, buf): stv = buf.contents attrs = self.operations('statfs', path) @@ -410,23 +410,23 @@ class FUSE(object): if hasattr(stv, key): setattr(stv, key, val) return 0 - + def flush(self, path, fip): fh = fip.contents if self.raw_fi else fip.contents.fh return self.operations('flush', path, fh) - + def release(self, path, fip): fh = fip.contents if self.raw_fi else fip.contents.fh return self.operations('release', path, fh) - + def fsync(self, path, datasync, fip): fh = fip.contents if self.raw_fi else fip.contents.fh return self.operations('fsync', path, datasync, fh) - + def setxattr(self, path, name, value, size, options, *args): data = string_at(value, size) return self.operations('setxattr', path, name, data, options, *args) - + def getxattr(self, path, name, value, size, *args): ret = self.operations('getxattr', path, name, *args) retsize = len(ret) @@ -436,7 +436,7 @@ class FUSE(object): return -ERANGE memmove(value, buf, retsize) return retsize - + def listxattr(self, path, namebuf, size): ret = self.operations('listxattr', path) buf = create_string_buffer('\x00'.join(ret)) if ret else '' @@ -446,15 +446,15 @@ class FUSE(object): return -ERANGE memmove(namebuf, buf, bufsize) return bufsize - + def removexattr(self, path, name): return self.operations('removexattr', path, name) - + def opendir(self, path, fip): # Ignore raw_fi fip.contents.fh = self.operations('opendir', path) return 0 - + def readdir(self, path, buf, filler, offset, fip): # Ignore raw_fi for item in self.operations('readdir', path, fip.contents.fh): @@ -470,24 +470,24 @@ class FUSE(object): if filler(buf, name, st, offset) != 0: break return 0 - + def releasedir(self, path, fip): # Ignore raw_fi return self.operations('releasedir', path, fip.contents.fh) - + def fsyncdir(self, path, datasync, fip): # Ignore raw_fi return self.operations('fsyncdir', path, datasync, fip.contents.fh) - + def init(self, conn): return self.operations('init', '/') - + def destroy(self, private_data): return self.operations('destroy', '/') - + def access(self, path, amode): return self.operations('access', path, amode) - + def create(self, path, mode, fip): fi = fip.contents if self.raw_fi: @@ -495,11 +495,11 @@ class FUSE(object): else: fi.fh = self.operations('create', path, mode) return 0 - + def ftruncate(self, path, length, fip): fh = fip.contents if self.raw_fi else fip.contents.fh return self.operations('truncate', path, length, fh) - + def fgetattr(self, path, buf, fip): memset(buf, 0, sizeof(c_stat)) st = buf.contents @@ -507,11 +507,11 @@ class FUSE(object): attrs = self.operations('getattr', path, fh) set_st_attrs(st, attrs) return 0 - + def lock(self, path, fip, cmd, lock): fh = fip.contents if self.raw_fi else fip.contents.fh return self.operations('lock', path, fh, cmd, lock) - + def utimens(self, path, buf): if buf: atime = time_of_timespec(buf.contents.actime) @@ -520,7 +520,7 @@ class FUSE(object): else: times = None return self.operations('utimens', path, times) - + def bmap(self, path, blocksize, idx): return self.operations('bmap', path, blocksize, idx) @@ -529,46 +529,46 @@ class Operations(object): """This class should be subclassed and passed as an argument to FUSE on initialization. All operations should raise a FuseOSError exception on error. - + When in doubt of what an operation should do, check the FUSE header file or the corresponding system call man page.""" - + def __call__(self, op, *args): if not hasattr(self, op): raise FuseOSError(EFAULT) return getattr(self, op)(*args) - + def access(self, path, amode): return 0 - + bmap = None - + def chmod(self, path, mode): raise FuseOSError(EROFS) - + def chown(self, path, uid, gid): raise FuseOSError(EROFS) - + def create(self, path, mode, fi=None): """When raw_fi is False (default case), fi is None and create should return a numerical file handle. When raw_fi is True the file handle should be set directly by create and return 0.""" raise FuseOSError(EROFS) - + def destroy(self, path): """Called on filesystem destruction. Path is always /""" pass - + def flush(self, path, fh): return 0 - + def fsync(self, path, datasync, fh): return 0 - + def fsyncdir(self, path, datasync, fh): return 0 - + def getattr(self, path, fh=None): """Returns a dictionary with keys identical to the stat C structure of stat(2). @@ -576,33 +576,33 @@ class Operations(object): NOTE: There is an incombatibility between Linux and Mac OS X concerning st_nlink of directories. Mac OS X counts all files inside the directory, while Linux counts only the subdirectories.""" - + if path != '/': raise FuseOSError(ENOENT) return dict(st_mode=(S_IFDIR | 0755), st_nlink=2) - + def getxattr(self, path, name, position=0): raise FuseOSError(ENOTSUP) - + def init(self, path): """Called on filesystem initialization. Path is always / Use it instead of __init__ if you start threads on initialization.""" pass - + def link(self, target, source): raise FuseOSError(EROFS) - + def listxattr(self, path): return [] - + lock = None - + def mkdir(self, path, mode): raise FuseOSError(EROFS) - + def mknod(self, path, mode, dev): raise FuseOSError(EROFS) - + def open(self, path, flags): """When raw_fi is False (default case), open should return a numerical file handle. @@ -610,60 +610,60 @@ class Operations(object): open(self, path, fi) and the file handle should be set directly.""" return 0 - + def opendir(self, path): """Returns a numerical file handle.""" return 0 - + def read(self, path, size, offset, fh): """Returns a string containing the data requested.""" raise FuseOSError(EIO) - + def readdir(self, path, fh): """Can return either a list of names, or a list of (name, attrs, offset) tuples. attrs is a dict as in getattr.""" return ['.', '..'] - + def readlink(self, path): raise FuseOSError(ENOENT) - + def release(self, path, fh): return 0 - + def releasedir(self, path, fh): return 0 - + def removexattr(self, path, name): raise FuseOSError(ENOTSUP) - + def rename(self, old, new): raise FuseOSError(EROFS) - + def rmdir(self, path): raise FuseOSError(EROFS) - + def setxattr(self, path, name, value, options, position=0): raise FuseOSError(ENOTSUP) - + def statfs(self, path): """Returns a dictionary with keys identical to the statvfs C structure of statvfs(3). On Mac OS X f_bsize and f_frsize must be a power of 2 (minimum 512).""" return {} - + def symlink(self, target, source): raise FuseOSError(EROFS) - + def truncate(self, path, length, fh=None): raise FuseOSError(EROFS) - + def unlink(self, path): raise FuseOSError(EROFS) - + def utimens(self, path, times=None): """Times is a (atime, mtime) tuple. If None use current time.""" return 0 - + def write(self, path, data, offset, fh): raise FuseOSError(EROFS) diff --git a/fs/expose/fuse/fuse_ctypes.py b/fs/expose/fuse/fuse_ctypes.py index 375a819..4e247f0 100644 --- a/fs/expose/fuse/fuse_ctypes.py +++ b/fs/expose/fuse/fuse_ctypes.py @@ -1,9 +1,9 @@ # Copyright (c) 2008 Giorgos Verigakis <verigak@gmail.com> -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR @@ -104,7 +104,7 @@ if _system in ('Darwin', 'Darwin-MacFuse', 'FreeBSD'): ('st_size', c_off_t), ('st_blocks', c_int64), ('st_blksize', c_int32)] - + elif _system == 'Linux': ENOTSUP = 95 c_dev_t = c_ulonglong @@ -117,7 +117,7 @@ elif _system == 'Linux': c_uid_t = c_uint setxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), c_size_t, c_int) getxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), c_size_t) - + _machine = machine() if _machine == 'x86_64': c_stat._fields_ = [ @@ -296,12 +296,12 @@ class FUSE(object): """This class is the lower level interface and should not be subclassed under normal use. Its methods are called by fuse. Assumes API version 2.6 or later.""" - + def __init__(self, operations, mountpoint, raw_fi=False, **kwargs): """Setting raw_fi to True will cause FUSE to pass the fuse_file_info class as is to Operations, instead of just the fh field. This gives you access to direct_io, keep_cache, etc.""" - + self.operations = operations self.raw_fi = raw_fi args = ['fuse'] @@ -317,7 +317,7 @@ class FUSE(object): for key, val in kwargs.items())) args.append(mountpoint) argv = (c_char_p * len(args))(*args) - + fuse_ops = fuse_operations() for name, prototype in fuse_operations._fields_: if prototype != c_voidp and getattr(operations, name, None): @@ -326,7 +326,7 @@ class FUSE(object): _libfuse.fuse_main_real(len(args), argv, pointer(fuse_ops), sizeof(fuse_ops), None) del self.operations # Invoke the destructor - + def _wrapper_(self, func, *args, **kwargs): """Decorator for the methods that follow""" try: @@ -336,46 +336,46 @@ class FUSE(object): except: print_exc() return -EFAULT - + def getattr(self, path, buf): return self.fgetattr(path, buf, None) - + def readlink(self, path, buf, bufsize): ret = self.operations('readlink', path) data = create_string_buffer(ret[:bufsize - 1]) memmove(buf, data, len(data)) return 0 - + def mknod(self, path, mode, dev): return self.operations('mknod', path, mode, dev) - + def mkdir(self, path, mode): return self.operations('mkdir', path, mode) - + def unlink(self, path): return self.operations('unlink', path) - + def rmdir(self, path): return self.operations('rmdir', path) - + def symlink(self, source, target): return self.operations('symlink', target, source) - + def rename(self, old, new): return self.operations('rename', old, new) - + def link(self, source, target): return self.operations('link', target, source) - + def chmod(self, path, mode): return self.operations('chmod', path, mode) - + def chown(self, path, uid, gid): return self.operations('chown', path, uid, gid) - + def truncate(self, path, length): return self.operations('truncate', path, length) - + def open(self, path, fip): fi = fip.contents if self.raw_fi: @@ -383,7 +383,7 @@ class FUSE(object): else: fi.fh = self.operations('open', path, fi.flags) return 0 - + def read(self, path, buf, size, offset, fip): fh = fip.contents if self.raw_fi else fip.contents.fh ret = self.operations('read', path, size, offset, fh) @@ -391,12 +391,12 @@ class FUSE(object): strbuf = create_string_buffer(ret) memmove(buf, strbuf, len(strbuf)) return len(ret) - + def write(self, path, buf, size, offset, fip): data = string_at(buf, size) fh = fip.contents if self.raw_fi else fip.contents.fh return self.operations('write', path, data, offset, fh) - + def statfs(self, path, buf): stv = buf.contents attrs = self.operations('statfs', path) @@ -404,23 +404,23 @@ class FUSE(object): if hasattr(stv, key): setattr(stv, key, val) return 0 - + def flush(self, path, fip): fh = fip.contents if self.raw_fi else fip.contents.fh return self.operations('flush', path, fh) - + def release(self, path, fip): fh = fip.contents if self.raw_fi else fip.contents.fh return self.operations('release', path, fh) - + def fsync(self, path, datasync, fip): fh = fip.contents if self.raw_fi else fip.contents.fh return self.operations('fsync', path, datasync, fh) - + def setxattr(self, path, name, value, size, options, *args): data = string_at(value, size) return self.operations('setxattr', path, name, data, options, *args) - + def getxattr(self, path, name, value, size, *args): ret = self.operations('getxattr', path, name, *args) retsize = len(ret) @@ -430,7 +430,7 @@ class FUSE(object): return -ERANGE memmove(value, buf, retsize) return retsize - + def listxattr(self, path, namebuf, size): ret = self.operations('listxattr', path) if ret: @@ -443,15 +443,15 @@ class FUSE(object): return -ERANGE memmove(namebuf, buf, bufsize) return bufsize - + def removexattr(self, path, name): return self.operations('removexattr', path, name) - + def opendir(self, path, fip): # Ignore raw_fi fip.contents.fh = self.operations('opendir', path) return 0 - + def readdir(self, path, buf, filler, offset, fip): # Ignore raw_fi for item in self.operations('readdir', path, fip.contents.fh): @@ -467,24 +467,24 @@ class FUSE(object): if filler(buf, name, st, offset) != 0: break return 0 - + def releasedir(self, path, fip): # Ignore raw_fi return self.operations('releasedir', path, fip.contents.fh) - + def fsyncdir(self, path, datasync, fip): # Ignore raw_fi return self.operations('fsyncdir', path, datasync, fip.contents.fh) - + def init(self, conn): return self.operations('init', '/') - + def destroy(self, private_data): return self.operations('destroy', '/') - + def access(self, path, amode): return self.operations('access', path, amode) - + def create(self, path, mode, fip): fi = fip.contents if self.raw_fi: @@ -492,11 +492,11 @@ class FUSE(object): else: fi.fh = self.operations('create', path, mode) return 0 - + def ftruncate(self, path, length, fip): fh = fip.contents if self.raw_fi else fip.contents.fh return self.operations('truncate', path, length, fh) - + def fgetattr(self, path, buf, fip): memset(buf, 0, sizeof(c_stat)) st = buf.contents @@ -504,11 +504,11 @@ class FUSE(object): attrs = self.operations('getattr', path, fh) set_st_attrs(st, attrs) return 0 - + def lock(self, path, fip, cmd, lock): fh = fip.contents if self.raw_fi else fip.contents.fh return self.operations('lock', path, fh, cmd, lock) - + def utimens(self, path, buf): if buf: atime = time_of_timespec(buf.contents.actime) @@ -517,7 +517,7 @@ class FUSE(object): else: times = None return self.operations('utimens', path, times) - + def bmap(self, path, blocksize, idx): return self.operations('bmap', path, blocksize, idx) @@ -526,46 +526,46 @@ class Operations(object): """This class should be subclassed and passed as an argument to FUSE on initialization. All operations should raise an OSError exception on error. - + When in doubt of what an operation should do, check the FUSE header file or the corresponding system call man page.""" - + def __call__(self, op, *args): if not hasattr(self, op): raise OSError(EFAULT, '') return getattr(self, op)(*args) - + def access(self, path, amode): return 0 - + bmap = None - + def chmod(self, path, mode): raise OSError(EROFS, '') - + def chown(self, path, uid, gid): raise OSError(EROFS, '') - + def create(self, path, mode, fi=None): """When raw_fi is False (default case), fi is None and create should return a numerical file handle. When raw_fi is True the file handle should be set directly by create and return 0.""" raise OSError(EROFS, '') - + def destroy(self, path): """Called on filesystem destruction. Path is always /""" pass - + def flush(self, path, fh): return 0 - + def fsync(self, path, datasync, fh): return 0 - + def fsyncdir(self, path, datasync, fh): return 0 - + def getattr(self, path, fh=None): """Returns a dictionary with keys identical to the stat C structure of stat(2). @@ -573,33 +573,33 @@ class Operations(object): NOTE: There is an incombatibility between Linux and Mac OS X concerning st_nlink of directories. Mac OS X counts all files inside the directory, while Linux counts only the subdirectories.""" - + if path != '/': raise OSError(ENOENT, '') return dict(st_mode=(S_IFDIR | 0755), st_nlink=2) - + def getxattr(self, path, name, position=0): raise OSError(ENOTSUP, '') - + def init(self, path): """Called on filesystem initialization. Path is always / Use it instead of __init__ if you start threads on initialization.""" pass - + def link(self, target, source): raise OSError(EROFS, '') - + def listxattr(self, path): return [] - + lock = None - + def mkdir(self, path, mode): raise OSError(EROFS, '') - + def mknod(self, path, mode, dev): raise OSError(EROFS, '') - + def open(self, path, flags): """When raw_fi is False (default case), open should return a numerical file handle. @@ -607,60 +607,60 @@ class Operations(object): open(self, path, fi) and the file handle should be set directly.""" return 0 - + def opendir(self, path): """Returns a numerical file handle.""" return 0 - + def read(self, path, size, offset, fh): """Returns a string containing the data requested.""" raise OSError(ENOENT, '') - + def readdir(self, path, fh): """Can return either a list of names, or a list of (name, attrs, offset) tuples. attrs is a dict as in getattr.""" return ['.', '..'] - + def readlink(self, path): raise OSError(ENOENT, '') - + def release(self, path, fh): return 0 - + def releasedir(self, path, fh): return 0 - + def removexattr(self, path, name): raise OSError(ENOTSUP, '') - + def rename(self, old, new): raise OSError(EROFS, '') - + def rmdir(self, path): raise OSError(EROFS, '') - + def setxattr(self, path, name, value, options, position=0): raise OSError(ENOTSUP, '') - + def statfs(self, path): """Returns a dictionary with keys identical to the statvfs C structure of statvfs(3). On Mac OS X f_bsize and f_frsize must be a power of 2 (minimum 512).""" return {} - + def symlink(self, target, source): raise OSError(EROFS, '') - + def truncate(self, path, length, fh=None): raise OSError(EROFS, '') - + def unlink(self, path): raise OSError(EROFS, '') - + def utimens(self, path, times=None): """Times is a (atime, mtime) tuple. If None use current time.""" return 0 - + def write(self, path, data, offset, fh): raise OSError(EROFS, '') diff --git a/fs/expose/importhook.py b/fs/expose/importhook.py index e3d2d8e..cc8837f 100644 --- a/fs/expose/importhook.py +++ b/fs/expose/importhook.py @@ -240,4 +240,4 @@ class FSImportHook(object): info = self._get_module_info(fullname) (path,type,ispkg) = info return path - + diff --git a/fs/expose/serve/packetstream.py b/fs/expose/serve/packetstream.py index 6f24fd9..0449937 100644 --- a/fs/expose/serve/packetstream.py +++ b/fs/expose/serve/packetstream.py @@ -12,21 +12,21 @@ def encode(header='', payload=''): def textsize(s): if s: return str(len(s)) - return '' + return '' return '%i,%i:%s%s' % (textsize(header), textsize(payload), header, payload) class FileEncoder(object): - + def __init__(self, f): self.f = f - + def write(self, header='', payload=''): fwrite = self.f.write def textsize(s): if s: return str(len(s)) - return '' + return '' fwrite('%s,%s:' % (textsize(header), textsize(payload))) if header: fwrite(header) @@ -35,11 +35,11 @@ class FileEncoder(object): class JSONFileEncoder(FileEncoder): - + def write(self, header=None, payload=''): if header is None: super(JSONFileEncoder, self).write('', payload) - else: + else: header_json = dumps(header, separators=(',', ':')) super(JSONFileEncoder, self).write(header_json, payload) @@ -51,12 +51,12 @@ class PreludeError(DecoderError): pass class Decoder(object): - + STAGE_PRELUDE, STAGE_SIZE, STAGE_HEADER, STAGE_PAYLOAD = range(4) MAX_PRELUDE = 255 - + def __init__(self, no_prelude=False, prelude_callback=None): - + self.prelude_callback = prelude_callback self.stream_broken = False self.expecting_bytes = None @@ -64,49 +64,49 @@ class Decoder(object): self._prelude = [] self._size = [] self._expecting_bytes = None - + self.header_size = None self.payload_size = None - + self._header_bytes = None self._payload_bytes = None - + self._header_data = [] self._payload_data = [] - + self.header = None self.payload = None - + if no_prelude: self.stage = self.STAGE_SIZE - - + + def feed(self, data): - + if self.stream_broken: raise DecoderError('Stream is broken') - + STAGE_PRELUDE, STAGE_SIZE, STAGE_HEADER, STAGE_PAYLOAD = range(4) - + size_append = self._size.append header_append = self._header_data.append payload_append = self._payload_data.append datafind = data.find - + def reset_packet(): self.expecting_bytes = None del self._header_data[:] del self._payload_data[:] self.header = None self.payload = None - + data_len = len(data) data_pos = 0 - expecting_bytes = self.expecting_bytes + expecting_bytes = self.expecting_bytes stage = self.stage - + if stage == STAGE_PRELUDE: - max_find = min(len(data), data_pos + self.MAX_PRELUDE) + max_find = min(len(data), data_pos + self.MAX_PRELUDE) cr_pos = datafind('\n', data_pos, max_find) if cr_pos == -1: self._prelude.append(data[data_pos:]) @@ -119,53 +119,53 @@ class Decoder(object): if sum(len(s) for s in self._prelude) > self.MAX_PRELUDE: self.stream_broken = True raise PreludeError('Prelude not found') - data_pos = cr_pos + 1 + data_pos = cr_pos + 1 prelude = ''.join(self._prelude) del self._prelude[:] reset_packet() if not self.on_prelude(prelude): self.broken = True return - stage = STAGE_SIZE - + stage = STAGE_SIZE + while data_pos < data_len: - + if stage == STAGE_HEADER: bytes_to_read = min(data_len - data_pos, expecting_bytes) header_append(data[data_pos:data_pos + bytes_to_read]) data_pos += bytes_to_read - expecting_bytes -= bytes_to_read - if not expecting_bytes: - self.header = ''.join(self._header_data) + expecting_bytes -= bytes_to_read + if not expecting_bytes: + self.header = ''.join(self._header_data) if not self.payload_size: yield self.header, '' reset_packet() expecting_bytes = None stage = STAGE_SIZE - else: + else: stage = STAGE_PAYLOAD expecting_bytes = self.payload_size - + elif stage == STAGE_PAYLOAD: - bytes_to_read = min(data_len - data_pos, expecting_bytes) + bytes_to_read = min(data_len - data_pos, expecting_bytes) payload_append(data[data_pos:data_pos + bytes_to_read]) data_pos += bytes_to_read - expecting_bytes -= bytes_to_read - if not expecting_bytes: + expecting_bytes -= bytes_to_read + if not expecting_bytes: self.payload = ''.join(self._payload_data) yield self.header, self.payload reset_packet() stage = STAGE_SIZE expecting_bytes = None - + elif stage == STAGE_SIZE: term_pos = datafind(':', data_pos) if term_pos == -1: - size_append(data[data_pos:]) + size_append(data[data_pos:]) break else: size_append(data[data_pos:term_pos]) - data_pos = term_pos + 1 + data_pos = term_pos + 1 size = ''.join(self._size) del self._size[:] @@ -173,30 +173,30 @@ class Decoder(object): header_size, payload_size = size.split(',', 1) else: header_size = size - payload_size = '' + payload_size = '' try: self.header_size = int(header_size or '0') self.payload_size = int(payload_size or '0') except ValueError: self.stream_broken = False raise DecoderError('Invalid size in packet (%s)' % size) - + if self.header_size: - expecting_bytes = self.header_size + expecting_bytes = self.header_size stage = STAGE_HEADER elif self.payload_size: - expecting_bytes = self.payload_size + expecting_bytes = self.payload_size stage = STAGE_PAYLOAD else: # A completely empty packet, permitted, if a little odd yield '', '' - reset_packet() + reset_packet() expecting_bytes = None - self.expecting_bytes = expecting_bytes + self.expecting_bytes = expecting_bytes self.stage = stage - - + + def on_prelude(self, prelude): if self.prelude_callback and not self.prelude_callback(self, prelude): return False @@ -206,7 +206,7 @@ class Decoder(object): class JSONDecoder(Decoder): - + def feed(self, data): for header, payload in Decoder.feed(self, data): if header: @@ -215,9 +215,9 @@ class JSONDecoder(Decoder): header = {} yield header, payload - + if __name__ == "__main__": - + f = StringIO() encoder = JSONFileEncoder(f) encoder.write(dict(a=1, b=2), 'Payload') @@ -225,29 +225,29 @@ if __name__ == "__main__": encoder.write(None, 'Payload') encoder.write(dict(a=1)) encoder.write() - + stream = 'prelude\n' + f.getvalue() - + #print stream - + # packets = ['Prelude string\n', # encode('header', 'payload'), # encode('header number 2', 'second payload'), # encode('', '')] -# +# # stream = ''.join(packets) - + decoder = JSONDecoder() - + stream = 'pyfs/0.1\n59,13:{"type":"rpc","method":"ping","client_ref":"-1221142848:1"}Hello, World!' - + fdata = StringIO(stream) - + while 1: data = fdata.read(3) if not data: break for header, payload in decoder.feed(data): print "Header:", repr(header) - print "Payload:", repr(payload) -
\ No newline at end of file + print "Payload:", repr(payload) + diff --git a/fs/expose/serve/server.py b/fs/expose/serve/server.py index 7ac8c1f..d870175 100644 --- a/fs/expose/serve/server.py +++ b/fs/expose/serve/server.py @@ -9,24 +9,24 @@ from packetstream import JSONDecoder, JSONFileEncoder class _SocketFile(object): def __init__(self, socket): self.socket = socket - + def read(self, size): try: return self.socket.recv(size) except socket.error: return '' - + def write(self, data): self.socket.sendall(data) def remote_call(method_name=None): - method = method_name + method = method_name def deco(f): if not hasattr(f, '_remote_call_names'): - f._remote_call_names = [] + f._remote_call_names = [] f._remote_call_names.append(method or f.__name__) - return f + return f return deco @@ -36,34 +36,34 @@ class RemoteResponse(Exception): self.payload = payload class ConnectionHandlerBase(threading.Thread): - + _methods = {} - + def __init__(self, server, connection_id, socket, address): super(ConnectionHandlerBase, self).__init__() self.server = server - self.connection_id = connection_id + self.connection_id = connection_id self.socket = socket self.transport = _SocketFile(socket) - self.address = address + self.address = address self.encoder = JSONFileEncoder(self.transport) - self.decoder = JSONDecoder(prelude_callback=self.on_stream_prelude) - + self.decoder = JSONDecoder(prelude_callback=self.on_stream_prelude) + self._lock = threading.RLock() self.socket_error = None - - if not self._methods: + + if not self._methods: for method_name in dir(self): method = getattr(self, method_name) if callable(method) and hasattr(method, '_remote_call_names'): for name in method._remote_call_names: - + self._methods[name] = method - + print self._methods - - self.fs = None - + + self.fs = None + def run(self): self.transport.write('pyfs/1.0\n') while True: @@ -74,27 +74,27 @@ class ConnectionHandlerBase(threading.Thread): self.socket_error = socket_error break print "data", repr(data) - if data: - for packet in self.decoder.feed(data): + if data: + for packet in self.decoder.feed(data): print repr(packet) self.on_packet(*packet) else: - break + break self.on_connection_close() - + def close(self): with self._lock: self.socket.close() - + def on_connection_close(self): self.socket.shutdown(socket.SHUT_RDWR) self.socket.close() self.server.on_connection_close(self.connection_id) - + def on_stream_prelude(self, packet_stream, prelude): print "prelude", prelude return True - + def on_packet(self, header, payload): print '-' * 30 print repr(header) @@ -111,7 +111,7 @@ class ConnectionHandlerBase(threading.Thread): remote['response'] = response self.encoder.write(remote, '') except RemoteResponse, response: - self.encoder.write(response.header, response.payload) + self.encoder.write(response.header, response.payload) class RemoteFSConnection(ConnectionHandlerBase): @@ -122,9 +122,9 @@ class RemoteFSConnection(ConnectionHandlerBase): self.resource = resource from fs.memoryfs import MemoryFS self.fs = MemoryFS() - + class Server(object): - + def __init__(self, addr='', port=3000, connection_factory=RemoteFSConnection): self.addr = addr self.port = port @@ -133,41 +133,41 @@ class Server(object): self.connection_id = 0 self.threads = {} self._lock = threading.RLock() - + def serve_forever(self): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((self.addr, self.port)) - + sock.bind((self.addr, self.port)) + sock.listen(5) - + try: while True: clientsocket, address = sock.accept() self.on_connect(clientsocket, address) except KeyboardInterrupt: pass - + try: self._close_graceful() except KeyboardInterrupt: self._close_harsh() - + def _close_graceful(self): """Tell all threads to exit and wait for them""" with self._lock: for connection in self.threads.itervalues(): - connection.close() + connection.close() for connection in self.threads.itervalues(): connection.join() self.threads.clear() - + def _close_harsh(self): with self._lock: for connection in self.threads.itervalues(): connection.close() self.threads.clear() - + def on_connect(self, clientsocket, address): print "Connection from", address with self._lock: @@ -175,17 +175,17 @@ class Server(object): thread = self.connection_factory(self, self.connection_id, clientsocket, - address) - self.threads[self.connection_id] = thread - thread.start() - + address) + self.threads[self.connection_id] = thread + thread.start() + def on_connection_close(self, connection_id): pass #with self._lock: # self.threads[connection_id].join() # del self.threads[connection_id] - + if __name__ == "__main__": server = Server() server.serve_forever() -
\ No newline at end of file + diff --git a/fs/expose/serve/threadpool.py b/fs/expose/serve/threadpool.py index f448a12..5c2ecd5 100644 --- a/fs/expose/serve/threadpool.py +++ b/fs/expose/serve/threadpool.py @@ -10,48 +10,48 @@ def make_job(job_callable, *args, **kwargs): class _PoolThread(threading.Thread): """ Internal thread class that runs jobs. """ - + def __init__(self, queue, name): super(_PoolThread, self).__init__() self.queue = queue self.name = name - + def __str__(self): return self.name - + def run(self): - + while True: try: - _priority, job = self.queue.get() - except queue.Empty: + _priority, job = self.queue.get() + except queue.Empty: break - - if job is None: + + if job is None: break - + if callable(job): try: - job() + job() except Exception, e: - print e + print e self.queue.task_done() - + class ThreadPool(object): - + def __init__(self, num_threads, size=None, name=''): - + self.num_threads = num_threads self.name = name self.queue = queue.PriorityQueue(size) self.job_no = 0 - + self.threads = [_PoolThread(self.queue, '%s #%i' % (name, i)) for i in xrange(num_threads)] - + for thread in self.threads: thread.start() - + def _make_priority_key(self, i): no = self.job_no self.job_no += 1 @@ -62,38 +62,38 @@ class ThreadPool(object): def job(): return job_callable(*args, **kwargs) self.queue.put( (self._make_priority_key(1), job), True ) - return self.job_no - + return self.job_no + def flush_quit(self): - """ Quit after all tasks on the queue have been processed. """ + """ Quit after all tasks on the queue have been processed. """ for thread in self.threads: - self.queue.put( (self._make_priority_key(1), None) ) + self.queue.put( (self._make_priority_key(1), None) ) for thread in self.threads: thread.join() - + def quit(self): """ Quit as soon as possible, potentially leaving tasks on the queue. """ for thread in self.threads: - self.queue.put( (self._make_priority_key(0), None) ) + self.queue.put( (self._make_priority_key(0), None) ) for thread in self.threads: thread.join() if __name__ == "__main__": import time - + def job(n): print "Starting #%i" % n time.sleep(1) print "Ending #%i" % n - + pool = ThreadPool(5, 'test thread') - + for n in range(20): pool.job(job, n) - + pool.flush_quit() - -
\ No newline at end of file + + diff --git a/fs/memoryfs.py b/fs/memoryfs.py index cde0096..5688c40 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -1,694 +1,694 @@ -#!/usr/bin/env python
-"""
-fs.memoryfs
-===========
-
-A Filesystem that exists in memory only. Which makes them extremely fast, but non-permanent.
-
-If you open a file from a `memoryfs` you will get back a StringIO object from the standard library.
-
-
-"""
-
-import datetime
-import stat
-from fs.path import iteratepath, pathsplit, normpath
-from fs.base import *
-from fs.errors import *
-from fs import _thread_synchronize_default
-from fs.filelike import StringIO
-from fs import iotools
-from os import SEEK_END
-import threading
-
-import six
-from six import b
-
-
-def _check_mode(mode, mode_chars):
- for c in mode_chars:
- if c not in mode:
- return False
- return True
-
-
-class MemoryFile(object):
-
- def seek_and_lock(f):
- def deco(self, *args, **kwargs):
- try:
- self._lock.acquire()
- self.mem_file.seek(self.pos)
- ret = f(self, *args, **kwargs)
- self.pos = self.mem_file.tell()
- return ret
- finally:
- self._lock.release()
- return deco
-
- def __init__(self, path, memory_fs, mem_file, mode, lock):
- self.closed = False
- self.path = path
- self.memory_fs = memory_fs
- self.mem_file = mem_file
- self.mode = mode
- self._lock = lock
-
- self.pos = 0
-
- if _check_mode(mode, 'a'):
- lock.acquire()
- try:
- self.mem_file.seek(0, SEEK_END)
- self.pos = self.mem_file.tell()
- finally:
- lock.release()
-
- elif _check_mode(mode, 'w'):
- lock.acquire()
- try:
- self.mem_file.seek(0)
- self.mem_file.truncate()
- finally:
- lock.release()
-
- assert self.mem_file is not None, "self.mem_file should have a value"
-
- def __str__(self):
- return "<MemoryFile in %s %s>" % (self.memory_fs, self.path)
-
- def __repr__(self):
- return u"<MemoryFile in %s %s>" % (self.memory_fs, self.path)
-
- def __unicode__(self):
- return u"<MemoryFile in %s %s>" % (self.memory_fs, self.path)
-
- def __del__(self):
- if not self.closed:
- self.close()
-
- def flush(self):
- pass
-
- def __iter__(self):
- if 'r' not in self.mode and '+' not in self.mode:
- raise IOError("File not open for reading")
- self.mem_file.seek(self.pos)
- for line in self.mem_file:
- yield line
-
- @seek_and_lock
- def next(self):
- if 'r' not in self.mode and '+' not in self.mode:
- raise IOError("File not open for reading")
- return self.mem_file.next()
-
- @seek_and_lock
- def readline(self, *args, **kwargs):
- if 'r' not in self.mode and '+' not in self.mode:
- raise IOError("File not open for reading")
- return self.mem_file.readline(*args, **kwargs)
-
- def close(self):
- do_close = False
- self._lock.acquire()
- try:
- do_close = not self.closed and self.mem_file is not None
- if do_close:
- self.closed = True
- finally:
- self._lock.release()
- if do_close:
- self.memory_fs._on_close_memory_file(self, self.path)
-
- @seek_and_lock
- def read(self, size=None):
- if 'r' not in self.mode and '+' not in self.mode:
- raise IOError("File not open for reading")
- if size is None:
- size = -1
- return self.mem_file.read(size)
-
- @seek_and_lock
- def seek(self, *args, **kwargs):
- return self.mem_file.seek(*args, **kwargs)
-
- @seek_and_lock
- def tell(self):
- return self.pos
-
- @seek_and_lock
- def truncate(self, *args, **kwargs):
- if 'r' in self.mode and '+' not in self.mode:
- raise IOError("File not open for writing")
- return self.mem_file.truncate(*args, **kwargs)
-
- #@seek_and_lock
- def write(self, data):
- if 'r' in self.mode and '+' not in self.mode:
- raise IOError("File not open for writing")
- self.memory_fs._on_modify_memory_file(self.path)
- self._lock.acquire()
- try:
- self.mem_file.seek(self.pos)
- self.mem_file.write(data)
- self.pos = self.mem_file.tell()
- finally:
- self._lock.release()
-
- @seek_and_lock
- def writelines(self, *args, **kwargs):
- return self.mem_file.writelines(*args, **kwargs)
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, traceback):
- self.close()
- return False
-
-
-class DirEntry(object):
-
- def sync(f):
- def deco(self, *args, **kwargs):
- if self.lock is not None:
- try:
- self.lock.acquire()
- return f(self, *args, **kwargs)
- finally:
- self.lock.release()
- else:
- return f(self, *args, **kwargs)
- return deco
-
- def __init__(self, type, name, contents=None):
-
- assert type in ("dir", "file"), "Type must be dir or file!"
-
- self.type = type
- self.name = name
-
- if contents is None and type == "dir":
- contents = {}
-
- self.open_files = []
- self.contents = contents
- self.mem_file = None
- self.created_time = datetime.datetime.now()
- self.modified_time = self.created_time
- self.accessed_time = self.created_time
-
- self.xattrs = {}
-
- self.lock = None
- if self.type == 'file':
- self.mem_file = StringIO()
- self.lock = threading.RLock()
-
- def get_value(self):
- self.lock.acquire()
- try:
- return self.mem_file.getvalue()
- finally:
- self.lock.release()
- data = property(get_value)
-
- def desc_contents(self):
- if self.isfile():
- return "<file %s>" % self.name
- elif self.isdir():
- return "<dir %s>" % "".join("%s: %s" % (k, v.desc_contents()) for k, v in self.contents.iteritems())
-
- def isdir(self):
- return self.type == "dir"
-
- def isfile(self):
- return self.type == "file"
-
- def __str__(self):
- return "%s: %s" % (self.name, self.desc_contents())
-
- @sync
- def __getstate__(self):
- state = self.__dict__.copy()
- state.pop('lock')
- if self.mem_file is not None:
- state['mem_file'] = self.data
- return state
-
- def __setstate__(self, state):
- self.__dict__.update(state)
- if self.type == 'file':
- self.lock = threading.RLock()
- else:
- self.lock = None
- if self.mem_file is not None:
- data = self.mem_file
- self.mem_file = StringIO()
- self.mem_file.write(data)
-
-
-class MemoryFS(FS):
- """An in-memory filesystem.
-
- """
-
- _meta = {'thread_safe': True,
- 'network': False,
- 'virtual': False,
- 'read_only': False,
- 'unicode_paths': True,
- 'case_insensitive_paths': False,
- 'atomic.move': False,
- 'atomic.copy': False,
- 'atomic.makedir': True,
- 'atomic.rename': True,
- 'atomic.setcontents': False}
-
- def _make_dir_entry(self, *args, **kwargs):
- return self.dir_entry_factory(*args, **kwargs)
-
- def __init__(self, file_factory=None):
- super(MemoryFS, self).__init__(thread_synchronize=_thread_synchronize_default)
-
- self.dir_entry_factory = DirEntry
- self.file_factory = file_factory or MemoryFile
- if not callable(self.file_factory):
- raise ValueError("file_factory should be callable")
-
- self.root = self._make_dir_entry('dir', 'root')
-
- def __str__(self):
- return "<MemoryFS>"
-
- def __repr__(self):
- return "MemoryFS()"
-
- def __unicode__(self):
- return "<MemoryFS>"
-
- @synchronize
- def _get_dir_entry(self, dirpath):
- dirpath = normpath(dirpath)
- current_dir = self.root
- for path_component in iteratepath(dirpath):
- if current_dir.contents is None:
- return None
- dir_entry = current_dir.contents.get(path_component, None)
- if dir_entry is None:
- return None
- current_dir = dir_entry
- return current_dir
-
- @synchronize
- def _dir_entry(self, path):
- dir_entry = self._get_dir_entry(path)
- if dir_entry is None:
- raise ResourceNotFoundError(path)
- return dir_entry
-
- @synchronize
- def desc(self, path):
- if self.isdir(path):
- return "Memory dir"
- elif self.isfile(path):
- return "Memory file object"
- else:
- return "No description available"
-
- @synchronize
- def isdir(self, path):
- path = normpath(path)
- if path in ('', '/'):
- return True
- dir_item = self._get_dir_entry(path)
- if dir_item is None:
- return False
- return dir_item.isdir()
-
- @synchronize
- def isfile(self, path):
- path = normpath(path)
- if path in ('', '/'):
- return False
- dir_item = self._get_dir_entry(path)
- if dir_item is None:
- return False
- return dir_item.isfile()
-
- @synchronize
- def exists(self, path):
- path = normpath(path)
- if path in ('', '/'):
- return True
- return self._get_dir_entry(path) is not None
-
- @synchronize
- def makedir(self, dirname, recursive=False, allow_recreate=False):
- if not dirname and not allow_recreate:
- raise PathError(dirname)
- fullpath = normpath(dirname)
- if fullpath in ('', '/'):
- if allow_recreate:
- return
- raise DestinationExistsError(dirname)
- dirpath, dirname = pathsplit(dirname.rstrip('/'))
-
- if recursive:
- parent_dir = self._get_dir_entry(dirpath)
- if parent_dir is not None:
- if parent_dir.isfile():
- raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")
- else:
- if not allow_recreate:
- if dirname in parent_dir.contents:
- raise DestinationExistsError(dirname, msg="Can not create a directory that already exists (try allow_recreate=True): %(path)s")
-
- current_dir = self.root
- for path_component in iteratepath(dirpath)[:-1]:
- dir_item = current_dir.contents.get(path_component, None)
- if dir_item is None:
- break
- if not dir_item.isdir():
- raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")
- current_dir = dir_item
-
- current_dir = self.root
- for path_component in iteratepath(dirpath):
- dir_item = current_dir.contents.get(path_component, None)
- if dir_item is None:
- new_dir = self._make_dir_entry("dir", path_component)
- current_dir.contents[path_component] = new_dir
- current_dir = new_dir
- else:
- current_dir = dir_item
-
- parent_dir = current_dir
-
- else:
- parent_dir = self._get_dir_entry(dirpath)
- if parent_dir is None:
- raise ParentDirectoryMissingError(dirname, msg="Could not make dir, as parent dir does not exist: %(path)s")
-
- dir_item = parent_dir.contents.get(dirname, None)
- if dir_item is not None:
- if dir_item.isdir():
- if not allow_recreate:
- raise DestinationExistsError(dirname)
- else:
- raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")
-
- if dir_item is None:
- parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname)
-
-
- #@synchronize
- #def _orphan_files(self, file_dir_entry):
- # for f in file_dir_entry.open_files[:]:
- # f.close()
-
-
- @synchronize
- @iotools.filelike_to_stream
- def open(self, path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, line_buffering=False, **kwargs):
- path = normpath(path)
- filepath, filename = pathsplit(path)
- parent_dir_entry = self._get_dir_entry(filepath)
-
- if parent_dir_entry is None or not parent_dir_entry.isdir():
- raise ResourceNotFoundError(path)
-
- if 'r' in mode or 'a' in mode:
- if filename not in parent_dir_entry.contents:
- raise ResourceNotFoundError(path)
-
- file_dir_entry = parent_dir_entry.contents[filename]
- if file_dir_entry.isdir():
- raise ResourceInvalidError(path)
-
- file_dir_entry.accessed_time = datetime.datetime.now()
-
- mem_file = self.file_factory(path, self, file_dir_entry.mem_file, mode, file_dir_entry.lock)
- file_dir_entry.open_files.append(mem_file)
- return mem_file
-
- elif 'w' in mode:
- if filename not in parent_dir_entry.contents:
- file_dir_entry = self._make_dir_entry("file", filename)
- parent_dir_entry.contents[filename] = file_dir_entry
- else:
- file_dir_entry = parent_dir_entry.contents[filename]
-
- file_dir_entry.accessed_time = datetime.datetime.now()
-
- mem_file = self.file_factory(path, self, file_dir_entry.mem_file, mode, file_dir_entry.lock)
- file_dir_entry.open_files.append(mem_file)
- return mem_file
-
- if parent_dir_entry is None:
- raise ResourceNotFoundError(path)
-
- @synchronize
- def remove(self, path):
- dir_entry = self._get_dir_entry(path)
-
- if dir_entry is None:
- raise ResourceNotFoundError(path)
-
- if dir_entry.isdir():
- raise ResourceInvalidError(path, msg="That's a directory, not a file: %(path)s")
-
- pathname, dirname = pathsplit(path)
- parent_dir = self._get_dir_entry(pathname)
- del parent_dir.contents[dirname]
-
- @synchronize
- def removedir(self, path, recursive=False, force=False):
- path = normpath(path)
- if path in ('', '/'):
- raise RemoveRootError(path)
- dir_entry = self._get_dir_entry(path)
-
- if dir_entry is None:
- raise ResourceNotFoundError(path)
- if not dir_entry.isdir():
- raise ResourceInvalidError(path, msg="Can't remove resource, its not a directory: %(path)s" )
-
- if dir_entry.contents and not force:
- raise DirectoryNotEmptyError(path)
-
- if recursive:
- rpathname = path
- while rpathname:
- rpathname, dirname = pathsplit(rpathname)
- parent_dir = self._get_dir_entry(rpathname)
- if not dirname:
- raise RemoveRootError(path)
- del parent_dir.contents[dirname]
- # stop recursing if the directory has other contents
- if parent_dir.contents:
- break
- else:
- pathname, dirname = pathsplit(path)
- parent_dir = self._get_dir_entry(pathname)
- if not dirname:
- raise RemoveRootError(path)
- del parent_dir.contents[dirname]
-
- @synchronize
- def rename(self, src, dst):
- src = normpath(src)
- dst = normpath(dst)
- src_dir, src_name = pathsplit(src)
- src_entry = self._get_dir_entry(src)
- if src_entry is None:
- raise ResourceNotFoundError(src)
- open_files = src_entry.open_files[:]
- for f in open_files:
- f.flush()
- f.path = dst
-
- dst_dir,dst_name = pathsplit(dst)
- dst_entry = self._get_dir_entry(dst)
- if dst_entry is not None:
- raise DestinationExistsError(dst)
-
- src_dir_entry = self._get_dir_entry(src_dir)
- src_xattrs = src_dir_entry.xattrs.copy()
- dst_dir_entry = self._get_dir_entry(dst_dir)
- if dst_dir_entry is None:
- raise ParentDirectoryMissingError(dst)
- dst_dir_entry.contents[dst_name] = src_dir_entry.contents[src_name]
- dst_dir_entry.contents[dst_name].name = dst_name
- dst_dir_entry.xattrs.update(src_xattrs)
- del src_dir_entry.contents[src_name]
-
- @synchronize
- def settimes(self, path, accessed_time=None, modified_time=None):
- now = datetime.datetime.now()
- if accessed_time is None:
- accessed_time = now
- if modified_time is None:
- modified_time = now
-
- dir_entry = self._get_dir_entry(path)
- if dir_entry is not None:
- dir_entry.accessed_time = accessed_time
- dir_entry.modified_time = modified_time
- return True
- return False
-
- @synchronize
- def _on_close_memory_file(self, open_file, path):
- dir_entry = self._get_dir_entry(path)
- if dir_entry is not None and open_file in dir_entry.open_files:
- dir_entry.open_files.remove(open_file)
-
-
- @synchronize
- def _on_modify_memory_file(self, path):
- dir_entry = self._get_dir_entry(path)
- if dir_entry is not None:
- dir_entry.modified_time = datetime.datetime.now()
-
- @synchronize
- def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
- dir_entry = self._get_dir_entry(path)
- if dir_entry is None:
- raise ResourceNotFoundError(path)
- if dir_entry.isfile():
- raise ResourceInvalidError(path, msg="not a directory: %(path)s")
- paths = dir_entry.contents.keys()
- for (i,p) in enumerate(paths):
- if not isinstance(p,unicode):
- paths[i] = unicode(p)
- return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only)
-
- @synchronize
- def getinfo(self, path):
- dir_entry = self._get_dir_entry(path)
-
- if dir_entry is None:
- raise ResourceNotFoundError(path)
-
- info = {}
- info['created_time'] = dir_entry.created_time
- info['modified_time'] = dir_entry.modified_time
- info['accessed_time'] = dir_entry.accessed_time
-
- if dir_entry.isdir():
- info['st_mode'] = 0755 | stat.S_IFDIR
- else:
- info['size'] = len(dir_entry.data or b(''))
- info['st_mode'] = 0666 | stat.S_IFREG
-
- return info
-
- @synchronize
- def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=1024*64):
- src_dir_entry = self._get_dir_entry(src)
- if src_dir_entry is None:
- raise ResourceNotFoundError(src)
- src_xattrs = src_dir_entry.xattrs.copy()
- super(MemoryFS, self).copydir(src, dst, overwrite, ignore_errors=ignore_errors, chunk_size=chunk_size)
- dst_dir_entry = self._get_dir_entry(dst)
- if dst_dir_entry is not None:
- dst_dir_entry.xattrs.update(src_xattrs)
-
- @synchronize
- def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=1024*64):
- src_dir_entry = self._get_dir_entry(src)
- if src_dir_entry is None:
- raise ResourceNotFoundError(src)
- src_xattrs = src_dir_entry.xattrs.copy()
- super(MemoryFS, self).movedir(src, dst, overwrite, ignore_errors=ignore_errors, chunk_size=chunk_size)
- dst_dir_entry = self._get_dir_entry(dst)
- if dst_dir_entry is not None:
- dst_dir_entry.xattrs.update(src_xattrs)
-
- @synchronize
- def copy(self, src, dst, overwrite=False, chunk_size=1024*64):
- src_dir_entry = self._get_dir_entry(src)
- if src_dir_entry is None:
- raise ResourceNotFoundError(src)
- src_xattrs = src_dir_entry.xattrs.copy()
- super(MemoryFS, self).copy(src, dst, overwrite, chunk_size)
- dst_dir_entry = self._get_dir_entry(dst)
- if dst_dir_entry is not None:
- dst_dir_entry.xattrs.update(src_xattrs)
-
- @synchronize
- def move(self, src, dst, overwrite=False, chunk_size=1024*64):
- src_dir_entry = self._get_dir_entry(src)
- if src_dir_entry is None:
- raise ResourceNotFoundError(src)
- src_xattrs = src_dir_entry.xattrs.copy()
- super(MemoryFS, self).move(src, dst, overwrite, chunk_size)
- dst_dir_entry = self._get_dir_entry(dst)
- if dst_dir_entry is not None:
- dst_dir_entry.xattrs.update(src_xattrs)
-
- @synchronize
- def getcontents(self, path, mode="rb", encoding=None, errors=None, newline=None):
- dir_entry = self._get_dir_entry(path)
- if dir_entry is None:
- raise ResourceNotFoundError(path)
- if not dir_entry.isfile():
- raise ResourceInvalidError(path, msg="not a file: %(path)s")
- data = dir_entry.data or b('')
- if 'b' not in mode:
- return iotools.decode_binary(data, encoding=encoding, errors=errors, newline=newline)
- return data
-
- @synchronize
- def setcontents(self, path, data=b'', encoding=None, errors=None, chunk_size=1024*64):
- if isinstance(data, six.binary_type):
- if not self.exists(path):
- self.open(path, 'wb').close()
- dir_entry = self._get_dir_entry(path)
- if not dir_entry.isfile():
- raise ResourceInvalidError('Not a directory %(path)s', path)
- new_mem_file = StringIO()
- new_mem_file.write(data)
- dir_entry.mem_file = new_mem_file
- return len(data)
-
- return super(MemoryFS, self).setcontents(path, data=data, encoding=encoding, errors=errors, chunk_size=chunk_size)
-
- # if isinstance(data, six.text_type):
- # return super(MemoryFS, self).setcontents(path, data, encoding=encoding, errors=errors, chunk_size=chunk_size)
- # if not self.exists(path):
- # self.open(path, 'wb').close()
-
- # dir_entry = self._get_dir_entry(path)
- # if not dir_entry.isfile():
- # raise ResourceInvalidError('Not a directory %(path)s', path)
- # new_mem_file = StringIO()
- # new_mem_file.write(data)
- # dir_entry.mem_file = new_mem_file
-
- @synchronize
- def setxattr(self, path, key, value):
- dir_entry = self._dir_entry(path)
- key = unicode(key)
- dir_entry.xattrs[key] = value
-
- @synchronize
- def getxattr(self, path, key, default=None):
- key = unicode(key)
- dir_entry = self._dir_entry(path)
- return dir_entry.xattrs.get(key, default)
-
- @synchronize
- def delxattr(self, path, key):
- dir_entry = self._dir_entry(path)
- try:
- del dir_entry.xattrs[key]
- except KeyError:
- pass
-
- @synchronize
- def listxattrs(self, path):
- dir_entry = self._dir_entry(path)
- return dir_entry.xattrs.keys()
+#!/usr/bin/env python +""" +fs.memoryfs +=========== + +A Filesystem that exists in memory only. Which makes them extremely fast, but non-permanent. + +If you open a file from a `memoryfs` you will get back a StringIO object from the standard library. + + +""" + +import datetime +import stat +from fs.path import iteratepath, pathsplit, normpath +from fs.base import * +from fs.errors import * +from fs import _thread_synchronize_default +from fs.filelike import StringIO +from fs import iotools +from os import SEEK_END +import threading + +import six +from six import b + + +def _check_mode(mode, mode_chars): + for c in mode_chars: + if c not in mode: + return False + return True + + +class MemoryFile(object): + + def seek_and_lock(f): + def deco(self, *args, **kwargs): + try: + self._lock.acquire() + self.mem_file.seek(self.pos) + ret = f(self, *args, **kwargs) + self.pos = self.mem_file.tell() + return ret + finally: + self._lock.release() + return deco + + def __init__(self, path, memory_fs, mem_file, mode, lock): + self.closed = False + self.path = path + self.memory_fs = memory_fs + self.mem_file = mem_file + self.mode = mode + self._lock = lock + + self.pos = 0 + + if _check_mode(mode, 'a'): + lock.acquire() + try: + self.mem_file.seek(0, SEEK_END) + self.pos = self.mem_file.tell() + finally: + lock.release() + + elif _check_mode(mode, 'w'): + lock.acquire() + try: + self.mem_file.seek(0) + self.mem_file.truncate() + finally: + lock.release() + + assert self.mem_file is not None, "self.mem_file should have a value" + + def __str__(self): + return "<MemoryFile in %s %s>" % (self.memory_fs, self.path) + + def __repr__(self): + return u"<MemoryFile in %s %s>" % (self.memory_fs, self.path) + + def __unicode__(self): + return u"<MemoryFile in %s %s>" % (self.memory_fs, self.path) + + def __del__(self): + if not self.closed: + self.close() + + def flush(self): + pass + + def __iter__(self): + if 'r' not in self.mode and '+' not in self.mode: + raise IOError("File not open for reading") + self.mem_file.seek(self.pos) + for line in self.mem_file: + yield line + + @seek_and_lock + def next(self): + if 'r' not in self.mode and '+' not in self.mode: + raise IOError("File not open for reading") + return self.mem_file.next() + + @seek_and_lock + def readline(self, *args, **kwargs): + if 'r' not in self.mode and '+' not in self.mode: + raise IOError("File not open for reading") + return self.mem_file.readline(*args, **kwargs) + + def close(self): + do_close = False + self._lock.acquire() + try: + do_close = not self.closed and self.mem_file is not None + if do_close: + self.closed = True + finally: + self._lock.release() + if do_close: + self.memory_fs._on_close_memory_file(self, self.path) + + @seek_and_lock + def read(self, size=None): + if 'r' not in self.mode and '+' not in self.mode: + raise IOError("File not open for reading") + if size is None: + size = -1 + return self.mem_file.read(size) + + @seek_and_lock + def seek(self, *args, **kwargs): + return self.mem_file.seek(*args, **kwargs) + + @seek_and_lock + def tell(self): + return self.pos + + @seek_and_lock + def truncate(self, *args, **kwargs): + if 'r' in self.mode and '+' not in self.mode: + raise IOError("File not open for writing") + return self.mem_file.truncate(*args, **kwargs) + + #@seek_and_lock + def write(self, data): + if 'r' in self.mode and '+' not in self.mode: + raise IOError("File not open for writing") + self.memory_fs._on_modify_memory_file(self.path) + self._lock.acquire() + try: + self.mem_file.seek(self.pos) + self.mem_file.write(data) + self.pos = self.mem_file.tell() + finally: + self._lock.release() + + @seek_and_lock + def writelines(self, *args, **kwargs): + return self.mem_file.writelines(*args, **kwargs) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + return False + + +class DirEntry(object): + + def sync(f): + def deco(self, *args, **kwargs): + if self.lock is not None: + try: + self.lock.acquire() + return f(self, *args, **kwargs) + finally: + self.lock.release() + else: + return f(self, *args, **kwargs) + return deco + + def __init__(self, type, name, contents=None): + + assert type in ("dir", "file"), "Type must be dir or file!" + + self.type = type + self.name = name + + if contents is None and type == "dir": + contents = {} + + self.open_files = [] + self.contents = contents + self.mem_file = None + self.created_time = datetime.datetime.now() + self.modified_time = self.created_time + self.accessed_time = self.created_time + + self.xattrs = {} + + self.lock = None + if self.type == 'file': + self.mem_file = StringIO() + self.lock = threading.RLock() + + def get_value(self): + self.lock.acquire() + try: + return self.mem_file.getvalue() + finally: + self.lock.release() + data = property(get_value) + + def desc_contents(self): + if self.isfile(): + return "<file %s>" % self.name + elif self.isdir(): + return "<dir %s>" % "".join("%s: %s" % (k, v.desc_contents()) for k, v in self.contents.iteritems()) + + def isdir(self): + return self.type == "dir" + + def isfile(self): + return self.type == "file" + + def __str__(self): + return "%s: %s" % (self.name, self.desc_contents()) + + @sync + def __getstate__(self): + state = self.__dict__.copy() + state.pop('lock') + if self.mem_file is not None: + state['mem_file'] = self.data + return state + + def __setstate__(self, state): + self.__dict__.update(state) + if self.type == 'file': + self.lock = threading.RLock() + else: + self.lock = None + if self.mem_file is not None: + data = self.mem_file + self.mem_file = StringIO() + self.mem_file.write(data) + + +class MemoryFS(FS): + """An in-memory filesystem. + + """ + + _meta = {'thread_safe': True, + 'network': False, + 'virtual': False, + 'read_only': False, + 'unicode_paths': True, + 'case_insensitive_paths': False, + 'atomic.move': False, + 'atomic.copy': False, + 'atomic.makedir': True, + 'atomic.rename': True, + 'atomic.setcontents': False} + + def _make_dir_entry(self, *args, **kwargs): + return self.dir_entry_factory(*args, **kwargs) + + def __init__(self, file_factory=None): + super(MemoryFS, self).__init__(thread_synchronize=_thread_synchronize_default) + + self.dir_entry_factory = DirEntry + self.file_factory = file_factory or MemoryFile + if not callable(self.file_factory): + raise ValueError("file_factory should be callable") + + self.root = self._make_dir_entry('dir', 'root') + + def __str__(self): + return "<MemoryFS>" + + def __repr__(self): + return "MemoryFS()" + + def __unicode__(self): + return "<MemoryFS>" + + @synchronize + def _get_dir_entry(self, dirpath): + dirpath = normpath(dirpath) + current_dir = self.root + for path_component in iteratepath(dirpath): + if current_dir.contents is None: + return None + dir_entry = current_dir.contents.get(path_component, None) + if dir_entry is None: + return None + current_dir = dir_entry + return current_dir + + @synchronize + def _dir_entry(self, path): + dir_entry = self._get_dir_entry(path) + if dir_entry is None: + raise ResourceNotFoundError(path) + return dir_entry + + @synchronize + def desc(self, path): + if self.isdir(path): + return "Memory dir" + elif self.isfile(path): + return "Memory file object" + else: + return "No description available" + + @synchronize + def isdir(self, path): + path = normpath(path) + if path in ('', '/'): + return True + dir_item = self._get_dir_entry(path) + if dir_item is None: + return False + return dir_item.isdir() + + @synchronize + def isfile(self, path): + path = normpath(path) + if path in ('', '/'): + return False + dir_item = self._get_dir_entry(path) + if dir_item is None: + return False + return dir_item.isfile() + + @synchronize + def exists(self, path): + path = normpath(path) + if path in ('', '/'): + return True + return self._get_dir_entry(path) is not None + + @synchronize + def makedir(self, dirname, recursive=False, allow_recreate=False): + if not dirname and not allow_recreate: + raise PathError(dirname) + fullpath = normpath(dirname) + if fullpath in ('', '/'): + if allow_recreate: + return + raise DestinationExistsError(dirname) + dirpath, dirname = pathsplit(dirname.rstrip('/')) + + if recursive: + parent_dir = self._get_dir_entry(dirpath) + if parent_dir is not None: + if parent_dir.isfile(): + raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s") + else: + if not allow_recreate: + if dirname in parent_dir.contents: + raise DestinationExistsError(dirname, msg="Can not create a directory that already exists (try allow_recreate=True): %(path)s") + + current_dir = self.root + for path_component in iteratepath(dirpath)[:-1]: + dir_item = current_dir.contents.get(path_component, None) + if dir_item is None: + break + if not dir_item.isdir(): + raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s") + current_dir = dir_item + + current_dir = self.root + for path_component in iteratepath(dirpath): + dir_item = current_dir.contents.get(path_component, None) + if dir_item is None: + new_dir = self._make_dir_entry("dir", path_component) + current_dir.contents[path_component] = new_dir + current_dir = new_dir + else: + current_dir = dir_item + + parent_dir = current_dir + + else: + parent_dir = self._get_dir_entry(dirpath) + if parent_dir is None: + raise ParentDirectoryMissingError(dirname, msg="Could not make dir, as parent dir does not exist: %(path)s") + + dir_item = parent_dir.contents.get(dirname, None) + if dir_item is not None: + if dir_item.isdir(): + if not allow_recreate: + raise DestinationExistsError(dirname) + else: + raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s") + + if dir_item is None: + parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname) + + + #@synchronize + #def _orphan_files(self, file_dir_entry): + # for f in file_dir_entry.open_files[:]: + # f.close() + + + @synchronize + @iotools.filelike_to_stream + def open(self, path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, line_buffering=False, **kwargs): + path = normpath(path) + filepath, filename = pathsplit(path) + parent_dir_entry = self._get_dir_entry(filepath) + + if parent_dir_entry is None or not parent_dir_entry.isdir(): + raise ResourceNotFoundError(path) + + if 'r' in mode or 'a' in mode: + if filename not in parent_dir_entry.contents: + raise ResourceNotFoundError(path) + + file_dir_entry = parent_dir_entry.contents[filename] + if file_dir_entry.isdir(): + raise ResourceInvalidError(path) + + file_dir_entry.accessed_time = datetime.datetime.now() + + mem_file = self.file_factory(path, self, file_dir_entry.mem_file, mode, file_dir_entry.lock) + file_dir_entry.open_files.append(mem_file) + return mem_file + + elif 'w' in mode: + if filename not in parent_dir_entry.contents: + file_dir_entry = self._make_dir_entry("file", filename) + parent_dir_entry.contents[filename] = file_dir_entry + else: + file_dir_entry = parent_dir_entry.contents[filename] + + file_dir_entry.accessed_time = datetime.datetime.now() + + mem_file = self.file_factory(path, self, file_dir_entry.mem_file, mode, file_dir_entry.lock) + file_dir_entry.open_files.append(mem_file) + return mem_file + + if parent_dir_entry is None: + raise ResourceNotFoundError(path) + + @synchronize + def remove(self, path): + dir_entry = self._get_dir_entry(path) + + if dir_entry is None: + raise ResourceNotFoundError(path) + + if dir_entry.isdir(): + raise ResourceInvalidError(path, msg="That's a directory, not a file: %(path)s") + + pathname, dirname = pathsplit(path) + parent_dir = self._get_dir_entry(pathname) + del parent_dir.contents[dirname] + + @synchronize + def removedir(self, path, recursive=False, force=False): + path = normpath(path) + if path in ('', '/'): + raise RemoveRootError(path) + dir_entry = self._get_dir_entry(path) + + if dir_entry is None: + raise ResourceNotFoundError(path) + if not dir_entry.isdir(): + raise ResourceInvalidError(path, msg="Can't remove resource, its not a directory: %(path)s" ) + + if dir_entry.contents and not force: + raise DirectoryNotEmptyError(path) + + if recursive: + rpathname = path + while rpathname: + rpathname, dirname = pathsplit(rpathname) + parent_dir = self._get_dir_entry(rpathname) + if not dirname: + raise RemoveRootError(path) + del parent_dir.contents[dirname] + # stop recursing if the directory has other contents + if parent_dir.contents: + break + else: + pathname, dirname = pathsplit(path) + parent_dir = self._get_dir_entry(pathname) + if not dirname: + raise RemoveRootError(path) + del parent_dir.contents[dirname] + + @synchronize + def rename(self, src, dst): + src = normpath(src) + dst = normpath(dst) + src_dir, src_name = pathsplit(src) + src_entry = self._get_dir_entry(src) + if src_entry is None: + raise ResourceNotFoundError(src) + open_files = src_entry.open_files[:] + for f in open_files: + f.flush() + f.path = dst + + dst_dir,dst_name = pathsplit(dst) + dst_entry = self._get_dir_entry(dst) + if dst_entry is not None: + raise DestinationExistsError(dst) + + src_dir_entry = self._get_dir_entry(src_dir) + src_xattrs = src_dir_entry.xattrs.copy() + dst_dir_entry = self._get_dir_entry(dst_dir) + if dst_dir_entry is None: + raise ParentDirectoryMissingError(dst) + dst_dir_entry.contents[dst_name] = src_dir_entry.contents[src_name] + dst_dir_entry.contents[dst_name].name = dst_name + dst_dir_entry.xattrs.update(src_xattrs) + del src_dir_entry.contents[src_name] + + @synchronize + def settimes(self, path, accessed_time=None, modified_time=None): + now = datetime.datetime.now() + if accessed_time is None: + accessed_time = now + if modified_time is None: + modified_time = now + + dir_entry = self._get_dir_entry(path) + if dir_entry is not None: + dir_entry.accessed_time = accessed_time + dir_entry.modified_time = modified_time + return True + return False + + @synchronize + def _on_close_memory_file(self, open_file, path): + dir_entry = self._get_dir_entry(path) + if dir_entry is not None and open_file in dir_entry.open_files: + dir_entry.open_files.remove(open_file) + + + @synchronize + def _on_modify_memory_file(self, path): + dir_entry = self._get_dir_entry(path) + if dir_entry is not None: + dir_entry.modified_time = datetime.datetime.now() + + @synchronize + def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False): + dir_entry = self._get_dir_entry(path) + if dir_entry is None: + raise ResourceNotFoundError(path) + if dir_entry.isfile(): + raise ResourceInvalidError(path, msg="not a directory: %(path)s") + paths = dir_entry.contents.keys() + for (i,p) in enumerate(paths): + if not isinstance(p,unicode): + paths[i] = unicode(p) + return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only) + + @synchronize + def getinfo(self, path): + dir_entry = self._get_dir_entry(path) + + if dir_entry is None: + raise ResourceNotFoundError(path) + + info = {} + info['created_time'] = dir_entry.created_time + info['modified_time'] = dir_entry.modified_time + info['accessed_time'] = dir_entry.accessed_time + + if dir_entry.isdir(): + info['st_mode'] = 0755 | stat.S_IFDIR + else: + info['size'] = len(dir_entry.data or b('')) + info['st_mode'] = 0666 | stat.S_IFREG + + return info + + @synchronize + def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=1024*64): + src_dir_entry = self._get_dir_entry(src) + if src_dir_entry is None: + raise ResourceNotFoundError(src) + src_xattrs = src_dir_entry.xattrs.copy() + super(MemoryFS, self).copydir(src, dst, overwrite, ignore_errors=ignore_errors, chunk_size=chunk_size) + dst_dir_entry = self._get_dir_entry(dst) + if dst_dir_entry is not None: + dst_dir_entry.xattrs.update(src_xattrs) + + @synchronize + def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=1024*64): + src_dir_entry = self._get_dir_entry(src) + if src_dir_entry is None: + raise ResourceNotFoundError(src) + src_xattrs = src_dir_entry.xattrs.copy() + super(MemoryFS, self).movedir(src, dst, overwrite, ignore_errors=ignore_errors, chunk_size=chunk_size) + dst_dir_entry = self._get_dir_entry(dst) + if dst_dir_entry is not None: + dst_dir_entry.xattrs.update(src_xattrs) + + @synchronize + def copy(self, src, dst, overwrite=False, chunk_size=1024*64): + src_dir_entry = self._get_dir_entry(src) + if src_dir_entry is None: + raise ResourceNotFoundError(src) + src_xattrs = src_dir_entry.xattrs.copy() + super(MemoryFS, self).copy(src, dst, overwrite, chunk_size) + dst_dir_entry = self._get_dir_entry(dst) + if dst_dir_entry is not None: + dst_dir_entry.xattrs.update(src_xattrs) + + @synchronize + def move(self, src, dst, overwrite=False, chunk_size=1024*64): + src_dir_entry = self._get_dir_entry(src) + if src_dir_entry is None: + raise ResourceNotFoundError(src) + src_xattrs = src_dir_entry.xattrs.copy() + super(MemoryFS, self).move(src, dst, overwrite, chunk_size) + dst_dir_entry = self._get_dir_entry(dst) + if dst_dir_entry is not None: + dst_dir_entry.xattrs.update(src_xattrs) + + @synchronize + def getcontents(self, path, mode="rb", encoding=None, errors=None, newline=None): + dir_entry = self._get_dir_entry(path) + if dir_entry is None: + raise ResourceNotFoundError(path) + if not dir_entry.isfile(): + raise ResourceInvalidError(path, msg="not a file: %(path)s") + data = dir_entry.data or b('') + if 'b' not in mode: + return iotools.decode_binary(data, encoding=encoding, errors=errors, newline=newline) + return data + + @synchronize + def setcontents(self, path, data=b'', encoding=None, errors=None, chunk_size=1024*64): + if isinstance(data, six.binary_type): + if not self.exists(path): + self.open(path, 'wb').close() + dir_entry = self._get_dir_entry(path) + if not dir_entry.isfile(): + raise ResourceInvalidError('Not a directory %(path)s', path) + new_mem_file = StringIO() + new_mem_file.write(data) + dir_entry.mem_file = new_mem_file + return len(data) + + return super(MemoryFS, self).setcontents(path, data=data, encoding=encoding, errors=errors, chunk_size=chunk_size) + + # if isinstance(data, six.text_type): + # return super(MemoryFS, self).setcontents(path, data, encoding=encoding, errors=errors, chunk_size=chunk_size) + # if not self.exists(path): + # self.open(path, 'wb').close() + + # dir_entry = self._get_dir_entry(path) + # if not dir_entry.isfile(): + # raise ResourceInvalidError('Not a directory %(path)s', path) + # new_mem_file = StringIO() + # new_mem_file.write(data) + # dir_entry.mem_file = new_mem_file + + @synchronize + def setxattr(self, path, key, value): + dir_entry = self._dir_entry(path) + key = unicode(key) + dir_entry.xattrs[key] = value + + @synchronize + def getxattr(self, path, key, default=None): + key = unicode(key) + dir_entry = self._dir_entry(path) + return dir_entry.xattrs.get(key, default) + + @synchronize + def delxattr(self, path, key): + dir_entry = self._dir_entry(path) + try: + del dir_entry.xattrs[key] + except KeyError: + pass + + @synchronize + def listxattrs(self, path): + dir_entry = self._dir_entry(path) + return dir_entry.xattrs.keys() diff --git a/fs/remotefs.py b/fs/remotefs.py index 77fdfc2..529a8f9 100644 --- a/fs/remotefs.py +++ b/fs/remotefs.py @@ -14,20 +14,20 @@ from six import b class PacketHandler(threading.Thread): - + def __init__(self, transport, prelude_callback=None): super(PacketHandler, self).__init__() self.transport = transport self.encoder = packetstream.JSONFileEncoder(transport) self.decoder = packetstream.JSONDecoder(prelude_callback=None) - - self.queues = defaultdict(queue.Queue) + + self.queues = defaultdict(queue.Queue) self._encoder_lock = threading.Lock() self._queues_lock = threading.Lock() self._call_id_lock = threading.Lock() - + self.call_id = 0 - + def run(self): decoder = self.decoder read = self.transport.read @@ -37,96 +37,96 @@ class PacketHandler(threading.Thread): if not data: print "No data" break - print "data", repr(data) + print "data", repr(data) for header, payload in decoder.feed(data): print repr(header) print repr(payload) on_packet(header, payload) - + def _new_call_id(self): with self._call_id_lock: self.call_id += 1 return self.call_id - + def get_thread_queue(self, queue_id=None): if queue_id is None: queue_id = threading.current_thread().ident with self._queues_lock: return self.queues[queue_id] - + def send_packet(self, header, payload=''): - call_id = self._new_call_id() - queue_id = threading.current_thread().ident - client_ref = "%i:%i" % (queue_id, call_id) + call_id = self._new_call_id() + queue_id = threading.current_thread().ident + client_ref = "%i:%i" % (queue_id, call_id) header['client_ref'] = client_ref - + with self._encoder_lock: self.encoder.write(header, payload) - + return call_id - + def get_packet(self, call_id): - + if call_id is not None: - queue_id = threading.current_thread().ident + queue_id = threading.current_thread().ident client_ref = "%i:%i" % (queue_id, call_id) else: client_ref = None - + queue = self.get_thread_queue() - - while True: + + while True: header, payload = queue.get() print repr(header) print repr(payload) if client_ref is not None and header.get('client_ref') != client_ref: continue - break - + break + return header, payload - + def on_packet(self, header, payload): client_ref = header.get('client_ref', '') queue_id, call_id = client_ref.split(':', 1) queue_id = int(queue_id) #queue_id = header.get('queue_id', '') - queue = self.get_thread_queue(queue_id) + queue = self.get_thread_queue(queue_id) queue.put((header, payload)) - - + + class _SocketFile(object): def __init__(self, socket): self.socket = socket - + def read(self, size): try: return self.socket.recv(size) - except: - return b('') - + except: + return b('') + def write(self, data): self.socket.sendall(data) - + def close(self): self.socket.shutdown(socket.SHUT_RDWR) self.socket.close() - + class _RemoteFile(object): - + def __init__(self, path, connection): self.path = path - self.connection = connection + self.connection = connection class RemoteFS(FS): - + _meta = { 'thead_safe' : True, 'network' : True, 'virtual' : False, 'read_only' : False, 'unicode_paths' : True, } - + def __init__(self, addr='', port=3000, username=None, password=None, resource=None, transport=None): self.addr = addr self.port = port @@ -136,56 +136,56 @@ class RemoteFS(FS): self.transport = transport if self.transport is None: self.transport = self._open_connection() - self.packet_handler = PacketHandler(self.transport) - self.packet_handler.start() - + self.packet_handler = PacketHandler(self.transport) + self.packet_handler.start() + self._remote_call('auth', username=username, password=password, resource=resource) - + def _open_connection(self): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((self.addr, self.port)) socket_file = _SocketFile(sock) socket_file.write(b('pyfs/0.1\n')) return socket_file - + def _make_call(self, method_name, *args, **kwargs): call = dict(type='rpc', method=method_name, args=args, kwargs=kwargs) - return call - + return call + def _remote_call(self, method_name, *args, **kwargs): call = self._make_call(method_name, *args, **kwargs) call_id = self.packet_handler.send_packet(call) header, payload = self.packet_handler.get_packet(call_id) return header, payload - + def ping(self, msg): call_id = self.packet_handler.send_packet({'type':'rpc', 'method':'ping'}, msg) header, payload = self.packet_handler.get_packet(call_id) print "PING" print header print payload - + def close(self): self.transport.close() self.packet_handler.join() - + def open(self, path, mode="r", **kwargs): pass - + def exists(self, path): remote = self._remote_call('exists', path) return remote.get('response') - - + + if __name__ == "__main__": - - rfs = RemoteFS() + + rfs = RemoteFS() rfs.close() - + diff --git a/fs/tests/__init__.py b/fs/tests/__init__.py index 2d9625d..ae83726 100644 --- a/fs/tests/__init__.py +++ b/fs/tests/__init__.py @@ -531,7 +531,8 @@ class FSTestCases(object): self.fs.setcontents("info.txt", test_str) info = self.fs.getinfo("info.txt") for k, v in info.iteritems(): - self.assertEqual(self.fs.getinfokeys('info.txt', k), {k: v}) + if not (k == 'asbytes' and callable(v)): + self.assertEqual(self.fs.getinfokeys('info.txt', k), {k: v}) test_info = {} if 'modified_time' in info: @@ -836,27 +837,18 @@ class FSTestCases(object): self.assertEquals(f.read(), b("\x00")*25 + b("EOF")) def test_with_statement(self): - # This is a little tricky since 'with' is actually new syntax. - # We use eval() to make this method safe for old python versions. - import sys - if sys.version_info[0] >= 2 and sys.version_info[1] >= 5: - # A successful 'with' statement - contents = "testing the with statement" - code = "from __future__ import with_statement\n" - code += "with self.fs.open('f.txt','wb-') as testfile:\n" - code += " testfile.write(contents)\n" - code += "self.assertEquals(self.fs.getcontents('f.txt', 'rb'),contents)" - code = compile(code, "<string>", 'exec') - eval(code) - # A 'with' statement raising an error - contents = "testing the with statement" - code = "from __future__ import with_statement\n" - code += "with self.fs.open('f.txt','wb-') as testfile:\n" - code += " testfile.write(contents)\n" - code += " raise ValueError\n" - code = compile(code, "<string>", 'exec') - self.assertRaises(ValueError, eval, code, globals(), locals()) - self.assertEquals(self.fs.getcontents('f.txt', 'rb'), contents) + contents = b"testing the with statement" + # A successful 'with' statement + with self.fs.open('f.txt','wb-') as testfile: + testfile.write(contents) + self.assertEquals(self.fs.getcontents('f.txt', 'rb'), contents) + # A 'with' statement raising an error + def with_error(): + with self.fs.open('g.txt','wb-') as testfile: + testfile.write(contents) + raise ValueError + self.assertRaises(ValueError, with_error) + self.assertEquals(self.fs.getcontents('g.txt', 'rb'), contents) def test_pickling(self): if self.fs.getmeta('pickle_contents', True): diff --git a/fs/tests/test_ftpfs.py b/fs/tests/test_ftpfs.py index d37e5df..a9cd0b5 100644 --- a/fs/tests/test_ftpfs.py +++ b/fs/tests/test_ftpfs.py @@ -21,7 +21,7 @@ try: from pyftpdlib.servers import FTPServer except ImportError: if not PY3: - raise ImportError("Requires pyftpdlib <http://code.google.com/p/pyftpdlib/>") + raise ImportError("Requires pyftpdlib <https://github.com/giampaolo/pyftpdlib>") from fs.path import * diff --git a/fs/tests/test_importhook.py b/fs/tests/test_importhook.py index b04a86c..e0d1ad7 100644 --- a/fs/tests/test_importhook.py +++ b/fs/tests/test_importhook.py @@ -23,8 +23,8 @@ class TestFSImportHook(unittest.TestCase): if isinstance(mph,FSImportHook): sys.meta_path.remove(mph) for ph in list(sys.path_hooks): - if issubclass(ph,FSImportHook): - sys.path_hooks.remove(mph) + if isinstance(ph, type) and issubclass(ph,FSImportHook): + sys.path_hooks.remove(ph) for (k,v) in sys.modules.items(): if k.startswith("fsih_"): del sys.modules[k] diff --git a/fs/tests/test_multifs.py b/fs/tests/test_multifs.py index 59b272a..2d70f82 100644 --- a/fs/tests/test_multifs.py +++ b/fs/tests/test_multifs.py @@ -5,9 +5,9 @@ import unittest from six import b class TestMultiFS(unittest.TestCase): - + def test_auto_close(self): - """Test MultiFS auto close is working""" + """Test MultiFS auto close is working""" multi_fs = MultiFS() m1 = MemoryFS() m2 = MemoryFS() @@ -18,7 +18,7 @@ class TestMultiFS(unittest.TestCase): multi_fs.close() self.assert_(m1.closed) self.assert_(m2.closed) - + def test_no_auto_close(self): """Test MultiFS auto close can be disables""" multi_fs = MultiFS(auto_close=False) @@ -31,8 +31,8 @@ class TestMultiFS(unittest.TestCase): multi_fs.close() self.assert_(not m1.closed) self.assert_(not m2.closed) - - + + def test_priority(self): """Test priority order is working""" m1 = MemoryFS() @@ -46,7 +46,7 @@ class TestMultiFS(unittest.TestCase): multi_fs.addfs("m2", m2) multi_fs.addfs("m3", m3) self.assert_(multi_fs.getcontents("name") == b("m3")) - + m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() @@ -57,8 +57,8 @@ class TestMultiFS(unittest.TestCase): multi_fs.addfs("m1", m1) multi_fs.addfs("m2", m2, priority=10) multi_fs.addfs("m3", m3) - self.assert_(multi_fs.getcontents("name") == b("m2")) - + self.assert_(multi_fs.getcontents("name") == b("m2")) + m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() @@ -70,7 +70,7 @@ class TestMultiFS(unittest.TestCase): multi_fs.addfs("m2", m2, priority=10) multi_fs.addfs("m3", m3, priority=10) self.assert_(multi_fs.getcontents("name") == b("m3")) - + m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() @@ -82,4 +82,4 @@ class TestMultiFS(unittest.TestCase): multi_fs.addfs("m2", m2, priority=10) multi_fs.addfs("m3", m3, priority=10) self.assert_(multi_fs.getcontents("name") == b("m1")) - + diff --git a/fs/tests/test_s3fs.py b/fs/tests/test_s3fs.py index cbc5f92..7b0e51c 100644 --- a/fs/tests/test_s3fs.py +++ b/fs/tests/test_s3fs.py @@ -17,8 +17,8 @@ from six import PY3 try: from fs import s3fs except ImportError: - raise unittest.SkipTest("s3fs wasn't importable") - + raise unittest.SkipTest("s3fs wasn't importable") + class TestS3FS(unittest.TestCase,FSTestCases,ThreadingTestCases): @@ -27,7 +27,7 @@ class TestS3FS(unittest.TestCase,FSTestCases,ThreadingTestCases): bucket = "test-s3fs.rfk.id.au" - def setUp(self): + def setUp(self): self.fs = s3fs.S3FS(self.bucket) for k in self.fs._s3bukt.list(): self.fs._s3bukt.delete_key(k) diff --git a/fs/tests/test_sqlitefs.py b/fs/tests/test_sqlitefs.py index 14f5a55..f0a8ca9 100644 --- a/fs/tests/test_sqlitefs.py +++ b/fs/tests/test_sqlitefs.py @@ -10,8 +10,8 @@ import os if SqliteFS: class TestSqliteFS(unittest.TestCase, FSTestCases): def setUp(self): - self.fs = SqliteFS("sqlitefs.db") + self.fs = SqliteFS("sqlitefs.db") def tearDown(self): os.remove('sqlitefs.db') - -
\ No newline at end of file + + diff --git a/fs/tests/test_utils.py b/fs/tests/test_utils.py index d2e435f..f0d336c 100644 --- a/fs/tests/test_utils.py +++ b/fs/tests/test_utils.py @@ -7,14 +7,14 @@ from fs import utils from six import b class TestUtils(unittest.TestCase): - + def _make_fs(self, fs): fs.setcontents("f1", b("file 1")) fs.setcontents("f2", b("file 2")) fs.setcontents("f3", b("file 3")) fs.makedir("foo/bar", recursive=True) fs.setcontents("foo/bar/fruit", b("apple")) - + def _check_fs(self, fs): self.assert_(fs.isfile("f1")) self.assert_(fs.isfile("f2")) @@ -24,72 +24,72 @@ class TestUtils(unittest.TestCase): self.assertEqual(fs.getcontents("f1", "rb"), b("file 1")) self.assertEqual(fs.getcontents("f2", "rb"), b("file 2")) self.assertEqual(fs.getcontents("f3", "rb"), b("file 3")) - self.assertEqual(fs.getcontents("foo/bar/fruit", "rb"), b("apple")) - + self.assertEqual(fs.getcontents("foo/bar/fruit", "rb"), b("apple")) + def test_copydir_root(self): """Test copydir from root""" fs1 = MemoryFS() - self._make_fs(fs1) + self._make_fs(fs1) fs2 = MemoryFS() - utils.copydir(fs1, fs2) + utils.copydir(fs1, fs2) self._check_fs(fs2) - + fs1 = TempFS() - self._make_fs(fs1) + self._make_fs(fs1) fs2 = TempFS() - utils.copydir(fs1, fs2) + utils.copydir(fs1, fs2) self._check_fs(fs2) - + def test_copydir_indir(self): - """Test copydir in a directory""" + """Test copydir in a directory""" fs1 = MemoryFS() fs2 = MemoryFS() - self._make_fs(fs1) - utils.copydir(fs1, (fs2, "copy")) + self._make_fs(fs1) + utils.copydir(fs1, (fs2, "copy")) self._check_fs(fs2.opendir("copy")) fs1 = TempFS() fs2 = TempFS() - self._make_fs(fs1) - utils.copydir(fs1, (fs2, "copy")) + self._make_fs(fs1) + utils.copydir(fs1, (fs2, "copy")) self._check_fs(fs2.opendir("copy")) - + def test_movedir_indir(self): - """Test movedir in a directory""" + """Test movedir in a directory""" fs1 = MemoryFS() fs2 = MemoryFS() fs1sub = fs1.makeopendir("from") - self._make_fs(fs1sub) - utils.movedir((fs1, "from"), (fs2, "copy")) - self.assert_(not fs1.exists("from")) + self._make_fs(fs1sub) + utils.movedir((fs1, "from"), (fs2, "copy")) + self.assert_(not fs1.exists("from")) self._check_fs(fs2.opendir("copy")) fs1 = TempFS() fs2 = TempFS() fs1sub = fs1.makeopendir("from") - self._make_fs(fs1sub) + self._make_fs(fs1sub) utils.movedir((fs1, "from"), (fs2, "copy")) - self.assert_(not fs1.exists("from")) + self.assert_(not fs1.exists("from")) self._check_fs(fs2.opendir("copy")) - + def test_movedir_root(self): - """Test movedir to root dir""" + """Test movedir to root dir""" fs1 = MemoryFS() fs2 = MemoryFS() fs1sub = fs1.makeopendir("from") - self._make_fs(fs1sub) + self._make_fs(fs1sub) utils.movedir((fs1, "from"), fs2) - self.assert_(not fs1.exists("from")) + self.assert_(not fs1.exists("from")) self._check_fs(fs2) fs1 = TempFS() fs2 = TempFS() fs1sub = fs1.makeopendir("from") - self._make_fs(fs1sub) + self._make_fs(fs1sub) utils.movedir((fs1, "from"), fs2) - self.assert_(not fs1.exists("from")) + self.assert_(not fs1.exists("from")) self._check_fs(fs2) - + def test_remove_all(self): """Test remove_all function""" fs = TempFS() @@ -97,9 +97,9 @@ class TestUtils(unittest.TestCase): fs.setcontents("f2", b("file 2")) fs.setcontents("f3", b("file 3")) fs.makedir("foo/bar", recursive=True) - fs.setcontents("foo/bar/fruit", b("apple")) + fs.setcontents("foo/bar/fruit", b("apple")) fs.setcontents("foo/baz", b("baz")) - + utils.remove_all(fs, "foo/bar") self.assert_(not fs.exists("foo/bar/fruit")) self.assert_(fs.exists("foo/bar")) @@ -111,5 +111,5 @@ class TestUtils(unittest.TestCase): self.assert_(not fs.exists("foo")) self.assert_(not fs.exists("f1")) self.assert_(fs.isdirempty('/')) - - + + diff --git a/fs/tests/test_wrapfs.py b/fs/tests/test_wrapfs.py index 153026a..ab674eb 100644 --- a/fs/tests/test_wrapfs.py +++ b/fs/tests/test_wrapfs.py @@ -13,7 +13,7 @@ import shutil import tempfile from fs import osfs -from fs.errors import * +from fs.errors import * from fs.path import * from fs.utils import remove_all from fs import wrapfs @@ -22,9 +22,9 @@ import six from six import PY3, b class TestWrapFS(unittest.TestCase, FSTestCases, ThreadingTestCases): - + #__test__ = False - + def setUp(self): self.temp_dir = tempfile.mkdtemp(u"fstest") self.fs = wrapfs.WrapFS(osfs.OSFS(self.temp_dir)) @@ -39,7 +39,7 @@ class TestWrapFS(unittest.TestCase, FSTestCases, ThreadingTestCases): from fs.wrapfs.lazyfs import LazyFS class TestLazyFS(unittest.TestCase, FSTestCases, ThreadingTestCases): - + def setUp(self): self.temp_dir = tempfile.mkdtemp(u"fstest") self.fs = LazyFS((osfs.OSFS,(self.temp_dir,))) @@ -56,7 +56,7 @@ from fs.wrapfs.limitsizefs import LimitSizeFS class TestLimitSizeFS(TestWrapFS): _dont_retest = TestWrapFS._dont_retest + ("test_big_file",) - + def setUp(self): super(TestLimitSizeFS,self).setUp() self.fs = LimitSizeFS(self.fs,1024*1024*2) # 2MB limit @@ -78,7 +78,7 @@ class TestLimitSizeFS(TestWrapFS): self.assertTrue(total_written < 1024*1024*2 + 1030) break else: - self.assertTrue(False,"StorageSpaceError not raised") + self.assertTrue(False,"StorageSpaceError not raised") from fs.wrapfs.hidedotfilesfs import HideDotFilesFS diff --git a/fs/tests/test_xattr.py b/fs/tests/test_xattr.py index 7be5560..142ddd8 100644 --- a/fs/tests/test_xattr.py +++ b/fs/tests/test_xattr.py @@ -11,7 +11,7 @@ from fs.path import * from fs.errors import * from fs.tests import FSTestCases -from six import b +from six import b class XAttrTestCases: """Testcases for filesystems providing extended attribute support. @@ -108,7 +108,7 @@ class XAttrTestCases: # Check that removing a file without xattrs still works self.fs.createfile("myfile2") self.fs.remove("myfile2") - + def test_remove_dir(self): def listxattrs(path): return list(self.fs.listxattrs(path)) diff --git a/fs/tests/zipfs_binary_test.py b/fs/tests/zipfs_binary_test.py index 2a1e8bc..716cc4d 100644 --- a/fs/tests/zipfs_binary_test.py +++ b/fs/tests/zipfs_binary_test.py @@ -1,45 +1,45 @@ -"""
-Test case for ZipFS binary file reading/writing
-Passes ok on Linux, fails on Windows (tested: Win7, 64-bit):
-
-AssertionError: ' \r\n' != ' \n'
-"""
-
-import unittest
-from fs.zipfs import ZipFS
-import os
-
-from six import b
-
-class ZipFsBinaryWriteRead(unittest.TestCase):
- test_content = b(chr(32) + chr(10))
-
- def setUp(self):
- self.z = ZipFS('test.zip', 'w')
-
- def tearDown(self):
- try:
- os.remove('test.zip')
- except:
- pass
-
- def test_binary_write_read(self):
- # GIVEN zipfs
- z = self.z
-
- # WHEN binary data is written to a test file in zipfs
- f = z.open('test.data', 'wb')
- f.write(self.test_content)
- f.close()
- z.close()
-
- # THEN the same binary data is retrieved when opened again
- z = ZipFS('test.zip', 'r')
- f = z.open('test.data', 'rb')
- content = f.read()
- f.close()
- z.close()
- self.assertEqual(content, self.test_content)
-
-if __name__ == '__main__':
- unittest.main()
+""" +Test case for ZipFS binary file reading/writing +Passes ok on Linux, fails on Windows (tested: Win7, 64-bit): + +AssertionError: ' \r\n' != ' \n' +""" + +import unittest +from fs.zipfs import ZipFS +import os + +from six import b + +class ZipFsBinaryWriteRead(unittest.TestCase): + test_content = b(chr(32) + chr(10)) + + def setUp(self): + self.z = ZipFS('test.zip', 'w') + + def tearDown(self): + try: + os.remove('test.zip') + except: + pass + + def test_binary_write_read(self): + # GIVEN zipfs + z = self.z + + # WHEN binary data is written to a test file in zipfs + f = z.open('test.data', 'wb') + f.write(self.test_content) + f.close() + z.close() + + # THEN the same binary data is retrieved when opened again + z = ZipFS('test.zip', 'r') + f = z.open('test.data', 'rb') + content = f.read() + f.close() + z.close() + self.assertEqual(content, self.test_content) + +if __name__ == '__main__': + unittest.main() diff --git a/fs/wrapfs/debugfs.py b/fs/wrapfs/debugfs.py index 5810097..6e4d7d5 100644 --- a/fs/wrapfs/debugfs.py +++ b/fs/wrapfs/debugfs.py @@ -1,143 +1,143 @@ -'''
- @author: Marek Palatinus <marek@palatinus.cz>
- @license: Public domain
-
- DebugFS is a wrapper around filesystems to help developers
- debug their work. I wrote this class mainly for debugging
- TahoeLAFS and for fine tuning TahoeLAFS over Dokan with higher-level
- aplications like Total Comander, Winamp etc. Did you know
- that Total Commander need to open file before it delete them? :-)
-
- I hope DebugFS can be helpful also for other filesystem developers,
- especially for those who are trying to implement their first one (like me).
-
- DebugFS prints to stdout (by default) all attempts to
- filesystem interface, prints parameters and results.
-
- Basic usage:
- fs = DebugFS(OSFS('~'), identifier='OSFS@home', \
- skip=('_lock', 'listdir', 'listdirinfo'))
- print fs.listdir('.')
- print fs.unsupportedfunction()
-
- Error levels:
- DEBUG: Print everything (asking for methods, calls, response, exception)
- INFO: Print calls, responses, exception
- ERROR: Print only exceptions
- CRITICAL: Print only exceptions not derived from fs.errors.FSError
-
- How to change error level:
- import logging
- logger = logging.getLogger('fs.debugfs')
- logger.setLevel(logging.CRITICAL)
- fs = DebugFS(OSFS('~')
- print fs.listdir('.')
-
-'''
-import logging
-from logging import DEBUG, INFO, ERROR, CRITICAL
-import sys
-
-import fs
-from fs.errors import FSError
-
-logger = fs.getLogger('fs.debugfs')
-logger.setLevel(logging.DEBUG)
-logger.addHandler(logging.StreamHandler())
-
-class DebugFS(object):
- def __init__(self, fs, identifier=None, skip=(), verbose=True):
- '''
- fs - Reference to object to debug
- identifier - Custom string-like object will be added
- to each log line as identifier.
- skip - list of method names which DebugFS should not log
- '''
- self.__wrapped_fs = fs
- self.__identifier = identifier
- self.__skip = skip
- self.__verbose = verbose
- super(DebugFS, self).__init__()
-
- def __log(self, level, message):
- if self.__identifier:
- logger.log(level, '(%s) %s' % (self.__identifier, message))
- else:
- logger.log(level, message)
-
- def __parse_param(self, value):
- if isinstance(value, basestring):
- if len(value) > 60:
- value = "%s ... (length %d)" % (repr(value[:60]), len(value))
- else:
- value = repr(value)
- elif isinstance(value, list):
- value = "%s (%d items)" % (repr(value[:3]), len(value))
- elif isinstance(value, dict):
- items = {}
- for k, v in value.items()[:3]:
- items[k] = v
- value = "%s (%d items)" % (repr(items), len(value))
- else:
- value = repr(value)
- return value
-
- def __parse_args(self, *arguments, **kwargs):
- args = [self.__parse_param(a) for a in arguments]
- for k, v in kwargs.items():
- args.append("%s=%s" % (k, self.__parse_param(v)))
-
- args = ','.join(args)
- if args: args = "(%s)" % args
- return args
-
- def __report(self, msg, key, value, *arguments, **kwargs):
- if key in self.__skip: return
- args = self.__parse_args(*arguments, **kwargs)
- value = self.__parse_param(value)
- self.__log(INFO, "%s %s%s -> %s" % (msg, str(key), args, value))
-
- def __getattr__(self, key):
-
- if key.startswith('__'):
- # Internal calls, nothing interesting
- return object.__getattribute__(self, key)
-
- try:
- attr = getattr(self.__wrapped_fs, key)
- except AttributeError, e:
- self.__log(DEBUG, "Asking for not implemented method %s" % key)
- raise e
- except Exception, e:
- self.__log(CRITICAL, "Exception %s: %s" % \
- (e.__class__.__name__, str(e)))
- raise e
-
- if not callable(attr):
- if key not in self.__skip:
- self.__report("Get attribute", key, attr)
- return attr
-
- def _method(*args, **kwargs):
- try:
- value = attr(*args, **kwargs)
- self.__report("Call method", key, value, *args, **kwargs)
- except FSError, e:
- self.__log(ERROR, "Call method %s%s -> Exception %s: %s" % \
- (key, self.__parse_args(*args, **kwargs), \
- e.__class__.__name__, str(e)))
- (exc_type,exc_inst,tb) = sys.exc_info()
- raise e, None, tb
- except Exception, e:
- self.__log(CRITICAL,
- "Call method %s%s -> Non-FS exception %s: %s" %\
- (key, self.__parse_args(*args, **kwargs), \
- e.__class__.__name__, str(e)))
- (exc_type,exc_inst,tb) = sys.exc_info()
- raise e, None, tb
- return value
-
- if self.__verbose:
- if key not in self.__skip:
- self.__log(DEBUG, "Asking for method %s" % key)
- return _method
+''' + @author: Marek Palatinus <marek@palatinus.cz> + @license: Public domain + + DebugFS is a wrapper around filesystems to help developers + debug their work. I wrote this class mainly for debugging + TahoeLAFS and for fine tuning TahoeLAFS over Dokan with higher-level + aplications like Total Comander, Winamp etc. Did you know + that Total Commander need to open file before it delete them? :-) + + I hope DebugFS can be helpful also for other filesystem developers, + especially for those who are trying to implement their first one (like me). + + DebugFS prints to stdout (by default) all attempts to + filesystem interface, prints parameters and results. + + Basic usage: + fs = DebugFS(OSFS('~'), identifier='OSFS@home', \ + skip=('_lock', 'listdir', 'listdirinfo')) + print fs.listdir('.') + print fs.unsupportedfunction() + + Error levels: + DEBUG: Print everything (asking for methods, calls, response, exception) + INFO: Print calls, responses, exception + ERROR: Print only exceptions + CRITICAL: Print only exceptions not derived from fs.errors.FSError + + How to change error level: + import logging + logger = logging.getLogger('fs.debugfs') + logger.setLevel(logging.CRITICAL) + fs = DebugFS(OSFS('~') + print fs.listdir('.') + +''' +import logging +from logging import DEBUG, INFO, ERROR, CRITICAL +import sys + +import fs +from fs.errors import FSError + +logger = fs.getLogger('fs.debugfs') +logger.setLevel(logging.DEBUG) +logger.addHandler(logging.StreamHandler()) + +class DebugFS(object): + def __init__(self, fs, identifier=None, skip=(), verbose=True): + ''' + fs - Reference to object to debug + identifier - Custom string-like object will be added + to each log line as identifier. + skip - list of method names which DebugFS should not log + ''' + self.__wrapped_fs = fs + self.__identifier = identifier + self.__skip = skip + self.__verbose = verbose + super(DebugFS, self).__init__() + + def __log(self, level, message): + if self.__identifier: + logger.log(level, '(%s) %s' % (self.__identifier, message)) + else: + logger.log(level, message) + + def __parse_param(self, value): + if isinstance(value, basestring): + if len(value) > 60: + value = "%s ... (length %d)" % (repr(value[:60]), len(value)) + else: + value = repr(value) + elif isinstance(value, list): + value = "%s (%d items)" % (repr(value[:3]), len(value)) + elif isinstance(value, dict): + items = {} + for k, v in value.items()[:3]: + items[k] = v + value = "%s (%d items)" % (repr(items), len(value)) + else: + value = repr(value) + return value + + def __parse_args(self, *arguments, **kwargs): + args = [self.__parse_param(a) for a in arguments] + for k, v in kwargs.items(): + args.append("%s=%s" % (k, self.__parse_param(v))) + + args = ','.join(args) + if args: args = "(%s)" % args + return args + + def __report(self, msg, key, value, *arguments, **kwargs): + if key in self.__skip: return + args = self.__parse_args(*arguments, **kwargs) + value = self.__parse_param(value) + self.__log(INFO, "%s %s%s -> %s" % (msg, str(key), args, value)) + + def __getattr__(self, key): + + if key.startswith('__'): + # Internal calls, nothing interesting + return object.__getattribute__(self, key) + + try: + attr = getattr(self.__wrapped_fs, key) + except AttributeError, e: + self.__log(DEBUG, "Asking for not implemented method %s" % key) + raise e + except Exception, e: + self.__log(CRITICAL, "Exception %s: %s" % \ + (e.__class__.__name__, str(e))) + raise e + + if not callable(attr): + if key not in self.__skip: + self.__report("Get attribute", key, attr) + return attr + + def _method(*args, **kwargs): + try: + value = attr(*args, **kwargs) + self.__report("Call method", key, value, *args, **kwargs) + except FSError, e: + self.__log(ERROR, "Call method %s%s -> Exception %s: %s" % \ + (key, self.__parse_args(*args, **kwargs), \ + e.__class__.__name__, str(e))) + (exc_type,exc_inst,tb) = sys.exc_info() + raise e, None, tb + except Exception, e: + self.__log(CRITICAL, + "Call method %s%s -> Non-FS exception %s: %s" %\ + (key, self.__parse_args(*args, **kwargs), \ + e.__class__.__name__, str(e))) + (exc_type,exc_inst,tb) = sys.exc_info() + raise e, None, tb + return value + + if self.__verbose: + if key not in self.__skip: + self.__log(DEBUG, "Asking for method %s" % key) + return _method diff --git a/fs/xattrs.py b/fs/xattrs.py index 1089455..8250ded 100644 --- a/fs/xattrs.py +++ b/fs/xattrs.py @@ -39,7 +39,7 @@ def ensure_xattrs(fs): Given an FS object, this function returns an equivalent FS that has support for extended attributes. This may be the original object if they are supported natively, or a wrapper class is they must be simulated. - + :param fs: An FS object that must have xattrs """ try: @@ -59,7 +59,7 @@ class SimulateXAttr(WrapFS): * setxattr: set an xattr of a path by name * delxattr: delete an xattr of a path by name - For each file in the underlying FS, this class maintains a corresponding + For each file in the underlying FS, this class maintains a corresponding '.xattrs.FILENAME' file containing its extended attributes. Extended attributes of a directory are stored in the file '.xattrs' within the directory itself. @@ -202,4 +202,4 @@ class SimulateXAttr(WrapFS): except ResourceNotFoundError: pass - + diff --git a/requirements.txt b/requirements.txt index 61b599b..23e779a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,3 @@ paramiko six django dexml -wx
\ No newline at end of file @@ -33,7 +33,7 @@ classifiers = [ 'Topic :: System :: Filesystems', ] -with open('README.txt', 'r') as f: +with open('README.md', 'r') as f: long_desc = f.read() @@ -7,7 +7,6 @@ deps = six dexml nose - py31,py32,py33,py34: winpdb py26,py27: paramiko py26,py27: boto py26,py27: mako |