From 221c3725f109ed5a6126c1f406cda68f55d95894 Mon Sep 17 00:00:00 2001 From: "willmcgugan@gmail.com" Date: Thu, 13 Mar 2014 18:47:17 +0000 Subject: Test fixes and preparations for 0.5.0 release git-svn-id: http://pyfilesystem.googlecode.com/svn/trunk@887 67cdc799-7952-0410-af00-57a81ceafa0f --- CHANGES.txt | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++ ChangeLog | 90 --------------------------------------------------- LICENSE.txt | 2 +- MANIFEST.in | 3 ++ README.txt | 67 ++++++++++++++++++++++++++++++++++++++ fs/appdirfs.py | 1 + fs/commands/runner.py | 16 +++++---- fs/expose/xmlrpc.py | 12 +++++++ fs/iotools.py | 2 -- fs/memoryfs.py | 29 ++++++++--------- fs/rpcfs.py | 3 +- fs/tests/__init__.py | 4 +-- setup.py | 14 ++++---- 13 files changed, 209 insertions(+), 124 deletions(-) create mode 100644 CHANGES.txt delete mode 100644 ChangeLog create mode 100644 README.txt diff --git a/CHANGES.txt b/CHANGES.txt new file mode 100644 index 0000000..d6b28bd --- /dev/null +++ b/CHANGES.txt @@ -0,0 +1,90 @@ + +0.3: + + * New FS implementations: + * FTPFS: access a plain old FTP server + * S3FS: access remote files stored in Amazon S3 + * RPCFS: access remote files using a simple XML-RPC protocol + * SFTPFS: access remote files on a SFTP server + * WrapFS: filesystem that wraps an FS object and transparently + modifies its contents (think encryption, compression, ...) + * LazyFS: lazily instantiate an FS object the first time it is used + * ReadOnlyFS: a WrapFS that makes an fs read-only + * Ability to expose FS objects to the outside world: + * expose.fuse: expose an FS object using FUSE + * expose.xmlrpc: expose an FS object a simple XML-RPC protocol + * expose.sftp: expose an FS object SFTP + * expose.django_storage: convert FS object to Django Storage object + * Extended attribute support (getxattr/setxattr/delxattr/listxattrs) + * Change watching support (add_watcher/del_watcher) + * Insist on unicode paths throughout: + * output paths are always unicode + * bytestring input paths are decoded as early as possible + * Renamed "fs.helpers" to "fs.path", and renamed the contained functions + to match those offered by os.path + * fs.remote: utilities for implementing FS classes that interface + with a remote filesystem + * fs.errors: updated exception hierarchy, with support for converting + to/from standard OSError instances + * Added cache_hint method to base.py + * Added settimes method to base implementation + * New implementation of print_fs, accessible through tree method on base class + + +0.4: + + * New FS implementations (under fs.contrib): + * BigFS: read contents of a BIG file (C&C game file format) + * DAVFS: access remote files stored on a WebDAV server + * TahoeLAFS: access files stored in a Tahoe-LAFS grid + * New fs.expose implementations: + * dokan: mount an FS object as a drive using Dokan (win32-only) + * importhook: import modules from files in an FS object + * Modified listdir and walk methods to accept callables as well as strings + for wildcards. + * Added listdirinfo method, which yields both the entry names and the + corresponding info dicts in a single operation. + * Made SubFS a subclass of WrapFS, and moved it into its own module at + fs.wrapfs.subfs. + * Path-handling fixes for OSFS on win32: + * Work properly when pointing to the root of a drive. + * Better handling of remote UNC paths. + * Add ability to switch off use of long UNC paths. + * OSFSWatchMixin improvements: + * watch_inotify: allow more than one watcher on a single path. + * watch_win32: don't create immortal reference cycles. + * watch_win32: report errors if the filesystem does't support + ReadDirectoryChangesW. + * MountFS: added support for mounting at the root directory, and for + mounting over an existing mount. + * Added 'getpathurl' and 'haspathurl' methods. + * Added utils.isdir(fs,path,info) and utils.isfile(fs,path,info); these + can often determine whether a path is a file or directory by inspecting + the info dict and avoid an additional query to the filesystem. + * Added utility module 'fs.filelike' with some helpers for building and + manipulating file-like objects. + * Added getmeta and hasmeta methods + * Separated behaviour of setcontents and createfile + * Added a getmmap to base + * Added command line scripts fsls, fstree, fscat, fscp, fsmv + * Added command line scripts fsmkdir, fsmount + * Made SFTP automatically pick up keys if no other authentication + is available + * Optimized listdir and listdirinfo in SFTPFS + * Made memoryfs work with threads + * Added copyfile_non_atomic and movefile_non_atomic for improved performance of multi-threaded copies + * Added a concept of a writeable FS to MultiFS + * Added ilistdir() and ilistdirinfo() methods, which are generator-based + variants of listdir() and listdirinfo(). + * Removed obsolete module fs.objectree; use fs.path.PathMap instead. + * Added setcontents_async method to base + * Added `appdirfs` module to abstract per-user application directories + +0.5: + + * Ported to Python 3.X + * Added a DeleteRootError to exceptions thrown when trying to delete '/' + * Added a remove_all function to utils + * Added sqlitefs to fs.contrib, contributed by Nitin Bhide + * Added archivefs to fs.contrib, contributed by btimby + * Added some polish to fstree command and unicode box lines rather than ascii art diff --git a/ChangeLog b/ChangeLog deleted file mode 100644 index d6b28bd..0000000 --- a/ChangeLog +++ /dev/null @@ -1,90 +0,0 @@ - -0.3: - - * New FS implementations: - * FTPFS: access a plain old FTP server - * S3FS: access remote files stored in Amazon S3 - * RPCFS: access remote files using a simple XML-RPC protocol - * SFTPFS: access remote files on a SFTP server - * WrapFS: filesystem that wraps an FS object and transparently - modifies its contents (think encryption, compression, ...) - * LazyFS: lazily instantiate an FS object the first time it is used - * ReadOnlyFS: a WrapFS that makes an fs read-only - * Ability to expose FS objects to the outside world: - * expose.fuse: expose an FS object using FUSE - * expose.xmlrpc: expose an FS object a simple XML-RPC protocol - * expose.sftp: expose an FS object SFTP - * expose.django_storage: convert FS object to Django Storage object - * Extended attribute support (getxattr/setxattr/delxattr/listxattrs) - * Change watching support (add_watcher/del_watcher) - * Insist on unicode paths throughout: - * output paths are always unicode - * bytestring input paths are decoded as early as possible - * Renamed "fs.helpers" to "fs.path", and renamed the contained functions - to match those offered by os.path - * fs.remote: utilities for implementing FS classes that interface - with a remote filesystem - * fs.errors: updated exception hierarchy, with support for converting - to/from standard OSError instances - * Added cache_hint method to base.py - * Added settimes method to base implementation - * New implementation of print_fs, accessible through tree method on base class - - -0.4: - - * New FS implementations (under fs.contrib): - * BigFS: read contents of a BIG file (C&C game file format) - * DAVFS: access remote files stored on a WebDAV server - * TahoeLAFS: access files stored in a Tahoe-LAFS grid - * New fs.expose implementations: - * dokan: mount an FS object as a drive using Dokan (win32-only) - * importhook: import modules from files in an FS object - * Modified listdir and walk methods to accept callables as well as strings - for wildcards. - * Added listdirinfo method, which yields both the entry names and the - corresponding info dicts in a single operation. - * Made SubFS a subclass of WrapFS, and moved it into its own module at - fs.wrapfs.subfs. - * Path-handling fixes for OSFS on win32: - * Work properly when pointing to the root of a drive. - * Better handling of remote UNC paths. - * Add ability to switch off use of long UNC paths. - * OSFSWatchMixin improvements: - * watch_inotify: allow more than one watcher on a single path. - * watch_win32: don't create immortal reference cycles. - * watch_win32: report errors if the filesystem does't support - ReadDirectoryChangesW. - * MountFS: added support for mounting at the root directory, and for - mounting over an existing mount. - * Added 'getpathurl' and 'haspathurl' methods. - * Added utils.isdir(fs,path,info) and utils.isfile(fs,path,info); these - can often determine whether a path is a file or directory by inspecting - the info dict and avoid an additional query to the filesystem. - * Added utility module 'fs.filelike' with some helpers for building and - manipulating file-like objects. - * Added getmeta and hasmeta methods - * Separated behaviour of setcontents and createfile - * Added a getmmap to base - * Added command line scripts fsls, fstree, fscat, fscp, fsmv - * Added command line scripts fsmkdir, fsmount - * Made SFTP automatically pick up keys if no other authentication - is available - * Optimized listdir and listdirinfo in SFTPFS - * Made memoryfs work with threads - * Added copyfile_non_atomic and movefile_non_atomic for improved performance of multi-threaded copies - * Added a concept of a writeable FS to MultiFS - * Added ilistdir() and ilistdirinfo() methods, which are generator-based - variants of listdir() and listdirinfo(). - * Removed obsolete module fs.objectree; use fs.path.PathMap instead. - * Added setcontents_async method to base - * Added `appdirfs` module to abstract per-user application directories - -0.5: - - * Ported to Python 3.X - * Added a DeleteRootError to exceptions thrown when trying to delete '/' - * Added a remove_all function to utils - * Added sqlitefs to fs.contrib, contributed by Nitin Bhide - * Added archivefs to fs.contrib, contributed by btimby - * Added some polish to fstree command and unicode box lines rather than ascii art diff --git a/LICENSE.txt b/LICENSE.txt index 18c9da6..fc63077 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2009-2014, Will McGugan and contributors. +Copyright (c) 2009-2015, Will McGugan and contributors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, diff --git a/MANIFEST.in b/MANIFEST.in index 1701bbd..2720cd5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,5 @@ include AUTHORS +include README.txt +include LICENSE.txt +include CHANGES.txt \ No newline at end of file diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..a68b649 --- /dev/null +++ b/README.txt @@ -0,0 +1,67 @@ +PyFilesystem +============ + +PyFilesystem is an abstraction layer for *filesystems*. In the same way that Python's file-like objects provide a common way of accessing files, PyFilesystem provides a common way of accessing entire filesystems. You can write platform-independent code to work with local files, that also works with any of the supported filesystems (zip, ftp, S3 etc.). + +Pyfilesystem works with Linux, Windows and Mac. + +Suported Filesystems +--------------------- + +Here are a few of the filesystems that can be accessed with Pyfilesystem: + +* **DavFS** access files & directories on a WebDAV server +* **FTPFS** access files & directories on an FTP server +* **MemoryFS** access files & directories stored in memory (non-permanent but very fast) +* **MountFS** creates a virtual directory structure built from other filesystems +* **MultiFS** a virtual filesystem that combines a list of filesystems in to one, and checks them in order when opening files +* **OSFS** the native filesystem +* **SFTPFS** access files & directores stored on a Secure FTP server +* **S3FS** access files & directories stored on Amazon S3 storage +* **TahoeLAFS** access files & directories stored on a Tahoe distributed filesystem +* **ZipFS** access files and directories contained in a zip file + +Example +------- + +The following snippet prints the total number of bytes contained in all your Python files in `C:/projects` (including sub-directories):: + + from fs.osfs import OSFS + projects_fs = OSFS('C:/projects') + print sum(projects_fs.getsize(path) + for path in projects_fs.walkfiles(wildcard="*.py")) + +That is, assuming you are on Windows and have a directory called 'projects' in your C drive. If you are on Linux / Mac, you might replace the second line with something like:: + + projects_fs = OSFS('~/projects') + +If you later want to display the total size of Python files stored in a zip file, you could make the following change to the first two lines:: + + from fs.zipfs import ZipFS + projects_fs = ZipFS('source.zip') + +In fact, you could use any of the supported filesystems above, and the code would continue to work as before. + +An alternative to explicity importing the filesystem class you want, is to use an FS opener which opens a filesystem from a URL-like syntax:: + + from fs.opener import fsopendir + projects_fs = fsopendir('C:/projects') + +You could change ``C:/projects`` to ``zip://source.zip`` to open the zip file, or even ``ftp://ftp.example.org/code/projects/`` to sum up the bytes of Python stored on an ftp server. + +Screencast +---------- + +This is from an early version of PyFilesystem, but still relevant + +http://vimeo.com/12680842 + +Discussion Group +---------------- + +http://groups.google.com/group/pyfilesystem-discussion + +Further Information +------------------- + +http://www.willmcgugan.com/tag/fs/ \ No newline at end of file diff --git a/fs/appdirfs.py b/fs/appdirfs.py index ff7dd69..86d8082 100644 --- a/fs/appdirfs.py +++ b/fs/appdirfs.py @@ -21,6 +21,7 @@ __all__ = ['UserDataFS', 'UserCacheFS', 'UserLogFS'] + class UserDataFS(OSFS): """A filesystem for per-user application data.""" def __init__(self, appname, appauthor=None, version=None, roaming=False, create=True): diff --git a/fs/commands/runner.py b/fs/commands/runner.py index c5c7e6f..57d6e99 100644 --- a/fs/commands/runner.py +++ b/fs/commands/runner.py @@ -1,17 +1,19 @@ import warnings warnings.filterwarnings("ignore") -import sys -from optparse import OptionParser from fs.opener import opener, OpenerError, Opener from fs.errors import FSError from fs.path import splitext, pathsplit, isdotfile, iswildcard + +import re +import sys import platform -from collections import defaultdict import six +from optparse import OptionParser +from collections import defaultdict -if platform.system() == 'Windows': +if platform.system() == 'Windows': def getTerminalSize(): try: ## {{{ http://code.activestate.com/recipes/440694/ (r3) @@ -32,13 +34,12 @@ if platform.system() == 'Windows': sizex = right - left + 1 sizey = bottom - top + 1 else: - sizex, sizey = 80, 25 # can't determine actual size - return default values + sizex, sizey = 80, 25 # can't determine actual size - return default values return sizex, sizey except: return 80, 25 else: - def getTerminalSize(): def ioctl_GWINSZ(fd): try: @@ -65,11 +66,13 @@ else: pass return 80, 25 + def _unicode(text): if not isinstance(text, unicode): return text.decode('ascii', 'replace') return text + class Command(object): usage = '' @@ -146,6 +149,7 @@ class Command(object): if not self.terminal_colors: return text re_fs = r'(\S*?://\S*)' + def repl(matchobj): fs_url = matchobj.group(0) return self.wrap_link(fs_url) diff --git a/fs/expose/xmlrpc.py b/fs/expose/xmlrpc.py index 4a29564..fd78a32 100644 --- a/fs/expose/xmlrpc.py +++ b/fs/expose/xmlrpc.py @@ -31,6 +31,16 @@ class RPCFSInterface(object): the contents of files. """ + # info keys are restricted to a subset known to work over xmlrpc + # This fixes an issue with transporting Longs on Py3 + _allowed_info = ["size", + "created_time", + "modified_time", + "accessed_time", + "st_size", + "st_mode", + "type"] + def __init__(self, fs): super(RPCFSInterface, self).__init__() self.fs = fs @@ -118,6 +128,8 @@ class RPCFSInterface(object): def getinfo(self, path): path = self.decode_path(path) info = self.fs.getinfo(path) + info = dict((k, v) for k, v in info.iteritems() + if k in self._allowed_info) return info def desc(self, path): diff --git a/fs/iotools.py b/fs/iotools.py index fcf4818..f2e0f6e 100644 --- a/fs/iotools.py +++ b/fs/iotools.py @@ -188,8 +188,6 @@ def make_bytes_io(data, encoding=None, errors=None): return io.BytesIO(data) - - def copy_file_to_fs(f, fs, path, encoding=None, errors=None, progress_callback=None, chunk_size=64 * 1024): """Copy an open file to a path on an FS""" if progress_callback is None: diff --git a/fs/memoryfs.py b/fs/memoryfs.py index d48cf9f..9a87db3 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -31,6 +31,7 @@ def _check_mode(mode, mode_chars): return False return True + class MemoryFile(object): def seek_and_lock(f): @@ -71,7 +72,6 @@ class MemoryFile(object): finally: lock.release() - assert self.mem_file is not None, "self.mem_file should have a value" def __str__(self): @@ -163,7 +163,7 @@ class MemoryFile(object): def __enter__(self): return self - def __exit__(self,exc_type,exc_value,traceback): + def __exit__(self, exc_type, exc_value, traceback): self.close() return False @@ -218,7 +218,7 @@ class DirEntry(object): if self.isfile(): return "" % self.name elif self.isdir(): - return "" % "".join( "%s: %s" % (k, v.desc_contents()) for k, v in self.contents.iteritems()) + return "" % "".join("%s: %s" % (k, v.desc_contents()) for k, v in self.contents.iteritems()) def isdir(self): return self.type == "dir" @@ -248,24 +248,23 @@ class DirEntry(object): self.mem_file = StringIO() self.mem_file.write(data) -class MemoryFS(FS): +class MemoryFS(FS): """An in-memory filesystem. """ - _meta = {'thread_safe' : True, - 'network' : False, + _meta = {'thread_safe': True, + 'network': False, 'virtual': False, - 'read_only' : False, - 'unicode_paths' : True, - 'case_insensitive_paths' : False, - 'atomic.move' : False, - 'atomic.copy' : False, - 'atomic.makedir' : True, - 'atomic.rename' : True, - 'atomic.setcontents' : False, - } + 'read_only': False, + 'unicode_paths': True, + 'case_insensitive_paths': False, + 'atomic.move': False, + 'atomic.copy': False, + 'atomic.makedir': True, + 'atomic.rename': True, + 'atomic.setcontents': False} def _make_dir_entry(self, *args, **kwargs): return self.dir_entry_factory(*args, **kwargs) diff --git a/fs/rpcfs.py b/fs/rpcfs.py index eecdbf0..00ba86a 100644 --- a/fs/rpcfs.py +++ b/fs/rpcfs.py @@ -305,7 +305,8 @@ class RPCFS(FS): @synchronize def getinfo(self, path): path = self.encode_path(path) - return self.proxy.getinfo(path) + info = self.proxy.getinfo(path) + return info @synchronize def desc(self, path): diff --git a/fs/tests/__init__.py b/fs/tests/__init__.py index d99c1b1..3aa8ec8 100644 --- a/fs/tests/__init__.py +++ b/fs/tests/__init__.py @@ -163,8 +163,8 @@ class FSTestCases(object): b("to you, good sir!")), chunk_size=2) self.assertEquals(self.fs.getcontents( "hello", "rb"), b("to you, good sir!")) - self.fs.setcontents("hello", b"") - self.assertEquals(self.fs.getcontents("hello", "rb"), "") + self.fs.setcontents("hello", b("")) + self.assertEquals(self.fs.getcontents("hello", "rb"), b("")) def test_setcontents_async(self): # setcontents() should accept both a string... diff --git a/setup.py b/setup.py index e69a924..2778adc 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ from setuptools import setup import sys PY3 = sys.version_info >= (3,) -VERSION = "0.4.1" +VERSION = "0.5.0" COMMANDS = ['fscat', 'fscp', @@ -34,10 +34,9 @@ classifiers = [ 'Topic :: System :: Filesystems', ] -long_desc = """Pyfilesystem is a module that provides a simplified common interface to many types of filesystem. Filesystems exposed via Pyfilesystem can also be served over the network, or 'mounted' on the native filesystem. +with open('README.txt', 'r') as f: + long_desc = f.read() -Even if you only need to work with file and directories on the local hard-drive, Pyfilesystem can simplify your code and make it more robust -- with the added advantage that you can change where the files are located by changing a single line of code. -""" extra = {} if PY3: @@ -46,13 +45,14 @@ if PY3: setup(install_requires=['distribute', 'six'], name='fs', version=VERSION, - description="Filesystem abstraction", + description="Filesystem abstraction layer", long_description=long_desc, license="BSD", author="Will McGugan", author_email="will@willmcgugan.com", - url="http://code.google.com/p/pyfilesystem/", - download_url="http://code.google.com/p/pyfilesystem/downloads/list", + #url="http://code.google.com/p/pyfilesystem/", + #download_url="http://code.google.com/p/pyfilesystem/downloads/list", + url="http://pypi.python.org/pypi/fs/" platforms=['any'], packages=['fs', 'fs.expose', -- cgit v1.2.1