diff options
author | willmcgugan <willmcgugan@67cdc799-7952-0410-af00-57a81ceafa0f> | 2010-12-09 22:09:48 +0000 |
---|---|---|
committer | willmcgugan <willmcgugan@67cdc799-7952-0410-af00-57a81ceafa0f> | 2010-12-09 22:09:48 +0000 |
commit | 4dff33202b5957601286af8951b2e7e637dd6772 (patch) | |
tree | 33ed76811fffca8e0014842c5c4b853efc48c0ba /fs/opener.py | |
parent | f1f224c135ea4356d7805cb94a88f82d16608ca6 (diff) | |
download | pyfilesystem-git-4dff33202b5957601286af8951b2e7e637dd6772.tar.gz |
Changed syntax for commands to be more url like, optimized sftps to use fewer queries for listdir
Diffstat (limited to 'fs/opener.py')
-rw-r--r-- | fs/opener.py | 214 |
1 files changed, 141 insertions, 73 deletions
diff --git a/fs/opener.py b/fs/opener.py index 4a8ff59..a6f7b66 100644 --- a/fs/opener.py +++ b/fs/opener.py @@ -1,8 +1,10 @@ import sys from fs.osfs import OSFS -from fs.path import pathsplit +from fs.path import pathsplit, basename, join, iswildcard +import os import os.path import re +from urlparse import urlparse class OpenerError(Exception): pass @@ -31,11 +33,15 @@ def _expand_syspath(path): return path + + class OpenerRegistry(object): + re_fs_url = re.compile(r''' ^ -(?:\[(.*?)\])* +(.*?) +:\/\/ (?: \((.*?)\) @@ -46,6 +52,8 @@ class OpenerRegistry(object): \+(.*?)$ )*$ ''', re.VERBOSE) + + def __init__(self, openers=[]): self.registry = {} @@ -56,13 +64,12 @@ class OpenerRegistry(object): @classmethod def split_segments(self, fs_url): - match = self.re_fs_url.match(fs_url) - assert match is not None, "broken re?" - return match.groups() + match = self.re_fs_url.match(fs_url) + return match def get_opener(self, name): if name not in self.registry: - raise NoOpenerError("No opener for [%s]" % name) + raise NoOpenerError("No opener for %s" % name) index = self.registry[name] return self.openers[index] @@ -73,34 +80,56 @@ class OpenerRegistry(object): self.registry[name] = index def parse(self, fs_url, default_fs_name=None, writeable=False, create=False): - - fs_name, paren_url, fs_url, path = self.split_segments(fs_url) + + orig_url = fs_url + match = self.split_segments(fs_url) - fs_url = fs_url or paren_url - if fs_name is None and path is None: - fs_url = os.path.expanduser(os.path.expandvars(fs_url)) - fs_url = os.path.normpath(os.path.abspath(fs_url)) - fs_url, path = pathsplit(fs_url) - if not fs_url: - fs_url = '/' + if match: + fs_name, paren_url, fs_url, path = match.groups() + fs_url = fs_url or paren_url or '' + if ':' in fs_name: + fs_name, sub_protocol = fs_name.split(':', 1) + fs_url = '%s://%s' % (sub_protocol, fs_url) + + fs_name = fs_name or self.default_opener + + else: + fs_name = default_fs_name or self.default_opener + fs_url = _expand_syspath(fs_url) + path = '' + + + fs_name, fs_name_params = self.parse_name(fs_name) + opener = self.get_opener(fs_name) - fs_name = fs_name or self.default_opener + if fs_url is None: + raise OpenerError("Unable to parse '%s'" % orig_url) - if fs_name is None: - fs_name = fs_default_name + wildcard = None + if iswildcard(fs_url): + fs_url, wildcard = pathsplit(fs_url) - fs_name, fs_name_params = self.parse_name(fs_name) - opener = self.get_opener(fs_name) - - fs = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create) + fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create) + + if wildcard: + fs_path = join(fs_path or '', wildcard) + else: + path = join(fs_path or '', path) if path: pathname, resourcename = pathsplit(path) if pathname: fs = fs.opendir(pathname) path = resourcename + if not iswildcard(path): + if fs.isdir(path): + fs = fs.opendir(path) + fs_path = '' + else: + fs_path = path + - return fs, path + return fs, fs_path def parse_credentials(self, url): @@ -145,11 +174,21 @@ class OSFSOpener(Opener): names = ['osfs', 'file'] @classmethod - def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create): - username, password, fs_path = registry.parse_credentials(fs_path) - from fs.osfs import OSFS - osfs = OSFS(fs_path, create=create) - return osfs + def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create): + from fs.osfs import OSFS + username, password, fs_path = registry.parse_credentials(fs_path) + + + path = _expand_syspath(fs_path) + if create: + sys.makedirs(fs_path) + if os.path.isdir(path): + osfs = OSFS(path) + filepath = None + else: + path, filepath = pathsplit(path) + osfs = OSFS(path, create=create) + return osfs, filepath class ZipOpener(Opener): @@ -157,22 +196,21 @@ class ZipOpener(Opener): @classmethod def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create): - - create_zip = fs_name_params == 'new' - append_zip = fs_name_params == 'append' - - zip_file = None - if fs_path.startswith('['): - container_fs, container_path = registry.parse(fs_path) - if not container_path: - raise OpenerError("Not a file") - container_mode = 'r+b' - if create_zip: - container_mode = 'w+b' - elif writeable: - container_mode = 'w+b' + + append_zip = fs_name_params == 'add' - zip_file = container_fs.open(container_path, mode=container_mode) + zip_fs, zip_path = registry.parse(fs_path) + if zip_path is None: + raise OpenerError('File required for zip opener') + if create: + open_mode = 'wb' + if append_zip: + open_mode = 'r+b' + else: + open_mode = 'rb' + + zip_file = zip_fs.open(zip_path, mode=open_mode) + username, password, fs_path = registry.parse_credentials(fs_path) @@ -182,21 +220,18 @@ class ZipOpener(Opener): if append_zip: mode = 'a' - elif create_zip or create: + elif create: mode = 'w' else: if writeable: mode = 'w' else: mode = 'a' - - if fs_name == 'zip64': - allow_zip_64 = True - else: - allow_zip_64 = False + + allow_zip_64 = fs_name == 'zip64' zipfs = ZipFS(zip_file, mode=mode, allow_zip_64=allow_zip_64) - return zipfs + return zipfs, None class RPCOpener(Opener): names = ['rpc'] @@ -206,9 +241,16 @@ class RPCOpener(Opener): from fs.rpcfs import RPCFS username, password, fs_path = registry.parse_credentials(fs_path) if not fs_path.startswith('http://'): - fs_path = 'http://' + fs_path - rpcfs = RPCFS(fs_path) - return rpcfs + fs_path = 'http://' + fs_path + + scheme, netloc, path, params, query, fragment = urlparse(fs_path) + + rpcfs = RPCFS('%s://%s' % (scheme, netloc)) + + if create and path: + rpcfs.makedir(path, recursive=True, allow_recreate=True) + + return rpcfs, path or None class FTPOpener(Opener): names = ['ftp'] @@ -217,22 +259,28 @@ class FTPOpener(Opener): def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create): from fs.ftpfs import FTPFS username, password, fs_path = registry.parse_credentials(fs_path) - - if '/' in fs_path: - url, root_path = fs_path.split('/', 1) - else: - url = fs_path - root_path = '' - + + scheme, netloc, path, params, query, fragment = urlparse(fs_path) + if not scheme: + fs_path = 'ftp://' + fs_path + scheme, netloc, path, params, query, fragment = urlparse(fs_path) + + dirpath, resourcepath = pathsplit(path) + url = netloc + ftpfs = FTPFS(url, user=username or '', passwd=password or '') ftpfs.cache_hint(True) - if root_path not in ('', '/'): - if not ftpfs.isdir(root_path): - raise OpenerError("'%s' is not a directory on the server" % root_path) - return ftpfs.opendir(root_path) + if create and path: + ftpfs.makedir(path, recursive=True, allow_recreate=True) - return ftpfs + if dirpath: + ftpfs = ftpfs.opendir(dirpath) + + if not resourcepath: + return ftpfs, None + else: + return ftpfs, resourcepath class SFTPOpener(Opener): @@ -256,6 +304,8 @@ class SFTPOpener(Opener): addr = fs_path fs_path = '/' + fs_path, resourcename = pathsplit(fs_path) + host = addr port = None if ':' in host: @@ -265,10 +315,25 @@ class SFTPOpener(Opener): except ValueError: pass else: - host = (addr, port) + host = (addr, port) + + #if not username or not password: + # raise OpenerError('SFTP requires authentication') + + if create: + sftpfs = SFTPFS(host, root_path='/', **credentials) + if not sftpfs._transport.is_authenticated(): + sftpfs.close() + raise OpenerError('SFTP requires authentication') + sftpfs = sfspfs.makeopendir(fs_path) + return sftpfs, None sftpfs = SFTPFS(host, root_path=fs_path, **credentials) - return sftpfs + if not sftpfs._transport.is_authenticated(): + sftpfs.close() + raise OpenerError('SFTP requires authentication') + + return sftpfs, resourcename class MemOpener(Opener): @@ -277,7 +342,10 @@ class MemOpener(Opener): @classmethod def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create): from fs.memoryfs import MemoryFS - return MemoryFS() + memfs = MemoryFS() + if create: + memfs = memfs.makeopendir(fs_path) + return memfs, None class DebugOpener(Opener): names = ['debug'] @@ -287,13 +355,13 @@ class DebugOpener(Opener): from fs.wrapfs.debugfs import DebugFS if fs_path: fs, path = registry.parse(fs_path, writeable=writeable, create=create) - return DebugFS(fs, verbose=False) + return DebugFS(fs, verbose=False), None if fs_name_params == 'ram': from fs.memoryfs import MemoryFS - return DebugFS(MemoryFS(), identifier=fs_name_params, verbose=False) + return DebugFS(MemoryFS(), identifier=fs_name_params, verbose=False), None else: from fs.tempfs import TempFS - return DebugFS(TempFS(), identifier=fs_name_params, verbose=False) + return DebugFS(TempFS(), identifier=fs_name_params, verbose=False), None class TempOpener(Opener): names = ['temp'] @@ -301,7 +369,7 @@ class TempOpener(Opener): @classmethod def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create): from fs.tempfs import TempFS - return TempFS(identifier=fs_name_params, temp_dir=fs_path) + return TempFS(identifier=fs_name_params, temp_dir=fs_path), None opener = OpenerRegistry([OSFSOpener, @@ -317,7 +385,7 @@ opener = OpenerRegistry([OSFSOpener, def main(): - fs, path = opener.parse('galleries.zip') + fs, path = opener.parse('sftp://willmcgugan.com') print fs, path if __name__ == "__main__": |