Changed syntax for commands to be more url like, optimized sftps to use fewer queries for listdir

git-svn-id: http://pyfilesystem.googlecode.com/svn/trunk@543 67cdc799-7952-0410-af00-57a81ceafa0f

1 files changed, 141 insertions, 73 deletions

diff --git a/fs/opener.py b/fs/opener.py
index 4a8ff59..a6f7b66 100644
--- a/fs/opener.py
+++ b/fs/opener.py
@@ -1,8 +1,10 @@
 import sys
 from fs.osfs import OSFS
-from fs.path import pathsplit
+from fs.path import pathsplit, basename, join, iswildcard
+import os
 import os.path
 import re
+from urlparse import urlparse
 
 class OpenerError(Exception):
     pass
@@ -31,11 +33,15 @@ def _expand_syspath(path):
     return path
 
 
+
+
 class OpenerRegistry(object):
      
+
     re_fs_url = re.compile(r'''
 ^
-(?:\[(.*?)\])*
+(.*?)
+:\/\/
 
 (?:
 \((.*?)\)
@@ -46,6 +52,8 @@ class OpenerRegistry(object):
 \+(.*?)$
 )*$
 ''', re.VERBOSE)
+        
+    
      
     def __init__(self, openers=[]):
         self.registry = {}
@@ -56,13 +64,12 @@ class OpenerRegistry(object):
     
     @classmethod
     def split_segments(self, fs_url):        
-        match = self.re_fs_url.match(fs_url)
-        assert match is not None, "broken re?"
-        return match.groups()        
+        match = self.re_fs_url.match(fs_url)        
+        return match        
     
     def get_opener(self, name):
         if name not in self.registry:
-            raise NoOpenerError("No opener for [%s]" % name)
+            raise NoOpenerError("No opener for %s" % name)
         index = self.registry[name]
         return self.openers[index]        
     
@@ -73,34 +80,56 @@ class OpenerRegistry(object):
             self.registry[name] = index
     
     def parse(self, fs_url, default_fs_name=None, writeable=False, create=False):
-                        
-        fs_name, paren_url, fs_url, path = self.split_segments(fs_url)
+                   
+        orig_url = fs_url     
+        match = self.split_segments(fs_url)
         
-        fs_url = fs_url or paren_url
-        if fs_name is None and path is None:
-            fs_url = os.path.expanduser(os.path.expandvars(fs_url))
-            fs_url = os.path.normpath(os.path.abspath(fs_url))
-            fs_url, path = pathsplit(fs_url)
-            if not fs_url:
-                fs_url = '/'
+        if match:
+            fs_name, paren_url, fs_url, path = match.groups()                                
+            fs_url = fs_url or paren_url or ''          
+            if ':' in fs_name:
+                fs_name, sub_protocol = fs_name.split(':', 1)
+                fs_url = '%s://%s' % (sub_protocol, fs_url)
+            
+            fs_name = fs_name or self.default_opener                                                                    
+                
+        else:
+            fs_name = default_fs_name or self.default_opener
+            fs_url = _expand_syspath(fs_url) 
+            path = ''           
+
+    
+        fs_name,  fs_name_params = self.parse_name(fs_name)        
+        opener = self.get_opener(fs_name)
         
-        fs_name = fs_name or self.default_opener                                
+        if fs_url is None:
+            raise OpenerError("Unable to parse '%s'" % orig_url)
         
-        if fs_name is None:
-            fs_name = fs_default_name
+        wildcard = None
+        if iswildcard(fs_url):
+            fs_url, wildcard = pathsplit(fs_url)
         
-        fs_name,  fs_name_params = self.parse_name(fs_name)        
-        opener = self.get_opener(fs_name)
-                   
-        fs = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create)
+        fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create)
+                
+        if wildcard:
+            fs_path = join(fs_path or '', wildcard)
+        else:
+            path = join(fs_path or '', path)
         
         if path:
             pathname, resourcename = pathsplit(path)
             if pathname:
                 fs = fs.opendir(pathname)
                 path = resourcename
+            if not iswildcard(path):
+                if fs.isdir(path):
+                    fs = fs.opendir(path)
+                    fs_path = ''
+                else:
+                    fs_path = path
+                
         
-        return fs, path
+        return fs, fs_path
     
     def parse_credentials(self, url):
         
@@ -145,11 +174,21 @@ class OSFSOpener(Opener):
     names = ['osfs', 'file']
     
     @classmethod
-    def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create): 
-        username, password, fs_path = registry.parse_credentials(fs_path)       
-        from fs.osfs import OSFS
-        osfs = OSFS(fs_path, create=create)
-        return osfs
+    def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create):
+        from fs.osfs import OSFS 
+        username, password, fs_path = registry.parse_credentials(fs_path)                
+        
+                
+        path = _expand_syspath(fs_path)
+        if create:
+            sys.makedirs(fs_path)
+        if os.path.isdir(path):
+            osfs = OSFS(path)
+            filepath = None
+        else:
+            path, filepath = pathsplit(path)
+            osfs = OSFS(path, create=create)
+        return osfs, filepath
         
         
 class ZipOpener(Opener):
@@ -157,22 +196,21 @@ class ZipOpener(Opener):
     
     @classmethod
     def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create):
-                        
-        create_zip = fs_name_params == 'new' 
-        append_zip = fs_name_params == 'append'       
-        
-        zip_file = None
-        if fs_path.startswith('['):
-            container_fs, container_path = registry.parse(fs_path)
-            if not container_path:
-                raise OpenerError("Not a file")
-            container_mode = 'r+b'
-            if create_zip:
-                container_mode = 'w+b'
-            elif writeable:
-                container_mode = 'w+b'
+                                
+        append_zip = fs_name_params == 'add'     
                 
-            zip_file = container_fs.open(container_path, mode=container_mode)                                      
+        zip_fs, zip_path = registry.parse(fs_path)
+        if zip_path is None:
+            raise OpenerError('File required for zip opener')
+        if create:
+            open_mode = 'wb'        
+            if append_zip:
+                open_mode = 'r+b'
+        else:
+            open_mode = 'rb'
+                    
+        zip_file = zip_fs.open(zip_path, mode=open_mode)         
+                                            
                         
         username, password, fs_path = registry.parse_credentials(fs_path)
         
@@ -182,21 +220,18 @@ class ZipOpener(Opener):
            
         if append_zip:
             mode = 'a'            
-        elif create_zip or create:
+        elif create:
             mode = 'w'
         else:
             if writeable:
                 mode = 'w'
             else:
                 mode = 'a'        
-                
-        if fs_name == 'zip64':
-            allow_zip_64 = True
-        else:
-            allow_zip_64 = False  
+         
+        allow_zip_64 = fs_name == 'zip64'                
               
         zipfs = ZipFS(zip_file, mode=mode, allow_zip_64=allow_zip_64)
-        return zipfs
+        return zipfs, None
     
 class RPCOpener(Opener):
     names = ['rpc']
@@ -206,9 +241,16 @@ class RPCOpener(Opener):
         from fs.rpcfs import RPCFS             
         username, password, fs_path = registry.parse_credentials(fs_path)
         if not fs_path.startswith('http://'):
-            fs_path = 'http://' + fs_path                
-        rpcfs = RPCFS(fs_path)
-        return rpcfs
+            fs_path = 'http://' + fs_path
+            
+        scheme, netloc, path, params, query, fragment = urlparse(fs_path)
+
+        rpcfs = RPCFS('%s://%s' % (scheme, netloc))
+        
+        if create and path:
+            rpcfs.makedir(path, recursive=True, allow_recreate=True)
+        
+        return rpcfs, path or None
 
 class FTPOpener(Opener):
     names = ['ftp']
@@ -217,22 +259,28 @@ class FTPOpener(Opener):
     def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create):
         from fs.ftpfs import FTPFS
         username, password, fs_path = registry.parse_credentials(fs_path)
-        
-        if '/' in fs_path:
-            url, root_path = fs_path.split('/', 1)
-        else:
-            url = fs_path
-            root_path = ''
-                                                                
+                    
+        scheme, netloc, path, params, query, fragment = urlparse(fs_path)
+        if not scheme:
+            fs_path = 'ftp://' + fs_path
+        scheme, netloc, path, params, query, fragment = urlparse(fs_path)
+                 
+        dirpath, resourcepath = pathsplit(path)        
+        url = netloc
+                                
         ftpfs = FTPFS(url, user=username or '', passwd=password or '')
         ftpfs.cache_hint(True)
         
-        if root_path not in ('', '/'):
-            if not ftpfs.isdir(root_path):
-                raise OpenerError("'%s' is not a directory on the server" % root_path)
-            return ftpfs.opendir(root_path)
+        if create and path:
+            ftpfs.makedir(path, recursive=True, allow_recreate=True)
         
-        return ftpfs
+        if dirpath:
+            ftpfs = ftpfs.opendir(dirpath)
+                            
+        if not resourcepath:
+            return ftpfs, None        
+        else:
+            return ftpfs, resourcepath
 
 
 class SFTPOpener(Opener):
@@ -256,6 +304,8 @@ class SFTPOpener(Opener):
             addr = fs_path
             fs_path = '/'
             
+        fs_path, resourcename = pathsplit(fs_path)
+            
         host = addr
         port = None
         if ':' in host:
@@ -265,10 +315,25 @@ class SFTPOpener(Opener):
             except ValueError:
                 pass
             else:
-                host = (addr, port)                            
+                host = (addr, port)
+            
+        #if not username or not password:
+        #    raise OpenerError('SFTP requires authentication')
+            
+        if create:
+            sftpfs = SFTPFS(host, root_path='/', **credentials)
+            if not sftpfs._transport.is_authenticated():
+                sftpfs.close()
+                raise OpenerError('SFTP requires authentication')
+            sftpfs = sfspfs.makeopendir(fs_path)
+            return sftpfs, None
                 
         sftpfs = SFTPFS(host, root_path=fs_path, **credentials)
-        return sftpfs
+        if not sftpfs._transport.is_authenticated():
+            sftpfs.close()
+            raise OpenerError('SFTP requires authentication')            
+            
+        return sftpfs, resourcename
     
     
 class MemOpener(Opener):
@@ -277,7 +342,10 @@ class MemOpener(Opener):
     @classmethod
     def get_fs(cls, registry, fs_name, fs_name_params, fs_path,  writeable, create):
         from fs.memoryfs import MemoryFS
-        return MemoryFS()
+        memfs = MemoryFS()
+        if create:
+            memfs = memfs.makeopendir(fs_path)
+        return memfs, None
     
 class DebugOpener(Opener):
     names = ['debug']
@@ -287,13 +355,13 @@ class DebugOpener(Opener):
         from fs.wrapfs.debugfs import DebugFS
         if fs_path:
             fs, path = registry.parse(fs_path, writeable=writeable, create=create)
-            return DebugFS(fs, verbose=False)     
+            return DebugFS(fs, verbose=False), None     
         if fs_name_params == 'ram':
             from fs.memoryfs import MemoryFS
-            return DebugFS(MemoryFS(), identifier=fs_name_params, verbose=False)
+            return DebugFS(MemoryFS(), identifier=fs_name_params, verbose=False), None
         else:
             from fs.tempfs import TempFS
-            return DebugFS(TempFS(), identifier=fs_name_params, verbose=False)
+            return DebugFS(TempFS(), identifier=fs_name_params, verbose=False), None
     
 class TempOpener(Opener):
     names = ['temp']
@@ -301,7 +369,7 @@ class TempOpener(Opener):
     @classmethod
     def get_fs(cls, registry, fs_name, fs_name_params, fs_path,  writeable, create):
         from fs.tempfs import TempFS        
-        return TempFS(identifier=fs_name_params, temp_dir=fs_path)
+        return TempFS(identifier=fs_name_params, temp_dir=fs_path), None
     
 
 opener = OpenerRegistry([OSFSOpener,
@@ -317,7 +385,7 @@ opener = OpenerRegistry([OSFSOpener,
 
 def main():
     
-    fs, path = opener.parse('galleries.zip')
+    fs, path = opener.parse('sftp://willmcgugan.com')
     print fs, path
     
 if __name__ == "__main__":