diff options
| -rw-r--r-- | ChangeLog | 1 | ||||
| -rw-r--r-- | docs/expose.rst | 4 | ||||
| -rw-r--r-- | docs/expose/importhook.rst | 2 | ||||
| -rw-r--r-- | docs/expose/index.rst | 1 | ||||
| -rw-r--r-- | fs/contrib/davfs/__init__.py | 2 | ||||
| -rw-r--r-- | fs/expose/importhook.py | 238 | ||||
| -rw-r--r-- | fs/opener.py | 3 | ||||
| -rw-r--r-- | fs/tests/test_importhook.py | 141 |
8 files changed, 390 insertions, 2 deletions
@@ -39,6 +39,7 @@ * TahoeFS: access files stored in a Tahoe-LAFS grid * New fs.expose implementations: * dokan: mount an FS object as a drive using Dokan (win32-only) + * importhook: import modules from files in an FS object * Modified listdir and walk methods to accept callables as well as strings for wildcards. * Added listdirinfo method, which yields both the entry names and the diff --git a/docs/expose.rst b/docs/expose.rst index e4ee403..0e50274 100644 --- a/docs/expose.rst +++ b/docs/expose.rst @@ -20,6 +20,10 @@ XMLRPC ------ Makes an FS object available over XMLRPC. See :mod:`fs.expose.xmlrpc` +Import Hook +----------- +Allows importing python modules from the files in an FS object. See :mod:`fs.expose.importhook` + Django Storage -------------- Connects FS objects to Django. See :mod:`fs.expose.django_storage` diff --git a/docs/expose/importhook.rst b/docs/expose/importhook.rst new file mode 100644 index 0000000..dd30115 --- /dev/null +++ b/docs/expose/importhook.rst @@ -0,0 +1,2 @@ +.. automodule:: fs.expose.importhook + :members: diff --git a/docs/expose/index.rst b/docs/expose/index.rst index 19ee319..6a185e5 100644 --- a/docs/expose/index.rst +++ b/docs/expose/index.rst @@ -10,4 +10,5 @@ The ``fs.expose`` module contains a number of options for making an FS implement dokan.rst sftp.rst xmlrpc.rst + importhook.rst django_storage.rst diff --git a/fs/contrib/davfs/__init__.py b/fs/contrib/davfs/__init__.py index 5190807..b6e291e 100644 --- a/fs/contrib/davfs/__init__.py +++ b/fs/contrib/davfs/__init__.py @@ -254,7 +254,7 @@ class DAVFS(FS): resp = con.getresponse() self._cookiejar.extract_cookies(FakeResp(resp),FakeReq(con,url.scheme,url.path)) except Exception, e: - #logger.debug("DAVFS <ERR %s %s/%s",resp.status,method,url.hostname,url.path) + #logger.debug("DAVFS <ERR %s %s/%s",e,method,url.hostname,url.path) self._del_connection(con) raise else: diff --git a/fs/expose/importhook.py b/fs/expose/importhook.py new file mode 100644 index 0000000..0cd0413 --- /dev/null +++ b/fs/expose/importhook.py @@ -0,0 +1,238 @@ +""" +fs.expose.importhook +==================== + +Expose an FS object to the python import machinery, via a PEP-302 loader. + +This module allows you to import python modules from an arbitrary FS object, +by placing FS urls on sys.path and/or inserting objects into sys.meta_path. + +The main class in this module is FSImportHook, which is a PEP-302-compliant +module finder and loader. If you place an instance of FSImportHook on +sys.meta_path, you will be able to import modules from the exposed filesystem:: + + >>> from fs.memoryfs import MemoryFS + >>> m = MemoryFS() + >>> m.setcontents("helloworld.py","print 'hello world!") + >>> + >>> import sys + >>> from fs.expose.importhook import FSImportHook + >>> sys.meta_path.append(FSImportHook(m)) + >>> import helloworld + hello world! + +It is also possible to install FSImportHook as an import path handler. This +allows you to place filesystem URLs on sys.path and have them automagically +opened for importing. This example would allow modules to be imported from +an SFTP server:: + + >>> from fs.expose.importhook import FSImportHook + >>> FSImportHook.install() + >>> sys.path.append("sftp://some.remote.machine/mypath/") + +""" + +import sys +import imp +import marshal + +from fs.base import FS +from fs.opener import fsopendir, OpenerError +from fs.errors import * +from fs.path import * + +class FSImportHook(object): + """PEP-302-compliant module finder and loader for FS objects. + + FSImportHook is a module finder and loader that takes its data from an + arbitrary FS object. The FS must have .py or .pyc files stored in the + standard module structure. + + For easy use with sys.path, FSImportHook will also accept a filesystem + URL, which is automatically opened using fs.opener. + """ + + _VALID_MODULE_TYPES = set((imp.PY_SOURCE,imp.PY_COMPILED)) + + def __init__(self,fs_or_url): + # If given a string, try to open it as an FS url. + # Don't open things on the local filesystem though. + if isinstance(fs_or_url,basestring): + if ":" not in fs_or_url: + raise ImportError + try: + self.fs = fsopendir(fs_or_url) + except OpenerError: + raise ImportError + self.path = fs_or_url + # Otherwise, it must be an FS object of some sort. + else: + if not isinstance(fs_or_url,FS): + raise ImportError + self.fs = fs_or_url + self.path = None + + @classmethod + def install(cls): + """Install this class into the import machinery. + + This classmethod installs the custom FSImportHook class into the + import machinery of the running process, if it is not already + installed. + """ + for i,imp in enumerate(sys.path_hooks): + try: + if issubclass(cls,imp): + break + except TypeError: + pass + else: + sys.path_hooks.append(cls) + sys.path_importer_cache.clear() + + @classmethod + def uninstall(cls): + """Uninstall this class from the import machinery. + + This classmethod uninstalls the custom FSImportHook class from the + import machinery of the running process. + """ + to_rem = [] + for i,imp in enumerate(sys.path_hooks): + try: + if issubclass(cls,imp): + to_rem.append(imp) + break + except TypeError: + pass + for imp in to_rem: + sys.path_hooks.remove(imp) + sys.path_importer_cache.clear() + + def find_module(self,fullname,path=None): + """Find the FS loader for the given module. + + This object is always its own loader, so this really just checks + whether it's a valid module on the exposed filesystem. + """ + try: + self._get_module_info(fullname) + except ImportError: + return None + else: + return self + + def _get_module_info(self,fullname): + """Get basic information about the given module. + + If the specified module exists, this method returns a tuple giving + its filepath, file type and whether it's a package. Otherwise, + it raise ImportError. + """ + prefix = fullname.replace(".","/") + # Is it a regular module? + (path,type) = self._find_module_file(prefix) + if path is not None: + return (path,type,False) + # Is it a package? + prefix = pathjoin(prefix,"__init__") + (path,type) = self._find_module_file(prefix) + if path is not None: + return (path,type,True) + # No, it's nothing + raise ImportError(fullname) + + def _find_module_file(self,prefix): + """Find a module file from the given path prefix. + + This method iterates over the possible module suffixes, checking each + in turn and returning the first match found. It returns a two-tuple + (path,type) or (None,None) if there's no module. + """ + for (suffix,mode,type) in imp.get_suffixes(): + if type in self._VALID_MODULE_TYPES: + path = prefix + suffix + if self.fs.isfile(path): + return (path,type) + return (None,None) + + def load_module(self,fullname): + """Load the specified module. + + This method locates the file for the specified module, loads and + executes it and returns the created module object. + """ + # Reuse an existing module if present. + try: + return sys.modules[fullname] + except KeyError: + pass + # Try to create from source or bytecode. + info = self._get_module_info(fullname) + code = self.get_code(fullname,info) + if code is None: + raise ImportError(fullname) + mod = imp.new_module(fullname) + mod.__file__ = "<loading>" + mod.__loader__ = self + sys.modules[fullname] = mod + try: + exec code in mod.__dict__ + mod.__file__ = self.get_filename(fullname,info) + if self.is_package(fullname,info): + if self.path is None: + mod.__path__ = [] + else: + mod.__path__ = [self.path] + return mod + except Exception: + sys.modules.pop(fullname,None) + raise + + def is_package(self,fullname,info=None): + """Check whether the specified module is a package.""" + if info is None: + info = self._get_module_info(fullname) + (path,type,ispkg) = info + return ispkg + + def get_code(self,path,info=None): + """Get the bytecode for the specified module.""" + if info is None: + info = self._get_module_info(fullname) + (path,type,ispkg) = info + code = self.fs.getcontents(path) + if type == imp.PY_SOURCE: + code = code.replace("\r\n","\n") + return compile(code,path,"exec") + elif type == imp.PY_COMPILED: + if code[:4] != imp.get_magic(): + return None + return marshal.loads(code[8:]) + else: + return None + return code + + def get_source(self,fullname,info=None): + """Get the sourcecode for the specified module, if present.""" + if info is None: + info = self._get_module_info(fullname) + (path,type,ispkg) = info + if type != imp.PY_SOURCE: + return None + return self.fs.getcontents(path).replace("\r\n","\n") + + def get_data(self,path): + """Read the specified data file.""" + try: + return self.fs.getcontents(path) + except FSError, e: + raise IOError(str(e)) + + def get_filename(self,fullname,info=None): + """Get the __file__ attribute for the specified module.""" + if info is None: + info = self._get_module_info(fullname) + (path,type,ispkg) = info + return path + diff --git a/fs/opener.py b/fs/opener.py index 0bb7b5c..560c581 100644 --- a/fs/opener.py +++ b/fs/opener.py @@ -538,7 +538,8 @@ example: @classmethod def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create_dir): from fs.tempfs import TempFS - fs = TempFS(identifier=fs_name_params) + from fs.wrapfs.lazyfs import LazyFS + fs = LazyFS((TempFS,(),{"identifier":fs_name_params})) return fs, fs_path class S3Opener(Opener): diff --git a/fs/tests/test_importhook.py b/fs/tests/test_importhook.py new file mode 100644 index 0000000..86659b2 --- /dev/null +++ b/fs/tests/test_importhook.py @@ -0,0 +1,141 @@ + +import sys +import unittest +import marshal +import imp +import struct +from textwrap import dedent + +from fs.expose.importhook import FSImportHook +from fs.tempfs import TempFS +from fs.zipfs import ZipFS + + +class TestFSImportHook(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + for mph in list(sys.meta_path): + if isinstance(mph,FSImportHook): + sys.meta_path.remove(mph) + for ph in list(sys.path_hooks): + if issubclass(ph,FSImportHook): + sys.path_hooks.remove(mph) + for (k,v) in sys.modules.items(): + if k.startswith("fsih_"): + del sys.modules[k] + elif hasattr(v,"__loader__"): + if isinstance(v.__loader__,FSImportHook): + del sys.modules[k] + sys.path_importer_cache.clear() + + def _init_modules(self,fs): + fs.setcontents("fsih_hello.py",dedent(""" + message = 'hello world!' + """)) + fs.makedir("fsih_pkg") + fs.setcontents("fsih_pkg/__init__.py",dedent(""" + a = 42 + """)) + fs.setcontents("fsih_pkg/sub1.py",dedent(""" + import fsih_pkg + from fsih_hello import message + a = fsih_pkg.a + """)) + fs.setcontents("fsih_pkg/sub2.pyc",self._getpyc(dedent(""" + import fsih_pkg + from fsih_hello import message + a = fsih_pkg.a * 2 + """))) + + def _getpyc(self,src): + """Get the .pyc contents to match th given .py source code.""" + code = imp.get_magic() + struct.pack("<i",0) + code += marshal.dumps(compile(src,__file__,"exec")) + return code + + def test_loader_methods(self): + t = TempFS() + self._init_modules(t) + ih = FSImportHook(t) + sys.meta_path.append(ih) + try: + self.assertEquals(ih.find_module("fsih_hello"),ih) + self.assertEquals(ih.find_module("fsih_helo"),None) + self.assertEquals(ih.find_module("fsih_pkg"),ih) + self.assertEquals(ih.find_module("fsih_pkg.sub1"),ih) + self.assertEquals(ih.find_module("fsih_pkg.sub2"),ih) + self.assertEquals(ih.find_module("fsih_pkg.sub3"),None) + m = ih.load_module("fsih_hello") + self.assertEquals(m.message,"hello world!") + self.assertRaises(ImportError,ih.load_module,"fsih_helo") + m = ih.load_module("fsih_pkg.sub1") + self.assertEquals(m.message,"hello world!") + self.assertEquals(m.a,42) + m = ih.load_module("fsih_pkg.sub2") + self.assertEquals(m.message,"hello world!") + self.assertEquals(m.a,42 * 2) + self.assertRaises(ImportError,ih.load_module,"fsih_pkg.sub3") + finally: + sys.meta_path.remove(ih) + t.close() + + def _check_imports_are_working(self): + try: + import fsih_hello + self.assertEquals(fsih_hello.message,"hello world!") + try: + import fsih_helo + except ImportError: + pass + else: + assert False, "ImportError not raised" + import fsih_pkg + import fsih_pkg.sub1 + self.assertEquals(fsih_pkg.sub1.message,"hello world!") + self.assertEquals(fsih_pkg.sub1.a,42) + import fsih_pkg.sub2 + self.assertEquals(fsih_pkg.sub2.message,"hello world!") + self.assertEquals(fsih_pkg.sub2.a,42 * 2) + try: + import fsih_pkg.sub3 + except ImportError: + pass + else: + assert False, "ImportError not raised" + finally: + for k in sys.modules.keys(): + if k.startswith("fsih_"): + del sys.modules[k] + + def test_importer_on_meta_path(self): + t = TempFS() + self._init_modules(t) + ih = FSImportHook(t) + sys.meta_path.append(ih) + try: + self._check_imports_are_working() + finally: + sys.meta_path.remove(ih) + t.close() + + def test_url_on_sys_path(self): + t = TempFS() + zpath = t.getsyspath("modules.zip") + z = ZipFS(zpath,"w") + self._init_modules(z) + z.close() + z = ZipFS(zpath,"r") + assert z.isfile("fsih_hello.py") + z.close() + sys.path.append("zip://" + zpath) + FSImportHook.install() + try: + self._check_imports_are_working() + finally: + sys.path_hooks.remove(FSImportHook) + sys.path.pop() + t.close() + |