diff options
Diffstat (limited to 'pyximport/pyximport.py')
| -rw-r--r-- | pyximport/pyximport.py | 607 |
1 files changed, 6 insertions, 601 deletions
diff --git a/pyximport/pyximport.py b/pyximport/pyximport.py index b2077826a..9d575815a 100644 --- a/pyximport/pyximport.py +++ b/pyximport/pyximport.py @@ -1,606 +1,11 @@ -""" -Import hooks; when installed with the install() function, these hooks -allow importing .pyx files as if they were Python modules. - -If you want the hook installed every time you run Python -you can add it to your Python version by adding these lines to -sitecustomize.py (which you can create from scratch in site-packages -if it doesn't exist there or somewhere else on your python path):: - - import pyximport - pyximport.install() - -For instance on the Mac with a non-system Python 2.3, you could create -sitecustomize.py with only those two lines at -/usr/local/lib/python2.3/site-packages/sitecustomize.py . - -A custom distutils.core.Extension instance and setup() args -(Distribution) for for the build can be defined by a <modulename>.pyxbld -file like: - -# examplemod.pyxbld -def make_ext(modname, pyxfilename): - from distutils.extension import Extension - return Extension(name = modname, - sources=[pyxfilename, 'hello.c'], - include_dirs=['/myinclude'] ) -def make_setup_args(): - return dict(script_args=["--compiler=mingw32"]) - -Extra dependencies can be defined by a <modulename>.pyxdep . -See README. - -Since Cython 0.11, the :mod:`pyximport` module also has experimental -compilation support for normal Python modules. This allows you to -automatically run Cython on every .pyx and .py module that Python -imports, including parts of the standard library and installed -packages. Cython will still fail to compile a lot of Python modules, -in which case the import mechanism will fall back to loading the -Python source modules instead. The .py import mechanism is installed -like this:: - - pyximport.install(pyimport = True) - -Running this module as a top-level script will run a test and then print -the documentation. - -This code is based on the Py2.3+ import protocol as described in PEP 302. -""" - -import glob -import imp -import os +from __future__ import absolute_import import sys -from zipimport import zipimporter, ZipImportError - -mod_name = "pyximport" - -PYX_EXT = ".pyx" -PYXDEP_EXT = ".pyxdep" -PYXBLD_EXT = ".pyxbld" - -DEBUG_IMPORT = False - - -def _print(message, args): - if args: - message = message % args - print(message) - - -def _debug(message, *args): - if DEBUG_IMPORT: - _print(message, args) - - -def _info(message, *args): - _print(message, args) - - -# Performance problem: for every PYX file that is imported, we will -# invoke the whole distutils infrastructure even if the module is -# already built. It might be more efficient to only do it when the -# mod time of the .pyx is newer than the mod time of the .so but -# the question is how to get distutils to tell me the name of the .so -# before it builds it. Maybe it is easy...but maybe the performance -# issue isn't real. -def _load_pyrex(name, filename): - "Load a pyrex file given a name and filename." - - -def get_distutils_extension(modname, pyxfilename, language_level=None): -# try: -# import hashlib -# except ImportError: -# import md5 as hashlib -# extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest() -# modname = modname + extra - extension_mod,setup_args = handle_special_build(modname, pyxfilename) - if not extension_mod: - if not isinstance(pyxfilename, str): - # distutils is stupid in Py2 and requires exactly 'str' - # => encode accidentally coerced unicode strings back to str - pyxfilename = pyxfilename.encode(sys.getfilesystemencoding()) - from distutils.extension import Extension - extension_mod = Extension(name = modname, sources=[pyxfilename]) - if language_level is not None: - extension_mod.cython_directives = {'language_level': language_level} - return extension_mod,setup_args - - -def handle_special_build(modname, pyxfilename): - special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT - ext = None - setup_args={} - if os.path.exists(special_build): - # globls = {} - # locs = {} - # execfile(special_build, globls, locs) - # ext = locs["make_ext"](modname, pyxfilename) - with open(special_build) as fid: - mod = imp.load_source("XXXX", special_build, fid) - make_ext = getattr(mod,'make_ext',None) - if make_ext: - ext = make_ext(modname, pyxfilename) - assert ext and ext.sources, "make_ext in %s did not return Extension" % special_build - make_setup_args = getattr(mod, 'make_setup_args',None) - if make_setup_args: - setup_args = make_setup_args() - assert isinstance(setup_args,dict), ("make_setup_args in %s did not return a dict" - % special_build) - assert set or setup_args, ("neither make_ext nor make_setup_args %s" - % special_build) - ext.sources = [os.path.join(os.path.dirname(special_build), source) - for source in ext.sources] - return ext, setup_args - - -def handle_dependencies(pyxfilename): - testing = '_test_files' in globals() - dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT - - # by default let distutils decide whether to rebuild on its own - # (it has a better idea of what the output file will be) - - # but we know more about dependencies so force a rebuild if - # some of the dependencies are newer than the pyxfile. - if os.path.exists(dependfile): - with open(dependfile) as fid: - depends = fid.readlines() - depends = [depend.strip() for depend in depends] - - # gather dependencies in the "files" variable - # the dependency file is itself a dependency - files = [dependfile] - for depend in depends: - fullpath = os.path.join(os.path.dirname(dependfile), - depend) - files.extend(glob.glob(fullpath)) - - # only for unit testing to see we did the right thing - if testing: - _test_files[:] = [] #$pycheck_no - - # if any file that the pyxfile depends upon is newer than - # the pyx file, 'touch' the pyx file so that distutils will - # be tricked into rebuilding it. - for file in files: - from distutils.dep_util import newer - if newer(file, pyxfilename): - _debug("Rebuilding %s because of %s", pyxfilename, file) - filetime = os.path.getmtime(file) - os.utime(pyxfilename, (filetime, filetime)) - if testing: - _test_files.append(file) - - -def build_module(name, pyxfilename, pyxbuild_dir=None, inplace=False, language_level=None): - assert os.path.exists(pyxfilename), "Path does not exist: %s" % pyxfilename - handle_dependencies(pyxfilename) - - extension_mod, setup_args = get_distutils_extension(name, pyxfilename, language_level) - build_in_temp = pyxargs.build_in_temp - sargs = pyxargs.setup_args.copy() - sargs.update(setup_args) - build_in_temp = sargs.pop('build_in_temp',build_in_temp) - - from . import pyxbuild - so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod, - build_in_temp=build_in_temp, - pyxbuild_dir=pyxbuild_dir, - setup_args=sargs, - inplace=inplace, - reload_support=pyxargs.reload_support) - assert os.path.exists(so_path), "Cannot find: %s" % so_path - - junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ? yes, indeed, trying to eat my files ;) - junkstuff = glob.glob(junkpath) - for path in junkstuff: - if path != so_path: - try: - os.remove(path) - except IOError: - _info("Couldn't remove %s", path) - - return so_path - - -def load_module(name, pyxfilename, pyxbuild_dir=None, is_package=False, - build_inplace=False, language_level=None, so_path=None): - try: - if so_path is None: - if is_package: - module_name = name + '.__init__' - else: - module_name = name - so_path = build_module(module_name, pyxfilename, pyxbuild_dir, - inplace=build_inplace, language_level=language_level) - mod = imp.load_dynamic(name, so_path) - if is_package and not hasattr(mod, '__path__'): - mod.__path__ = [os.path.dirname(so_path)] - assert mod.__file__ == so_path, (mod.__file__, so_path) - except Exception as failure_exc: - _debug("Failed to load extension module: %r" % failure_exc) - if pyxargs.load_py_module_on_import_failure and pyxfilename.endswith('.py'): - # try to fall back to normal import - mod = imp.load_source(name, pyxfilename) - assert mod.__file__ in (pyxfilename, pyxfilename+'c', pyxfilename+'o'), (mod.__file__, pyxfilename) - else: - tb = sys.exc_info()[2] - import traceback - exc = ImportError("Building module %s failed: %s" % ( - name, traceback.format_exception_only(*sys.exc_info()[:2]))) - if sys.version_info[0] >= 3: - raise exc.with_traceback(tb) - else: - exec("raise exc, None, tb", {'exc': exc, 'tb': tb}) - return mod - - -# import hooks - -class PyxImporter(object): - """A meta-path importer for .pyx files. - """ - def __init__(self, extension=PYX_EXT, pyxbuild_dir=None, inplace=False, - language_level=None): - self.extension = extension - self.pyxbuild_dir = pyxbuild_dir - self.inplace = inplace - self.language_level = language_level - - def find_module(self, fullname, package_path=None): - if fullname in sys.modules and not pyxargs.reload_support: - return None # only here when reload() - - # package_path might be a _NamespacePath. Convert that into a list... - if package_path is not None and not isinstance(package_path, list): - package_path = list(package_path) - try: - fp, pathname, (ext,mode,ty) = imp.find_module(fullname,package_path) - if fp: fp.close() # Python should offer a Default-Loader to avoid this double find/open! - if pathname and ty == imp.PKG_DIRECTORY: - pkg_file = os.path.join(pathname, '__init__'+self.extension) - if os.path.isfile(pkg_file): - return PyxLoader(fullname, pathname, - init_path=pkg_file, - pyxbuild_dir=self.pyxbuild_dir, - inplace=self.inplace, - language_level=self.language_level) - if pathname and pathname.endswith(self.extension): - return PyxLoader(fullname, pathname, - pyxbuild_dir=self.pyxbuild_dir, - inplace=self.inplace, - language_level=self.language_level) - if ty != imp.C_EXTENSION: # only when an extension, check if we have a .pyx next! - return None - - # find .pyx fast, when .so/.pyd exist --inplace - pyxpath = os.path.splitext(pathname)[0]+self.extension - if os.path.isfile(pyxpath): - return PyxLoader(fullname, pyxpath, - pyxbuild_dir=self.pyxbuild_dir, - inplace=self.inplace, - language_level=self.language_level) - - # .so/.pyd's on PATH should not be remote from .pyx's - # think no need to implement PyxArgs.importer_search_remote here? - - except ImportError: - pass - - # searching sys.path ... - - #if DEBUG_IMPORT: print "SEARCHING", fullname, package_path - - mod_parts = fullname.split('.') - module_name = mod_parts[-1] - pyx_module_name = module_name + self.extension - - # this may work, but it returns the file content, not its path - #import pkgutil - #pyx_source = pkgutil.get_data(package, pyx_module_name) - - paths = package_path or sys.path - for path in paths: - pyx_data = None - if not path: - path = os.getcwd() - elif os.path.isfile(path): - try: - zi = zipimporter(path) - pyx_data = zi.get_data(pyx_module_name) - except (ZipImportError, IOError, OSError): - continue # Module not found. - # unzip the imported file into the build dir - # FIXME: can interfere with later imports if build dir is in sys.path and comes before zip file - path = self.pyxbuild_dir - elif not os.path.isabs(path): - path = os.path.abspath(path) - - pyx_module_path = os.path.join(path, pyx_module_name) - if pyx_data is not None: - if not os.path.exists(path): - try: - os.makedirs(path) - except OSError: - # concurrency issue? - if not os.path.exists(path): - raise - with open(pyx_module_path, "wb") as f: - f.write(pyx_data) - elif not os.path.isfile(pyx_module_path): - continue # Module not found. - - return PyxLoader(fullname, pyx_module_path, - pyxbuild_dir=self.pyxbuild_dir, - inplace=self.inplace, - language_level=self.language_level) - - # not found, normal package, not a .pyx file, none of our business - _debug("%s not found" % fullname) - return None - - -class PyImporter(PyxImporter): - """A meta-path importer for normal .py files. - """ - def __init__(self, pyxbuild_dir=None, inplace=False, language_level=None): - if language_level is None: - language_level = sys.version_info[0] - self.super = super(PyImporter, self) - self.super.__init__(extension='.py', pyxbuild_dir=pyxbuild_dir, inplace=inplace, - language_level=language_level) - self.uncompilable_modules = {} - self.blocked_modules = ['Cython', 'pyxbuild', 'pyximport.pyxbuild', - 'distutils'] - self.blocked_packages = ['Cython.', 'distutils.'] - - def find_module(self, fullname, package_path=None): - if fullname in sys.modules: - return None - if any([fullname.startswith(pkg) for pkg in self.blocked_packages]): - return None - if fullname in self.blocked_modules: - # prevent infinite recursion - return None - if _lib_loader.knows(fullname): - return _lib_loader - _debug("trying import of module '%s'", fullname) - if fullname in self.uncompilable_modules: - path, last_modified = self.uncompilable_modules[fullname] - try: - new_last_modified = os.stat(path).st_mtime - if new_last_modified > last_modified: - # import would fail again - return None - except OSError: - # module is no longer where we found it, retry the import - pass - - self.blocked_modules.append(fullname) - try: - importer = self.super.find_module(fullname, package_path) - if importer is not None: - if importer.init_path: - path = importer.init_path - real_name = fullname + '.__init__' - else: - path = importer.path - real_name = fullname - _debug("importer found path %s for module %s", path, real_name) - try: - so_path = build_module( - real_name, path, - pyxbuild_dir=self.pyxbuild_dir, - language_level=self.language_level, - inplace=self.inplace) - _lib_loader.add_lib(fullname, path, so_path, - is_package=bool(importer.init_path)) - return _lib_loader - except Exception: - if DEBUG_IMPORT: - import traceback - traceback.print_exc() - # build failed, not a compilable Python module - try: - last_modified = os.stat(path).st_mtime - except OSError: - last_modified = 0 - self.uncompilable_modules[fullname] = (path, last_modified) - importer = None - finally: - self.blocked_modules.pop() - return importer - - -class LibLoader(object): - def __init__(self): - self._libs = {} - - def load_module(self, fullname): - try: - source_path, so_path, is_package = self._libs[fullname] - except KeyError: - raise ValueError("invalid module %s" % fullname) - _debug("Loading shared library module '%s' from %s", fullname, so_path) - return load_module(fullname, source_path, so_path=so_path, is_package=is_package) - - def add_lib(self, fullname, path, so_path, is_package): - self._libs[fullname] = (path, so_path, is_package) - - def knows(self, fullname): - return fullname in self._libs - -_lib_loader = LibLoader() - - -class PyxLoader(object): - def __init__(self, fullname, path, init_path=None, pyxbuild_dir=None, - inplace=False, language_level=None): - _debug("PyxLoader created for loading %s from %s (init path: %s)", - fullname, path, init_path) - self.fullname = fullname - self.path, self.init_path = path, init_path - self.pyxbuild_dir = pyxbuild_dir - self.inplace = inplace - self.language_level = language_level - - def load_module(self, fullname): - assert self.fullname == fullname, ( - "invalid module, expected %s, got %s" % ( - self.fullname, fullname)) - if self.init_path: - # package - #print "PACKAGE", fullname - module = load_module(fullname, self.init_path, - self.pyxbuild_dir, is_package=True, - build_inplace=self.inplace, - language_level=self.language_level) - module.__path__ = [self.path] - else: - #print "MODULE", fullname - module = load_module(fullname, self.path, - self.pyxbuild_dir, - build_inplace=self.inplace, - language_level=self.language_level) - return module - - -#install args -class PyxArgs(object): - build_dir=True - build_in_temp=True - setup_args={} #None - -##pyxargs=None - - -def _have_importers(): - has_py_importer = False - has_pyx_importer = False - for importer in sys.meta_path: - if isinstance(importer, PyxImporter): - if isinstance(importer, PyImporter): - has_py_importer = True - else: - has_pyx_importer = True - - return has_py_importer, has_pyx_importer - - -def install(pyximport=True, pyimport=False, build_dir=None, build_in_temp=True, - setup_args=None, reload_support=False, - load_py_module_on_import_failure=False, inplace=False, - language_level=None): - """ Main entry point for pyxinstall. - - Call this to install the ``.pyx`` import hook in - your meta-path for a single Python process. If you want it to be - installed whenever you use Python, add it to your ``sitecustomize`` - (as described above). - - :param pyximport: If set to False, does not try to import ``.pyx`` files. - - :param pyimport: You can pass ``pyimport=True`` to also - install the ``.py`` import hook - in your meta-path. Note, however, that it is rather experimental, - will not work at all for some ``.py`` files and packages, and will - heavily slow down your imports due to search and compilation. - Use at your own risk. - - :param build_dir: By default, compiled modules will end up in a ``.pyxbld`` - directory in the user's home directory. Passing a different path - as ``build_dir`` will override this. - - :param build_in_temp: If ``False``, will produce the C files locally. Working - with complex dependencies and debugging becomes more easy. This - can principally interfere with existing files of the same name. - - :param setup_args: Dict of arguments for Distribution. - See ``distutils.core.setup()``. - - :param reload_support: Enables support for dynamic - ``reload(my_module)``, e.g. after a change in the Cython code. - Additional files ``<so_path>.reloadNN`` may arise on that account, when - the previously loaded module file cannot be overwritten. - - :param load_py_module_on_import_failure: If the compilation of a ``.py`` - file succeeds, but the subsequent import fails for some reason, - retry the import with the normal ``.py`` module instead of the - compiled module. Note that this may lead to unpredictable results - for modules that change the system state during their import, as - the second import will rerun these modifications in whatever state - the system was left after the import of the compiled module - failed. - - :param inplace: Install the compiled module - (``.so`` for Linux and Mac / ``.pyd`` for Windows) - next to the source file. - - :param language_level: The source language level to use: 2 or 3. - The default is to use the language level of the current Python - runtime for .py files and Py2 for ``.pyx`` files. - """ - if setup_args is None: - setup_args = {} - if not build_dir: - build_dir = os.path.join(os.path.expanduser('~'), '.pyxbld') - - global pyxargs - pyxargs = PyxArgs() #$pycheck_no - pyxargs.build_dir = build_dir - pyxargs.build_in_temp = build_in_temp - pyxargs.setup_args = (setup_args or {}).copy() - pyxargs.reload_support = reload_support - pyxargs.load_py_module_on_import_failure = load_py_module_on_import_failure - - has_py_importer, has_pyx_importer = _have_importers() - py_importer, pyx_importer = None, None - - if pyimport and not has_py_importer: - py_importer = PyImporter(pyxbuild_dir=build_dir, inplace=inplace, - language_level=language_level) - # make sure we import Cython before we install the import hook - import Cython.Compiler.Main, Cython.Compiler.Pipeline, Cython.Compiler.Optimize - sys.meta_path.insert(0, py_importer) - - if pyximport and not has_pyx_importer: - pyx_importer = PyxImporter(pyxbuild_dir=build_dir, inplace=inplace, - language_level=language_level) - sys.meta_path.append(pyx_importer) - - return py_importer, pyx_importer - - -def uninstall(py_importer, pyx_importer): - """ - Uninstall an import hook. - """ - try: - sys.meta_path.remove(py_importer) - except ValueError: - pass - - try: - sys.meta_path.remove(pyx_importer) - except ValueError: - pass - - -# MAIN - -def show_docs(): - import __main__ - __main__.__name__ = mod_name - for name in dir(__main__): - item = getattr(__main__, name) - try: - setattr(item, "__module__", mod_name) - except (AttributeError, TypeError): - pass - help(__main__) +if sys.version_info < (3, 5): + # _pyximport3 module requires at least Python 3.5 + from pyximport._pyximport2 import install, uninstall, show_docs +else: + from pyximport._pyximport3 import install, uninstall, show_docs if __name__ == '__main__': show_docs() |