summaryrefslogtreecommitdiff
path: root/pyximport
diff options
context:
space:
mode:
Diffstat (limited to 'pyximport')
-rw-r--r--pyximport/_pyximport2.py620
-rw-r--r--pyximport/_pyximport3.py478
-rw-r--r--pyximport/pyxbuild.py18
-rw-r--r--pyximport/pyximport.py603
-rw-r--r--pyximport/test/test_pyximport.py29
-rw-r--r--pyximport/test/test_reload.py10
6 files changed, 1135 insertions, 623 deletions
diff --git a/pyximport/_pyximport2.py b/pyximport/_pyximport2.py
new file mode 100644
index 000000000..00e88a8ac
--- /dev/null
+++ b/pyximport/_pyximport2.py
@@ -0,0 +1,620 @@
+"""
+Import hooks; when installed with the install() function, these hooks
+allow importing .pyx files as if they were Python modules.
+
+If you want the hook installed every time you run Python
+you can add it to your Python version by adding these lines to
+sitecustomize.py (which you can create from scratch in site-packages
+if it doesn't exist there or somewhere else on your python path)::
+
+ import pyximport
+ pyximport.install()
+
+For instance on the Mac with a non-system Python 2.3, you could create
+sitecustomize.py with only those two lines at
+/usr/local/lib/python2.3/site-packages/sitecustomize.py .
+
+A custom distutils.core.Extension instance and setup() args
+(Distribution) for for the build can be defined by a <modulename>.pyxbld
+file like:
+
+# examplemod.pyxbld
+def make_ext(modname, pyxfilename):
+ from distutils.extension import Extension
+ return Extension(name = modname,
+ sources=[pyxfilename, 'hello.c'],
+ include_dirs=['/myinclude'] )
+def make_setup_args():
+ return dict(script_args=["--compiler=mingw32"])
+
+Extra dependencies can be defined by a <modulename>.pyxdep .
+See README.
+
+Since Cython 0.11, the :mod:`pyximport` module also has experimental
+compilation support for normal Python modules. This allows you to
+automatically run Cython on every .pyx and .py module that Python
+imports, including parts of the standard library and installed
+packages. Cython will still fail to compile a lot of Python modules,
+in which case the import mechanism will fall back to loading the
+Python source modules instead. The .py import mechanism is installed
+like this::
+
+ pyximport.install(pyimport = True)
+
+Running this module as a top-level script will run a test and then print
+the documentation.
+
+This code is based on the Py2.3+ import protocol as described in PEP 302.
+"""
+
+import glob
+import imp
+import os
+import sys
+from zipimport import zipimporter, ZipImportError
+
+mod_name = "pyximport"
+
+PYX_EXT = ".pyx"
+PYXDEP_EXT = ".pyxdep"
+PYXBLD_EXT = ".pyxbld"
+
+DEBUG_IMPORT = False
+
+
+def _print(message, args):
+ if args:
+ message = message % args
+ print(message)
+
+
+def _debug(message, *args):
+ if DEBUG_IMPORT:
+ _print(message, args)
+
+
+def _info(message, *args):
+ _print(message, args)
+
+
+# Performance problem: for every PYX file that is imported, we will
+# invoke the whole distutils infrastructure even if the module is
+# already built. It might be more efficient to only do it when the
+# mod time of the .pyx is newer than the mod time of the .so but
+# the question is how to get distutils to tell me the name of the .so
+# before it builds it. Maybe it is easy...but maybe the performance
+# issue isn't real.
+def _load_pyrex(name, filename):
+ "Load a pyrex file given a name and filename."
+
+
+def get_distutils_extension(modname, pyxfilename, language_level=None):
+# try:
+# import hashlib
+# except ImportError:
+# import md5 as hashlib
+# extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()
+# modname = modname + extra
+ extension_mod,setup_args = handle_special_build(modname, pyxfilename)
+ if not extension_mod:
+ if not isinstance(pyxfilename, str):
+ # distutils is stupid in Py2 and requires exactly 'str'
+ # => encode accidentally coerced unicode strings back to str
+ pyxfilename = pyxfilename.encode(sys.getfilesystemencoding())
+ from distutils.extension import Extension
+ extension_mod = Extension(name = modname, sources=[pyxfilename])
+ if language_level is not None:
+ extension_mod.cython_directives = {'language_level': language_level}
+ return extension_mod,setup_args
+
+
+def handle_special_build(modname, pyxfilename):
+ special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT
+ ext = None
+ setup_args={}
+ if os.path.exists(special_build):
+ # globls = {}
+ # locs = {}
+ # execfile(special_build, globls, locs)
+ # ext = locs["make_ext"](modname, pyxfilename)
+ with open(special_build) as fid:
+ mod = imp.load_source("XXXX", special_build, fid)
+ make_ext = getattr(mod,'make_ext',None)
+ if make_ext:
+ ext = make_ext(modname, pyxfilename)
+ assert ext and ext.sources, "make_ext in %s did not return Extension" % special_build
+ make_setup_args = getattr(mod, 'make_setup_args',None)
+ if make_setup_args:
+ setup_args = make_setup_args()
+ assert isinstance(setup_args,dict), ("make_setup_args in %s did not return a dict"
+ % special_build)
+ assert set or setup_args, ("neither make_ext nor make_setup_args %s"
+ % special_build)
+ ext.sources = [os.path.join(os.path.dirname(special_build), source)
+ for source in ext.sources]
+ return ext, setup_args
+
+
+def handle_dependencies(pyxfilename):
+ testing = '_test_files' in globals()
+ dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT
+
+ # by default let distutils decide whether to rebuild on its own
+ # (it has a better idea of what the output file will be)
+
+ # but we know more about dependencies so force a rebuild if
+ # some of the dependencies are newer than the pyxfile.
+ if os.path.exists(dependfile):
+ with open(dependfile) as fid:
+ depends = fid.readlines()
+ depends = [depend.strip() for depend in depends]
+
+ # gather dependencies in the "files" variable
+ # the dependency file is itself a dependency
+ files = [dependfile]
+ for depend in depends:
+ fullpath = os.path.join(os.path.dirname(dependfile),
+ depend)
+ files.extend(glob.glob(fullpath))
+
+ # only for unit testing to see we did the right thing
+ if testing:
+ _test_files[:] = [] #$pycheck_no
+
+ # if any file that the pyxfile depends upon is newer than
+ # the pyx file, 'touch' the pyx file so that distutils will
+ # be tricked into rebuilding it.
+ for file in files:
+ from distutils.dep_util import newer
+ if newer(file, pyxfilename):
+ _debug("Rebuilding %s because of %s", pyxfilename, file)
+ filetime = os.path.getmtime(file)
+ os.utime(pyxfilename, (filetime, filetime))
+ if testing:
+ _test_files.append(file)
+
+
+def build_module(name, pyxfilename, pyxbuild_dir=None, inplace=False, language_level=None):
+ assert os.path.exists(pyxfilename), "Path does not exist: %s" % pyxfilename
+ handle_dependencies(pyxfilename)
+
+ extension_mod, setup_args = get_distutils_extension(name, pyxfilename, language_level)
+ build_in_temp = pyxargs.build_in_temp
+ sargs = pyxargs.setup_args.copy()
+ sargs.update(setup_args)
+ build_in_temp = sargs.pop('build_in_temp',build_in_temp)
+
+ from . import pyxbuild
+ olddir = os.getcwd()
+ common = ''
+ if pyxbuild_dir:
+ # Windows concantenates the pyxbuild_dir to the pyxfilename when
+ # compiling, and then complains that the filename is too long
+ common = os.path.commonprefix([pyxbuild_dir, pyxfilename])
+ if len(common) > 30:
+ pyxfilename = os.path.relpath(pyxfilename)
+ pyxbuild_dir = os.path.relpath(pyxbuild_dir)
+ os.chdir(common)
+ try:
+ so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod,
+ build_in_temp=build_in_temp,
+ pyxbuild_dir=pyxbuild_dir,
+ setup_args=sargs,
+ inplace=inplace,
+ reload_support=pyxargs.reload_support)
+ finally:
+ os.chdir(olddir)
+ so_path = os.path.join(common, so_path)
+ assert os.path.exists(so_path), "Cannot find: %s" % so_path
+
+ junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ? yes, indeed, trying to eat my files ;)
+ junkstuff = glob.glob(junkpath)
+ for path in junkstuff:
+ if path != so_path:
+ try:
+ os.remove(path)
+ except IOError:
+ _info("Couldn't remove %s", path)
+
+ return so_path
+
+
+def load_module(name, pyxfilename, pyxbuild_dir=None, is_package=False,
+ build_inplace=False, language_level=None, so_path=None):
+ try:
+ if so_path is None:
+ if is_package:
+ module_name = name + '.__init__'
+ else:
+ module_name = name
+ so_path = build_module(module_name, pyxfilename, pyxbuild_dir,
+ inplace=build_inplace, language_level=language_level)
+ mod = imp.load_dynamic(name, so_path)
+ if is_package and not hasattr(mod, '__path__'):
+ mod.__path__ = [os.path.dirname(so_path)]
+ assert mod.__file__ == so_path, (mod.__file__, so_path)
+ except Exception as failure_exc:
+ _debug("Failed to load extension module: %r" % failure_exc)
+ if pyxargs.load_py_module_on_import_failure and pyxfilename.endswith('.py'):
+ # try to fall back to normal import
+ mod = imp.load_source(name, pyxfilename)
+ assert mod.__file__ in (pyxfilename, pyxfilename+'c', pyxfilename+'o'), (mod.__file__, pyxfilename)
+ else:
+ tb = sys.exc_info()[2]
+ import traceback
+ exc = ImportError("Building module %s failed: %s" % (
+ name, traceback.format_exception_only(*sys.exc_info()[:2])))
+ if sys.version_info[0] >= 3:
+ raise exc.with_traceback(tb)
+ else:
+ exec("raise exc, None, tb", {'exc': exc, 'tb': tb})
+ return mod
+
+
+# import hooks
+
+class PyxImporter(object):
+ """A meta-path importer for .pyx files.
+ """
+ def __init__(self, extension=PYX_EXT, pyxbuild_dir=None, inplace=False,
+ language_level=None):
+ self.extension = extension
+ self.pyxbuild_dir = pyxbuild_dir
+ self.inplace = inplace
+ self.language_level = language_level
+
+ def find_module(self, fullname, package_path=None):
+ if fullname in sys.modules and not pyxargs.reload_support:
+ return None # only here when reload()
+
+ # package_path might be a _NamespacePath. Convert that into a list...
+ if package_path is not None and not isinstance(package_path, list):
+ package_path = list(package_path)
+ try:
+ fp, pathname, (ext,mode,ty) = imp.find_module(fullname,package_path)
+ if fp: fp.close() # Python should offer a Default-Loader to avoid this double find/open!
+ if pathname and ty == imp.PKG_DIRECTORY:
+ pkg_file = os.path.join(pathname, '__init__'+self.extension)
+ if os.path.isfile(pkg_file):
+ return PyxLoader(fullname, pathname,
+ init_path=pkg_file,
+ pyxbuild_dir=self.pyxbuild_dir,
+ inplace=self.inplace,
+ language_level=self.language_level)
+ if pathname and pathname.endswith(self.extension):
+ return PyxLoader(fullname, pathname,
+ pyxbuild_dir=self.pyxbuild_dir,
+ inplace=self.inplace,
+ language_level=self.language_level)
+ if ty != imp.C_EXTENSION: # only when an extension, check if we have a .pyx next!
+ return None
+
+ # find .pyx fast, when .so/.pyd exist --inplace
+ pyxpath = os.path.splitext(pathname)[0]+self.extension
+ if os.path.isfile(pyxpath):
+ return PyxLoader(fullname, pyxpath,
+ pyxbuild_dir=self.pyxbuild_dir,
+ inplace=self.inplace,
+ language_level=self.language_level)
+
+ # .so/.pyd's on PATH should not be remote from .pyx's
+ # think no need to implement PyxArgs.importer_search_remote here?
+
+ except ImportError:
+ pass
+
+ # searching sys.path ...
+
+ #if DEBUG_IMPORT: print "SEARCHING", fullname, package_path
+
+ mod_parts = fullname.split('.')
+ module_name = mod_parts[-1]
+ pyx_module_name = module_name + self.extension
+
+ # this may work, but it returns the file content, not its path
+ #import pkgutil
+ #pyx_source = pkgutil.get_data(package, pyx_module_name)
+
+ paths = package_path or sys.path
+ for path in paths:
+ pyx_data = None
+ if not path:
+ path = os.getcwd()
+ elif os.path.isfile(path):
+ try:
+ zi = zipimporter(path)
+ pyx_data = zi.get_data(pyx_module_name)
+ except (ZipImportError, IOError, OSError):
+ continue # Module not found.
+ # unzip the imported file into the build dir
+ # FIXME: can interfere with later imports if build dir is in sys.path and comes before zip file
+ path = self.pyxbuild_dir
+ elif not os.path.isabs(path):
+ path = os.path.abspath(path)
+
+ pyx_module_path = os.path.join(path, pyx_module_name)
+ if pyx_data is not None:
+ if not os.path.exists(path):
+ try:
+ os.makedirs(path)
+ except OSError:
+ # concurrency issue?
+ if not os.path.exists(path):
+ raise
+ with open(pyx_module_path, "wb") as f:
+ f.write(pyx_data)
+ elif not os.path.isfile(pyx_module_path):
+ continue # Module not found.
+
+ return PyxLoader(fullname, pyx_module_path,
+ pyxbuild_dir=self.pyxbuild_dir,
+ inplace=self.inplace,
+ language_level=self.language_level)
+
+ # not found, normal package, not a .pyx file, none of our business
+ _debug("%s not found" % fullname)
+ return None
+
+
+class PyImporter(PyxImporter):
+ """A meta-path importer for normal .py files.
+ """
+ def __init__(self, pyxbuild_dir=None, inplace=False, language_level=None):
+ if language_level is None:
+ language_level = sys.version_info[0]
+ self.super = super(PyImporter, self)
+ self.super.__init__(extension='.py', pyxbuild_dir=pyxbuild_dir, inplace=inplace,
+ language_level=language_level)
+ self.uncompilable_modules = {}
+ self.blocked_modules = ['Cython', 'pyxbuild', 'pyximport.pyxbuild',
+ 'distutils']
+ self.blocked_packages = ['Cython.', 'distutils.']
+
+ def find_module(self, fullname, package_path=None):
+ if fullname in sys.modules:
+ return None
+ if any([fullname.startswith(pkg) for pkg in self.blocked_packages]):
+ return None
+ if fullname in self.blocked_modules:
+ # prevent infinite recursion
+ return None
+ if _lib_loader.knows(fullname):
+ return _lib_loader
+ _debug("trying import of module '%s'", fullname)
+ if fullname in self.uncompilable_modules:
+ path, last_modified = self.uncompilable_modules[fullname]
+ try:
+ new_last_modified = os.stat(path).st_mtime
+ if new_last_modified > last_modified:
+ # import would fail again
+ return None
+ except OSError:
+ # module is no longer where we found it, retry the import
+ pass
+
+ self.blocked_modules.append(fullname)
+ try:
+ importer = self.super.find_module(fullname, package_path)
+ if importer is not None:
+ if importer.init_path:
+ path = importer.init_path
+ real_name = fullname + '.__init__'
+ else:
+ path = importer.path
+ real_name = fullname
+ _debug("importer found path %s for module %s", path, real_name)
+ try:
+ so_path = build_module(
+ real_name, path,
+ pyxbuild_dir=self.pyxbuild_dir,
+ language_level=self.language_level,
+ inplace=self.inplace)
+ _lib_loader.add_lib(fullname, path, so_path,
+ is_package=bool(importer.init_path))
+ return _lib_loader
+ except Exception:
+ if DEBUG_IMPORT:
+ import traceback
+ traceback.print_exc()
+ # build failed, not a compilable Python module
+ try:
+ last_modified = os.stat(path).st_mtime
+ except OSError:
+ last_modified = 0
+ self.uncompilable_modules[fullname] = (path, last_modified)
+ importer = None
+ finally:
+ self.blocked_modules.pop()
+ return importer
+
+
+class LibLoader(object):
+ def __init__(self):
+ self._libs = {}
+
+ def load_module(self, fullname):
+ try:
+ source_path, so_path, is_package = self._libs[fullname]
+ except KeyError:
+ raise ValueError("invalid module %s" % fullname)
+ _debug("Loading shared library module '%s' from %s", fullname, so_path)
+ return load_module(fullname, source_path, so_path=so_path, is_package=is_package)
+
+ def add_lib(self, fullname, path, so_path, is_package):
+ self._libs[fullname] = (path, so_path, is_package)
+
+ def knows(self, fullname):
+ return fullname in self._libs
+
+_lib_loader = LibLoader()
+
+
+class PyxLoader(object):
+ def __init__(self, fullname, path, init_path=None, pyxbuild_dir=None,
+ inplace=False, language_level=None):
+ _debug("PyxLoader created for loading %s from %s (init path: %s)",
+ fullname, path, init_path)
+ self.fullname = fullname
+ self.path, self.init_path = path, init_path
+ self.pyxbuild_dir = pyxbuild_dir
+ self.inplace = inplace
+ self.language_level = language_level
+
+ def load_module(self, fullname):
+ assert self.fullname == fullname, (
+ "invalid module, expected %s, got %s" % (
+ self.fullname, fullname))
+ if self.init_path:
+ # package
+ #print "PACKAGE", fullname
+ module = load_module(fullname, self.init_path,
+ self.pyxbuild_dir, is_package=True,
+ build_inplace=self.inplace,
+ language_level=self.language_level)
+ module.__path__ = [self.path]
+ else:
+ #print "MODULE", fullname
+ module = load_module(fullname, self.path,
+ self.pyxbuild_dir,
+ build_inplace=self.inplace,
+ language_level=self.language_level)
+ return module
+
+
+#install args
+class PyxArgs(object):
+ build_dir=True
+ build_in_temp=True
+ setup_args={} #None
+
+##pyxargs=None
+
+
+def _have_importers():
+ has_py_importer = False
+ has_pyx_importer = False
+ for importer in sys.meta_path:
+ if isinstance(importer, PyxImporter):
+ if isinstance(importer, PyImporter):
+ has_py_importer = True
+ else:
+ has_pyx_importer = True
+
+ return has_py_importer, has_pyx_importer
+
+
+def install(pyximport=True, pyimport=False, build_dir=None, build_in_temp=True,
+ setup_args=None, reload_support=False,
+ load_py_module_on_import_failure=False, inplace=False,
+ language_level=None):
+ """ Main entry point for pyxinstall.
+
+ Call this to install the ``.pyx`` import hook in
+ your meta-path for a single Python process. If you want it to be
+ installed whenever you use Python, add it to your ``sitecustomize``
+ (as described above).
+
+ :param pyximport: If set to False, does not try to import ``.pyx`` files.
+
+ :param pyimport: You can pass ``pyimport=True`` to also
+ install the ``.py`` import hook
+ in your meta-path. Note, however, that it is rather experimental,
+ will not work at all for some ``.py`` files and packages, and will
+ heavily slow down your imports due to search and compilation.
+ Use at your own risk.
+
+ :param build_dir: By default, compiled modules will end up in a ``.pyxbld``
+ directory in the user's home directory. Passing a different path
+ as ``build_dir`` will override this.
+
+ :param build_in_temp: If ``False``, will produce the C files locally. Working
+ with complex dependencies and debugging becomes more easy. This
+ can principally interfere with existing files of the same name.
+
+ :param setup_args: Dict of arguments for Distribution.
+ See ``distutils.core.setup()``.
+
+ :param reload_support: Enables support for dynamic
+ ``reload(my_module)``, e.g. after a change in the Cython code.
+ Additional files ``<so_path>.reloadNN`` may arise on that account, when
+ the previously loaded module file cannot be overwritten.
+
+ :param load_py_module_on_import_failure: If the compilation of a ``.py``
+ file succeeds, but the subsequent import fails for some reason,
+ retry the import with the normal ``.py`` module instead of the
+ compiled module. Note that this may lead to unpredictable results
+ for modules that change the system state during their import, as
+ the second import will rerun these modifications in whatever state
+ the system was left after the import of the compiled module
+ failed.
+
+ :param inplace: Install the compiled module
+ (``.so`` for Linux and Mac / ``.pyd`` for Windows)
+ next to the source file.
+
+ :param language_level: The source language level to use: 2 or 3.
+ The default is to use the language level of the current Python
+ runtime for .py files and Py2 for ``.pyx`` files.
+ """
+ if setup_args is None:
+ setup_args = {}
+ if not build_dir:
+ build_dir = os.path.join(os.path.expanduser('~'), '.pyxbld')
+
+ global pyxargs
+ pyxargs = PyxArgs() #$pycheck_no
+ pyxargs.build_dir = build_dir
+ pyxargs.build_in_temp = build_in_temp
+ pyxargs.setup_args = (setup_args or {}).copy()
+ pyxargs.reload_support = reload_support
+ pyxargs.load_py_module_on_import_failure = load_py_module_on_import_failure
+
+ has_py_importer, has_pyx_importer = _have_importers()
+ py_importer, pyx_importer = None, None
+
+ if pyimport and not has_py_importer:
+ py_importer = PyImporter(pyxbuild_dir=build_dir, inplace=inplace,
+ language_level=language_level)
+ # make sure we import Cython before we install the import hook
+ import Cython.Compiler.Main, Cython.Compiler.Pipeline, Cython.Compiler.Optimize
+ sys.meta_path.insert(0, py_importer)
+
+ if pyximport and not has_pyx_importer:
+ pyx_importer = PyxImporter(pyxbuild_dir=build_dir, inplace=inplace,
+ language_level=language_level)
+ sys.meta_path.append(pyx_importer)
+
+ return py_importer, pyx_importer
+
+
+def uninstall(py_importer, pyx_importer):
+ """
+ Uninstall an import hook.
+ """
+ try:
+ sys.meta_path.remove(py_importer)
+ except ValueError:
+ pass
+
+ try:
+ sys.meta_path.remove(pyx_importer)
+ except ValueError:
+ pass
+
+
+# MAIN
+
+def show_docs():
+ import __main__
+ __main__.__name__ = mod_name
+ for name in dir(__main__):
+ item = getattr(__main__, name)
+ try:
+ setattr(item, "__module__", mod_name)
+ except (AttributeError, TypeError):
+ pass
+ help(__main__)
+
+
+if __name__ == '__main__':
+ show_docs()
diff --git a/pyximport/_pyximport3.py b/pyximport/_pyximport3.py
new file mode 100644
index 000000000..4fa811f8a
--- /dev/null
+++ b/pyximport/_pyximport3.py
@@ -0,0 +1,478 @@
+"""
+Import hooks; when installed with the install() function, these hooks
+allow importing .pyx files as if they were Python modules.
+
+If you want the hook installed every time you run Python
+you can add it to your Python version by adding these lines to
+sitecustomize.py (which you can create from scratch in site-packages
+if it doesn't exist there or somewhere else on your python path)::
+
+ import pyximport
+ pyximport.install()
+
+For instance on the Mac with a non-system Python 2.3, you could create
+sitecustomize.py with only those two lines at
+/usr/local/lib/python2.3/site-packages/sitecustomize.py .
+
+A custom distutils.core.Extension instance and setup() args
+(Distribution) for for the build can be defined by a <modulename>.pyxbld
+file like:
+
+# examplemod.pyxbld
+def make_ext(modname, pyxfilename):
+ from distutils.extension import Extension
+ return Extension(name = modname,
+ sources=[pyxfilename, 'hello.c'],
+ include_dirs=['/myinclude'] )
+def make_setup_args():
+ return dict(script_args=["--compiler=mingw32"])
+
+Extra dependencies can be defined by a <modulename>.pyxdep .
+See README.
+
+Since Cython 0.11, the :mod:`pyximport` module also has experimental
+compilation support for normal Python modules. This allows you to
+automatically run Cython on every .pyx and .py module that Python
+imports, including parts of the standard library and installed
+packages. Cython will still fail to compile a lot of Python modules,
+in which case the import mechanism will fall back to loading the
+Python source modules instead. The .py import mechanism is installed
+like this::
+
+ pyximport.install(pyimport = True)
+
+Running this module as a top-level script will run a test and then print
+the documentation.
+"""
+
+import glob
+import importlib
+import os
+import sys
+from importlib.abc import MetaPathFinder
+from importlib.machinery import ExtensionFileLoader, SourceFileLoader
+from importlib.util import spec_from_file_location
+
+mod_name = "pyximport"
+
+PY_EXT = ".py"
+PYX_EXT = ".pyx"
+PYXDEP_EXT = ".pyxdep"
+PYXBLD_EXT = ".pyxbld"
+
+DEBUG_IMPORT = False
+
+
+def _print(message, args):
+ if args:
+ message = message % args
+ print(message)
+
+
+def _debug(message, *args):
+ if DEBUG_IMPORT:
+ _print(message, args)
+
+
+def _info(message, *args):
+ _print(message, args)
+
+
+def load_source(file_path):
+ import importlib.util
+ from importlib.machinery import SourceFileLoader
+ spec = importlib.util.spec_from_file_location("XXXX", file_path, loader=SourceFileLoader("XXXX", file_path))
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def get_distutils_extension(modname, pyxfilename, language_level=None):
+# try:
+# import hashlib
+# except ImportError:
+# import md5 as hashlib
+# extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()
+# modname = modname + extra
+ extension_mod,setup_args = handle_special_build(modname, pyxfilename)
+ if not extension_mod:
+ if not isinstance(pyxfilename, str):
+ # distutils is stupid in Py2 and requires exactly 'str'
+ # => encode accidentally coerced unicode strings back to str
+ pyxfilename = pyxfilename.encode(sys.getfilesystemencoding())
+ from distutils.extension import Extension
+ extension_mod = Extension(name = modname, sources=[pyxfilename])
+ if language_level is not None:
+ extension_mod.cython_directives = {'language_level': language_level}
+ return extension_mod,setup_args
+
+
+def handle_special_build(modname, pyxfilename):
+ special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT
+ ext = None
+ setup_args={}
+ if os.path.exists(special_build):
+ # globls = {}
+ # locs = {}
+ # execfile(special_build, globls, locs)
+ # ext = locs["make_ext"](modname, pyxfilename)
+ mod = load_source(special_build)
+ make_ext = getattr(mod,'make_ext',None)
+ if make_ext:
+ ext = make_ext(modname, pyxfilename)
+ assert ext and ext.sources, "make_ext in %s did not return Extension" % special_build
+ make_setup_args = getattr(mod, 'make_setup_args',None)
+ if make_setup_args:
+ setup_args = make_setup_args()
+ assert isinstance(setup_args,dict), ("make_setup_args in %s did not return a dict"
+ % special_build)
+ assert set or setup_args, ("neither make_ext nor make_setup_args %s"
+ % special_build)
+ ext.sources = [os.path.join(os.path.dirname(special_build), source)
+ for source in ext.sources]
+ return ext, setup_args
+
+
+def handle_dependencies(pyxfilename):
+ testing = '_test_files' in globals()
+ dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT
+
+ # by default let distutils decide whether to rebuild on its own
+ # (it has a better idea of what the output file will be)
+
+ # but we know more about dependencies so force a rebuild if
+ # some of the dependencies are newer than the pyxfile.
+ if os.path.exists(dependfile):
+ with open(dependfile) as fid:
+ depends = fid.readlines()
+ depends = [depend.strip() for depend in depends]
+
+ # gather dependencies in the "files" variable
+ # the dependency file is itself a dependency
+ files = [dependfile]
+ for depend in depends:
+ fullpath = os.path.join(os.path.dirname(dependfile),
+ depend)
+ files.extend(glob.glob(fullpath))
+
+ # only for unit testing to see we did the right thing
+ if testing:
+ _test_files[:] = [] #$pycheck_no
+
+ # if any file that the pyxfile depends upon is newer than
+ # the pyx file, 'touch' the pyx file so that distutils will
+ # be tricked into rebuilding it.
+ for file in files:
+ from distutils.dep_util import newer
+ if newer(file, pyxfilename):
+ _debug("Rebuilding %s because of %s", pyxfilename, file)
+ filetime = os.path.getmtime(file)
+ os.utime(pyxfilename, (filetime, filetime))
+ if testing:
+ _test_files.append(file)
+
+
+def build_module(name, pyxfilename, pyxbuild_dir=None, inplace=False, language_level=None):
+ assert os.path.exists(pyxfilename), "Path does not exist: %s" % pyxfilename
+ handle_dependencies(pyxfilename)
+
+ extension_mod, setup_args = get_distutils_extension(name, pyxfilename, language_level)
+ build_in_temp = pyxargs.build_in_temp
+ sargs = pyxargs.setup_args.copy()
+ sargs.update(setup_args)
+ build_in_temp = sargs.pop('build_in_temp',build_in_temp)
+
+ from . import pyxbuild
+ olddir = os.getcwd()
+ common = ''
+ if pyxbuild_dir:
+ # Windows concantenates the pyxbuild_dir to the pyxfilename when
+ # compiling, and then complains that the filename is too long
+ common = os.path.commonprefix([pyxbuild_dir, pyxfilename])
+ if len(common) > 30:
+ pyxfilename = os.path.relpath(pyxfilename)
+ pyxbuild_dir = os.path.relpath(pyxbuild_dir)
+ os.chdir(common)
+ try:
+ so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod,
+ build_in_temp=build_in_temp,
+ pyxbuild_dir=pyxbuild_dir,
+ setup_args=sargs,
+ inplace=inplace,
+ reload_support=pyxargs.reload_support)
+ finally:
+ os.chdir(olddir)
+ so_path = os.path.join(common, so_path)
+ assert os.path.exists(so_path), "Cannot find: %s" % so_path
+
+ junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ? yes, indeed, trying to eat my files ;)
+ junkstuff = glob.glob(junkpath)
+ for path in junkstuff:
+ if path != so_path:
+ try:
+ os.remove(path)
+ except IOError:
+ _info("Couldn't remove %s", path)
+
+ return so_path
+
+
+# import hooks
+
+class PyxImportMetaFinder(MetaPathFinder):
+
+ def __init__(self, extension=PYX_EXT, pyxbuild_dir=None, inplace=False, language_level=None):
+ self.pyxbuild_dir = pyxbuild_dir
+ self.inplace = inplace
+ self.language_level = language_level
+ self.extension = extension
+
+ def find_spec(self, fullname, path, target=None):
+ if not path:
+ path = [os.getcwd()] # top level import --
+ if "." in fullname:
+ *parents, name = fullname.split(".")
+ else:
+ name = fullname
+ for entry in path:
+ if os.path.isdir(os.path.join(entry, name)):
+ # this module has child modules
+ filename = os.path.join(entry, name, "__init__" + self.extension)
+ submodule_locations = [os.path.join(entry, name)]
+ else:
+ filename = os.path.join(entry, name + self.extension)
+ submodule_locations = None
+ if not os.path.exists(filename):
+ continue
+
+ return spec_from_file_location(
+ fullname, filename,
+ loader=PyxImportLoader(filename, self.pyxbuild_dir, self.inplace, self.language_level),
+ submodule_search_locations=submodule_locations)
+
+ return None # we don't know how to import this
+
+
+class PyImportMetaFinder(MetaPathFinder):
+
+ def __init__(self, extension=PY_EXT, pyxbuild_dir=None, inplace=False, language_level=None):
+ self.pyxbuild_dir = pyxbuild_dir
+ self.inplace = inplace
+ self.language_level = language_level
+ self.extension = extension
+ self.uncompilable_modules = {}
+ self.blocked_modules = ['Cython', 'pyxbuild', 'pyximport.pyxbuild',
+ 'distutils', 'cython']
+ self.blocked_packages = ['Cython.', 'distutils.']
+
+ def find_spec(self, fullname, path, target=None):
+ if fullname in sys.modules:
+ return None
+ if any([fullname.startswith(pkg) for pkg in self.blocked_packages]):
+ return None
+ if fullname in self.blocked_modules:
+ # prevent infinite recursion
+ return None
+
+ self.blocked_modules.append(fullname)
+ name = fullname
+ if not path:
+ path = [os.getcwd()] # top level import --
+ try:
+ for entry in path:
+ if os.path.isdir(os.path.join(entry, name)):
+ # this module has child modules
+ filename = os.path.join(entry, name, "__init__" + self.extension)
+ submodule_locations = [os.path.join(entry, name)]
+ else:
+ filename = os.path.join(entry, name + self.extension)
+ submodule_locations = None
+ if not os.path.exists(filename):
+ continue
+
+ return spec_from_file_location(
+ fullname, filename,
+ loader=PyxImportLoader(filename, self.pyxbuild_dir, self.inplace, self.language_level),
+ submodule_search_locations=submodule_locations)
+ finally:
+ self.blocked_modules.pop()
+
+ return None # we don't know how to import this
+
+
+class PyxImportLoader(ExtensionFileLoader):
+
+ def __init__(self, filename, pyxbuild_dir, inplace, language_level):
+ module_name = os.path.splitext(os.path.basename(filename))[0]
+ super().__init__(module_name, filename)
+ self._pyxbuild_dir = pyxbuild_dir
+ self._inplace = inplace
+ self._language_level = language_level
+
+ def create_module(self, spec):
+ try:
+ so_path = build_module(spec.name, pyxfilename=spec.origin, pyxbuild_dir=self._pyxbuild_dir,
+ inplace=self._inplace, language_level=self._language_level)
+ self.path = so_path
+ spec.origin = so_path
+ return super().create_module(spec)
+ except Exception as failure_exc:
+ _debug("Failed to load extension module: %r" % failure_exc)
+ if pyxargs.load_py_module_on_import_failure and spec.origin.endswith(PY_EXT):
+ spec = importlib.util.spec_from_file_location(spec.name, spec.origin,
+ loader=SourceFileLoader(spec.name, spec.origin))
+ mod = importlib.util.module_from_spec(spec)
+ assert mod.__file__ in (spec.origin, spec.origin + 'c', spec.origin + 'o'), (mod.__file__, spec.origin)
+ return mod
+ else:
+ tb = sys.exc_info()[2]
+ import traceback
+ exc = ImportError("Building module %s failed: %s" % (
+ spec.name, traceback.format_exception_only(*sys.exc_info()[:2])))
+ raise exc.with_traceback(tb)
+
+ def exec_module(self, module):
+ try:
+ return super().exec_module(module)
+ except Exception as failure_exc:
+ import traceback
+ _debug("Failed to load extension module: %r" % failure_exc)
+ raise ImportError("Executing module %s failed %s" % (
+ module.__file__, traceback.format_exception_only(*sys.exc_info()[:2])))
+
+
+#install args
+class PyxArgs(object):
+ build_dir=True
+ build_in_temp=True
+ setup_args={} #None
+
+
+def _have_importers():
+ has_py_importer = False
+ has_pyx_importer = False
+ for importer in sys.meta_path:
+ if isinstance(importer, PyxImportMetaFinder):
+ if isinstance(importer, PyImportMetaFinder):
+ has_py_importer = True
+ else:
+ has_pyx_importer = True
+
+ return has_py_importer, has_pyx_importer
+
+
+def install(pyximport=True, pyimport=False, build_dir=None, build_in_temp=True,
+ setup_args=None, reload_support=False,
+ load_py_module_on_import_failure=False, inplace=False,
+ language_level=None):
+ """ Main entry point for pyxinstall.
+
+ Call this to install the ``.pyx`` import hook in
+ your meta-path for a single Python process. If you want it to be
+ installed whenever you use Python, add it to your ``sitecustomize``
+ (as described above).
+
+ :param pyximport: If set to False, does not try to import ``.pyx`` files.
+
+ :param pyimport: You can pass ``pyimport=True`` to also
+ install the ``.py`` import hook
+ in your meta-path. Note, however, that it is rather experimental,
+ will not work at all for some ``.py`` files and packages, and will
+ heavily slow down your imports due to search and compilation.
+ Use at your own risk.
+
+ :param build_dir: By default, compiled modules will end up in a ``.pyxbld``
+ directory in the user's home directory. Passing a different path
+ as ``build_dir`` will override this.
+
+ :param build_in_temp: If ``False``, will produce the C files locally. Working
+ with complex dependencies and debugging becomes more easy. This
+ can principally interfere with existing files of the same name.
+
+ :param setup_args: Dict of arguments for Distribution.
+ See ``distutils.core.setup()``.
+
+ :param reload_support: Enables support for dynamic
+ ``reload(my_module)``, e.g. after a change in the Cython code.
+ Additional files ``<so_path>.reloadNN`` may arise on that account, when
+ the previously loaded module file cannot be overwritten.
+
+ :param load_py_module_on_import_failure: If the compilation of a ``.py``
+ file succeeds, but the subsequent import fails for some reason,
+ retry the import with the normal ``.py`` module instead of the
+ compiled module. Note that this may lead to unpredictable results
+ for modules that change the system state during their import, as
+ the second import will rerun these modifications in whatever state
+ the system was left after the import of the compiled module
+ failed.
+
+ :param inplace: Install the compiled module
+ (``.so`` for Linux and Mac / ``.pyd`` for Windows)
+ next to the source file.
+
+ :param language_level: The source language level to use: 2 or 3.
+ The default is to use the language level of the current Python
+ runtime for .py files and Py2 for ``.pyx`` files.
+ """
+ if setup_args is None:
+ setup_args = {}
+ if not build_dir:
+ build_dir = os.path.join(os.path.expanduser('~'), '.pyxbld')
+
+ global pyxargs
+ pyxargs = PyxArgs() #$pycheck_no
+ pyxargs.build_dir = build_dir
+ pyxargs.build_in_temp = build_in_temp
+ pyxargs.setup_args = (setup_args or {}).copy()
+ pyxargs.reload_support = reload_support
+ pyxargs.load_py_module_on_import_failure = load_py_module_on_import_failure
+
+ has_py_importer, has_pyx_importer = _have_importers()
+ py_importer, pyx_importer = None, None
+
+ if pyimport and not has_py_importer:
+ py_importer = PyImportMetaFinder(pyxbuild_dir=build_dir, inplace=inplace,
+ language_level=language_level)
+ # make sure we import Cython before we install the import hook
+ import Cython.Compiler.Main, Cython.Compiler.Pipeline, Cython.Compiler.Optimize
+ sys.meta_path.insert(0, py_importer)
+
+ if pyximport and not has_pyx_importer:
+ pyx_importer = PyxImportMetaFinder(pyxbuild_dir=build_dir, inplace=inplace,
+ language_level=language_level)
+ sys.meta_path.append(pyx_importer)
+
+ return py_importer, pyx_importer
+
+
+def uninstall(py_importer, pyx_importer):
+ """
+ Uninstall an import hook.
+ """
+ try:
+ sys.meta_path.remove(py_importer)
+ except ValueError:
+ pass
+
+ try:
+ sys.meta_path.remove(pyx_importer)
+ except ValueError:
+ pass
+
+
+# MAIN
+
+def show_docs():
+ import __main__
+ __main__.__name__ = mod_name
+ for name in dir(__main__):
+ item = getattr(__main__, name)
+ try:
+ setattr(item, "__module__", mod_name)
+ except (AttributeError, TypeError):
+ pass
+ help(__main__)
+
+
+if __name__ == '__main__':
+ show_docs()
diff --git a/pyximport/pyxbuild.py b/pyximport/pyxbuild.py
index de4a2241f..61f9747ee 100644
--- a/pyximport/pyxbuild.py
+++ b/pyximport/pyxbuild.py
@@ -10,7 +10,7 @@ from distutils.errors import DistutilsArgError, DistutilsError, CCompilerError
from distutils.extension import Extension
from distutils.util import grok_environment_error
try:
- from Cython.Distutils.old_build_ext import old_build_ext as build_ext
+ from Cython.Distutils.build_ext import build_ext
HAS_CYTHON = True
except ImportError:
HAS_CYTHON = False
@@ -53,7 +53,10 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil
quiet = "--verbose"
else:
quiet = "--quiet"
- args = [quiet, "build_ext"]
+ if build_in_temp:
+ args = [quiet, "build_ext", '--cython-c-in-temp']
+ else:
+ args = [quiet, "build_ext"]
if force_rebuild:
args.append("--force")
if inplace and package_base_dir:
@@ -65,8 +68,6 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil
elif 'set_initial_path' not in ext.cython_directives:
ext.cython_directives['set_initial_path'] = 'SOURCEFILE'
- if HAS_CYTHON and build_in_temp:
- args.append("--pyrex-c-in-temp")
sargs = setup_args.copy()
sargs.update({
"script_name": None,
@@ -103,7 +104,7 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil
so_path = obj_build_ext.get_outputs()[0]
if obj_build_ext.inplace:
# Python distutils get_outputs()[ returns a wrong so_path
- # when --inplace ; see http://bugs.python.org/issue5977
+ # when --inplace ; see https://bugs.python.org/issue5977
# workaround:
so_path = os.path.join(os.path.dirname(filename),
os.path.basename(so_path))
@@ -119,9 +120,9 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil
while count < 100:
count += 1
r_path = os.path.join(obj_build_ext.build_lib,
- basename + '.reload%s'%count)
+ basename + '.reload%s' % count)
try:
- import shutil # late import / reload_support is: debugging
+ import shutil # late import / reload_support is: debugging
try:
# Try to unlink first --- if the .so file
# is mmapped by another process,
@@ -140,7 +141,7 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil
break
else:
# used up all 100 slots
- raise ImportError("reload count for %s reached maximum"%org_path)
+ raise ImportError("reload count for %s reached maximum" % org_path)
_reloads[org_path]=(timestamp, so_path, count)
return so_path
except KeyboardInterrupt:
@@ -157,4 +158,3 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil
if __name__=="__main__":
pyx_to_dll("dummy.pyx")
from . import test
-
diff --git a/pyximport/pyximport.py b/pyximport/pyximport.py
index 5628f301b..9d575815a 100644
--- a/pyximport/pyximport.py
+++ b/pyximport/pyximport.py
@@ -1,602 +1,11 @@
-"""
-Import hooks; when installed with the install() function, these hooks
-allow importing .pyx files as if they were Python modules.
-
-If you want the hook installed every time you run Python
-you can add it to your Python version by adding these lines to
-sitecustomize.py (which you can create from scratch in site-packages
-if it doesn't exist there or somewhere else on your python path)::
-
- import pyximport
- pyximport.install()
-
-For instance on the Mac with a non-system Python 2.3, you could create
-sitecustomize.py with only those two lines at
-/usr/local/lib/python2.3/site-packages/sitecustomize.py .
-
-A custom distutils.core.Extension instance and setup() args
-(Distribution) for for the build can be defined by a <modulename>.pyxbld
-file like:
-
-# examplemod.pyxbld
-def make_ext(modname, pyxfilename):
- from distutils.extension import Extension
- return Extension(name = modname,
- sources=[pyxfilename, 'hello.c'],
- include_dirs=['/myinclude'] )
-def make_setup_args():
- return dict(script_args=["--compiler=mingw32"])
-
-Extra dependencies can be defined by a <modulename>.pyxdep .
-See README.
-
-Since Cython 0.11, the :mod:`pyximport` module also has experimental
-compilation support for normal Python modules. This allows you to
-automatically run Cython on every .pyx and .py module that Python
-imports, including parts of the standard library and installed
-packages. Cython will still fail to compile a lot of Python modules,
-in which case the import mechanism will fall back to loading the
-Python source modules instead. The .py import mechanism is installed
-like this::
-
- pyximport.install(pyimport = True)
-
-Running this module as a top-level script will run a test and then print
-the documentation.
-
-This code is based on the Py2.3+ import protocol as described in PEP 302.
-"""
-
-import glob
-import imp
-import os
+from __future__ import absolute_import
import sys
-from zipimport import zipimporter, ZipImportError
-
-mod_name = "pyximport"
-
-PYX_EXT = ".pyx"
-PYXDEP_EXT = ".pyxdep"
-PYXBLD_EXT = ".pyxbld"
-
-DEBUG_IMPORT = False
-
-
-def _print(message, args):
- if args:
- message = message % args
- print(message)
-
-
-def _debug(message, *args):
- if DEBUG_IMPORT:
- _print(message, args)
-
-
-def _info(message, *args):
- _print(message, args)
-
-
-# Performance problem: for every PYX file that is imported, we will
-# invoke the whole distutils infrastructure even if the module is
-# already built. It might be more efficient to only do it when the
-# mod time of the .pyx is newer than the mod time of the .so but
-# the question is how to get distutils to tell me the name of the .so
-# before it builds it. Maybe it is easy...but maybe the performance
-# issue isn't real.
-def _load_pyrex(name, filename):
- "Load a pyrex file given a name and filename."
-
-
-def get_distutils_extension(modname, pyxfilename, language_level=None):
-# try:
-# import hashlib
-# except ImportError:
-# import md5 as hashlib
-# extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()
-# modname = modname + extra
- extension_mod,setup_args = handle_special_build(modname, pyxfilename)
- if not extension_mod:
- if not isinstance(pyxfilename, str):
- # distutils is stupid in Py2 and requires exactly 'str'
- # => encode accidentally coerced unicode strings back to str
- pyxfilename = pyxfilename.encode(sys.getfilesystemencoding())
- from distutils.extension import Extension
- extension_mod = Extension(name = modname, sources=[pyxfilename])
- if language_level is not None:
- extension_mod.cython_directives = {'language_level': language_level}
- return extension_mod,setup_args
-
-
-def handle_special_build(modname, pyxfilename):
- special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT
- ext = None
- setup_args={}
- if os.path.exists(special_build):
- # globls = {}
- # locs = {}
- # execfile(special_build, globls, locs)
- # ext = locs["make_ext"](modname, pyxfilename)
- mod = imp.load_source("XXXX", special_build, open(special_build))
- make_ext = getattr(mod,'make_ext',None)
- if make_ext:
- ext = make_ext(modname, pyxfilename)
- assert ext and ext.sources, "make_ext in %s did not return Extension" % special_build
- make_setup_args = getattr(mod, 'make_setup_args',None)
- if make_setup_args:
- setup_args = make_setup_args()
- assert isinstance(setup_args,dict), ("make_setup_args in %s did not return a dict"
- % special_build)
- assert set or setup_args, ("neither make_ext nor make_setup_args %s"
- % special_build)
- ext.sources = [os.path.join(os.path.dirname(special_build), source)
- for source in ext.sources]
- return ext, setup_args
-
-
-def handle_dependencies(pyxfilename):
- testing = '_test_files' in globals()
- dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT
-
- # by default let distutils decide whether to rebuild on its own
- # (it has a better idea of what the output file will be)
-
- # but we know more about dependencies so force a rebuild if
- # some of the dependencies are newer than the pyxfile.
- if os.path.exists(dependfile):
- depends = open(dependfile).readlines()
- depends = [depend.strip() for depend in depends]
-
- # gather dependencies in the "files" variable
- # the dependency file is itself a dependency
- files = [dependfile]
- for depend in depends:
- fullpath = os.path.join(os.path.dirname(dependfile),
- depend)
- files.extend(glob.glob(fullpath))
-
- # only for unit testing to see we did the right thing
- if testing:
- _test_files[:] = [] #$pycheck_no
-
- # if any file that the pyxfile depends upon is newer than
- # the pyx file, 'touch' the pyx file so that distutils will
- # be tricked into rebuilding it.
- for file in files:
- from distutils.dep_util import newer
- if newer(file, pyxfilename):
- _debug("Rebuilding %s because of %s", pyxfilename, file)
- filetime = os.path.getmtime(file)
- os.utime(pyxfilename, (filetime, filetime))
- if testing:
- _test_files.append(file)
-
-
-def build_module(name, pyxfilename, pyxbuild_dir=None, inplace=False, language_level=None):
- assert os.path.exists(pyxfilename), "Path does not exist: %s" % pyxfilename
- handle_dependencies(pyxfilename)
-
- extension_mod, setup_args = get_distutils_extension(name, pyxfilename, language_level)
- build_in_temp = pyxargs.build_in_temp
- sargs = pyxargs.setup_args.copy()
- sargs.update(setup_args)
- build_in_temp = sargs.pop('build_in_temp',build_in_temp)
-
- from . import pyxbuild
- so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod,
- build_in_temp=build_in_temp,
- pyxbuild_dir=pyxbuild_dir,
- setup_args=sargs,
- inplace=inplace,
- reload_support=pyxargs.reload_support)
- assert os.path.exists(so_path), "Cannot find: %s" % so_path
-
- junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ? yes, indeed, trying to eat my files ;)
- junkstuff = glob.glob(junkpath)
- for path in junkstuff:
- if path != so_path:
- try:
- os.remove(path)
- except IOError:
- _info("Couldn't remove %s", path)
-
- return so_path
-
-
-def load_module(name, pyxfilename, pyxbuild_dir=None, is_package=False,
- build_inplace=False, language_level=None, so_path=None):
- try:
- if so_path is None:
- if is_package:
- module_name = name + '.__init__'
- else:
- module_name = name
- so_path = build_module(module_name, pyxfilename, pyxbuild_dir,
- inplace=build_inplace, language_level=language_level)
- mod = imp.load_dynamic(name, so_path)
- if is_package and not hasattr(mod, '__path__'):
- mod.__path__ = [os.path.dirname(so_path)]
- assert mod.__file__ == so_path, (mod.__file__, so_path)
- except Exception:
- if pyxargs.load_py_module_on_import_failure and pyxfilename.endswith('.py'):
- # try to fall back to normal import
- mod = imp.load_source(name, pyxfilename)
- assert mod.__file__ in (pyxfilename, pyxfilename+'c', pyxfilename+'o'), (mod.__file__, pyxfilename)
- else:
- tb = sys.exc_info()[2]
- import traceback
- exc = ImportError("Building module %s failed: %s" % (
- name, traceback.format_exception_only(*sys.exc_info()[:2])))
- if sys.version_info[0] >= 3:
- raise exc.with_traceback(tb)
- else:
- exec("raise exc, None, tb", {'exc': exc, 'tb': tb})
- return mod
-
-
-# import hooks
-
-class PyxImporter(object):
- """A meta-path importer for .pyx files.
- """
- def __init__(self, extension=PYX_EXT, pyxbuild_dir=None, inplace=False,
- language_level=None):
- self.extension = extension
- self.pyxbuild_dir = pyxbuild_dir
- self.inplace = inplace
- self.language_level = language_level
-
- def find_module(self, fullname, package_path=None):
- if fullname in sys.modules and not pyxargs.reload_support:
- return None # only here when reload()
-
- # package_path might be a _NamespacePath. Convert that into a list...
- if package_path is not None and not isinstance(package_path, list):
- package_path = list(package_path)
- try:
- fp, pathname, (ext,mode,ty) = imp.find_module(fullname,package_path)
- if fp: fp.close() # Python should offer a Default-Loader to avoid this double find/open!
- if pathname and ty == imp.PKG_DIRECTORY:
- pkg_file = os.path.join(pathname, '__init__'+self.extension)
- if os.path.isfile(pkg_file):
- return PyxLoader(fullname, pathname,
- init_path=pkg_file,
- pyxbuild_dir=self.pyxbuild_dir,
- inplace=self.inplace,
- language_level=self.language_level)
- if pathname and pathname.endswith(self.extension):
- return PyxLoader(fullname, pathname,
- pyxbuild_dir=self.pyxbuild_dir,
- inplace=self.inplace,
- language_level=self.language_level)
- if ty != imp.C_EXTENSION: # only when an extension, check if we have a .pyx next!
- return None
-
- # find .pyx fast, when .so/.pyd exist --inplace
- pyxpath = os.path.splitext(pathname)[0]+self.extension
- if os.path.isfile(pyxpath):
- return PyxLoader(fullname, pyxpath,
- pyxbuild_dir=self.pyxbuild_dir,
- inplace=self.inplace,
- language_level=self.language_level)
-
- # .so/.pyd's on PATH should not be remote from .pyx's
- # think no need to implement PyxArgs.importer_search_remote here?
-
- except ImportError:
- pass
-
- # searching sys.path ...
-
- #if DEBUG_IMPORT: print "SEARCHING", fullname, package_path
-
- mod_parts = fullname.split('.')
- module_name = mod_parts[-1]
- pyx_module_name = module_name + self.extension
-
- # this may work, but it returns the file content, not its path
- #import pkgutil
- #pyx_source = pkgutil.get_data(package, pyx_module_name)
-
- paths = package_path or sys.path
- for path in paths:
- pyx_data = None
- if not path:
- path = os.getcwd()
- elif os.path.isfile(path):
- try:
- zi = zipimporter(path)
- pyx_data = zi.get_data(pyx_module_name)
- except (ZipImportError, IOError, OSError):
- continue # Module not found.
- # unzip the imported file into the build dir
- # FIXME: can interfere with later imports if build dir is in sys.path and comes before zip file
- path = self.pyxbuild_dir
- elif not os.path.isabs(path):
- path = os.path.abspath(path)
-
- pyx_module_path = os.path.join(path, pyx_module_name)
- if pyx_data is not None:
- if not os.path.exists(path):
- try:
- os.makedirs(path)
- except OSError:
- # concurrency issue?
- if not os.path.exists(path):
- raise
- with open(pyx_module_path, "wb") as f:
- f.write(pyx_data)
- elif not os.path.isfile(pyx_module_path):
- continue # Module not found.
-
- return PyxLoader(fullname, pyx_module_path,
- pyxbuild_dir=self.pyxbuild_dir,
- inplace=self.inplace,
- language_level=self.language_level)
-
- # not found, normal package, not a .pyx file, none of our business
- _debug("%s not found" % fullname)
- return None
-
-
-class PyImporter(PyxImporter):
- """A meta-path importer for normal .py files.
- """
- def __init__(self, pyxbuild_dir=None, inplace=False, language_level=None):
- if language_level is None:
- language_level = sys.version_info[0]
- self.super = super(PyImporter, self)
- self.super.__init__(extension='.py', pyxbuild_dir=pyxbuild_dir, inplace=inplace,
- language_level=language_level)
- self.uncompilable_modules = {}
- self.blocked_modules = ['Cython', 'pyxbuild', 'pyximport.pyxbuild',
- 'distutils.extension', 'distutils.sysconfig']
-
- def find_module(self, fullname, package_path=None):
- if fullname in sys.modules:
- return None
- if fullname.startswith('Cython.'):
- return None
- if fullname in self.blocked_modules:
- # prevent infinite recursion
- return None
- if _lib_loader.knows(fullname):
- return _lib_loader
- _debug("trying import of module '%s'", fullname)
- if fullname in self.uncompilable_modules:
- path, last_modified = self.uncompilable_modules[fullname]
- try:
- new_last_modified = os.stat(path).st_mtime
- if new_last_modified > last_modified:
- # import would fail again
- return None
- except OSError:
- # module is no longer where we found it, retry the import
- pass
-
- self.blocked_modules.append(fullname)
- try:
- importer = self.super.find_module(fullname, package_path)
- if importer is not None:
- if importer.init_path:
- path = importer.init_path
- real_name = fullname + '.__init__'
- else:
- path = importer.path
- real_name = fullname
- _debug("importer found path %s for module %s", path, real_name)
- try:
- so_path = build_module(
- real_name, path,
- pyxbuild_dir=self.pyxbuild_dir,
- language_level=self.language_level,
- inplace=self.inplace)
- _lib_loader.add_lib(fullname, path, so_path,
- is_package=bool(importer.init_path))
- return _lib_loader
- except Exception:
- if DEBUG_IMPORT:
- import traceback
- traceback.print_exc()
- # build failed, not a compilable Python module
- try:
- last_modified = os.stat(path).st_mtime
- except OSError:
- last_modified = 0
- self.uncompilable_modules[fullname] = (path, last_modified)
- importer = None
- finally:
- self.blocked_modules.pop()
- return importer
-
-
-class LibLoader(object):
- def __init__(self):
- self._libs = {}
-
- def load_module(self, fullname):
- try:
- source_path, so_path, is_package = self._libs[fullname]
- except KeyError:
- raise ValueError("invalid module %s" % fullname)
- _debug("Loading shared library module '%s' from %s", fullname, so_path)
- return load_module(fullname, source_path, so_path=so_path, is_package=is_package)
-
- def add_lib(self, fullname, path, so_path, is_package):
- self._libs[fullname] = (path, so_path, is_package)
-
- def knows(self, fullname):
- return fullname in self._libs
-
-_lib_loader = LibLoader()
-
-
-class PyxLoader(object):
- def __init__(self, fullname, path, init_path=None, pyxbuild_dir=None,
- inplace=False, language_level=None):
- _debug("PyxLoader created for loading %s from %s (init path: %s)",
- fullname, path, init_path)
- self.fullname = fullname
- self.path, self.init_path = path, init_path
- self.pyxbuild_dir = pyxbuild_dir
- self.inplace = inplace
- self.language_level = language_level
-
- def load_module(self, fullname):
- assert self.fullname == fullname, (
- "invalid module, expected %s, got %s" % (
- self.fullname, fullname))
- if self.init_path:
- # package
- #print "PACKAGE", fullname
- module = load_module(fullname, self.init_path,
- self.pyxbuild_dir, is_package=True,
- build_inplace=self.inplace,
- language_level=self.language_level)
- module.__path__ = [self.path]
- else:
- #print "MODULE", fullname
- module = load_module(fullname, self.path,
- self.pyxbuild_dir,
- build_inplace=self.inplace,
- language_level=self.language_level)
- return module
-
-
-#install args
-class PyxArgs(object):
- build_dir=True
- build_in_temp=True
- setup_args={} #None
-
-##pyxargs=None
-
-
-def _have_importers():
- has_py_importer = False
- has_pyx_importer = False
- for importer in sys.meta_path:
- if isinstance(importer, PyxImporter):
- if isinstance(importer, PyImporter):
- has_py_importer = True
- else:
- has_pyx_importer = True
-
- return has_py_importer, has_pyx_importer
-
-
-def install(pyximport=True, pyimport=False, build_dir=None, build_in_temp=True,
- setup_args=None, reload_support=False,
- load_py_module_on_import_failure=False, inplace=False,
- language_level=None):
- """ Main entry point for pyxinstall.
-
- Call this to install the ``.pyx`` import hook in
- your meta-path for a single Python process. If you want it to be
- installed whenever you use Python, add it to your ``sitecustomize``
- (as described above).
-
- :param pyximport: If set to False, does not try to import ``.pyx`` files.
-
- :param pyimport: You can pass ``pyimport=True`` to also
- install the ``.py`` import hook
- in your meta-path. Note, however, that it is rather experimental,
- will not work at all for some ``.py`` files and packages, and will
- heavily slow down your imports due to search and compilation.
- Use at your own risk.
-
- :param build_dir: By default, compiled modules will end up in a ``.pyxbld``
- directory in the user's home directory. Passing a different path
- as ``build_dir`` will override this.
-
- :param build_in_temp: If ``False``, will produce the C files locally. Working
- with complex dependencies and debugging becomes more easy. This
- can principally interfere with existing files of the same name.
-
- :param setup_args: Dict of arguments for Distribution.
- See ``distutils.core.setup()``.
-
- :param reload_support: Enables support for dynamic
- ``reload(my_module)``, e.g. after a change in the Cython code.
- Additional files ``<so_path>.reloadNN`` may arise on that account, when
- the previously loaded module file cannot be overwritten.
-
- :param load_py_module_on_import_failure: If the compilation of a ``.py``
- file succeeds, but the subsequent import fails for some reason,
- retry the import with the normal ``.py`` module instead of the
- compiled module. Note that this may lead to unpredictable results
- for modules that change the system state during their import, as
- the second import will rerun these modifications in whatever state
- the system was left after the import of the compiled module
- failed.
-
- :param inplace: Install the compiled module
- (``.so`` for Linux and Mac / ``.pyd`` for Windows)
- next to the source file.
-
- :param language_level: The source language level to use: 2 or 3.
- The default is to use the language level of the current Python
- runtime for .py files and Py2 for ``.pyx`` files.
- """
- if setup_args is None:
- setup_args = {}
- if not build_dir:
- build_dir = os.path.join(os.path.expanduser('~'), '.pyxbld')
-
- global pyxargs
- pyxargs = PyxArgs() #$pycheck_no
- pyxargs.build_dir = build_dir
- pyxargs.build_in_temp = build_in_temp
- pyxargs.setup_args = (setup_args or {}).copy()
- pyxargs.reload_support = reload_support
- pyxargs.load_py_module_on_import_failure = load_py_module_on_import_failure
-
- has_py_importer, has_pyx_importer = _have_importers()
- py_importer, pyx_importer = None, None
-
- if pyimport and not has_py_importer:
- py_importer = PyImporter(pyxbuild_dir=build_dir, inplace=inplace,
- language_level=language_level)
- # make sure we import Cython before we install the import hook
- import Cython.Compiler.Main, Cython.Compiler.Pipeline, Cython.Compiler.Optimize
- sys.meta_path.insert(0, py_importer)
-
- if pyximport and not has_pyx_importer:
- pyx_importer = PyxImporter(pyxbuild_dir=build_dir, inplace=inplace,
- language_level=language_level)
- sys.meta_path.append(pyx_importer)
-
- return py_importer, pyx_importer
-
-
-def uninstall(py_importer, pyx_importer):
- """
- Uninstall an import hook.
- """
- try:
- sys.meta_path.remove(py_importer)
- except ValueError:
- pass
-
- try:
- sys.meta_path.remove(pyx_importer)
- except ValueError:
- pass
-
-
-# MAIN
-
-def show_docs():
- import __main__
- __main__.__name__ = mod_name
- for name in dir(__main__):
- item = getattr(__main__, name)
- try:
- setattr(item, "__module__", mod_name)
- except (AttributeError, TypeError):
- pass
- help(__main__)
+if sys.version_info < (3, 5):
+ # _pyximport3 module requires at least Python 3.5
+ from pyximport._pyximport2 import install, uninstall, show_docs
+else:
+ from pyximport._pyximport3 import install, uninstall, show_docs
if __name__ == '__main__':
show_docs()
diff --git a/pyximport/test/test_pyximport.py b/pyximport/test/test_pyximport.py
index b3a4a9058..b7fd8d7ee 100644
--- a/pyximport/test/test_pyximport.py
+++ b/pyximport/test/test_pyximport.py
@@ -41,34 +41,37 @@ def test_with_reload():
tempdir = make_tempdir()
sys.path.append(tempdir)
filename = os.path.join(tempdir, "dummy.pyx")
- open(filename, "w").write("print 'Hello world from the Pyrex install hook'")
+ with open(filename, "w") as fid:
+ fid.write("print 'Hello world from the Pyrex install hook'")
import dummy
reload(dummy)
depend_filename = os.path.join(tempdir, "dummy.pyxdep")
- depend_file = open(depend_filename, "w")
- depend_file.write("*.txt\nfoo.bar")
- depend_file.close()
+ with open(depend_filename, "w") as depend_file:
+ depend_file.write("*.txt\nfoo.bar")
build_filename = os.path.join(tempdir, "dummy.pyxbld")
- build_file = open(build_filename, "w")
- build_file.write("""
+ with open(build_filename, "w") as build_file:
+ build_file.write("""
from distutils.extension import Extension
def make_ext(name, filename):
return Extension(name=name, sources=[filename])
""")
- build_file.close()
- open(os.path.join(tempdir, "foo.bar"), "w").write(" ")
- open(os.path.join(tempdir, "1.txt"), "w").write(" ")
- open(os.path.join(tempdir, "abc.txt"), "w").write(" ")
+ with open(os.path.join(tempdir, "foo.bar"), "w") as fid:
+ fid.write(" ")
+ with open(os.path.join(tempdir, "1.txt"), "w") as fid:
+ fid.write(" ")
+ with open(os.path.join(tempdir, "abc.txt"), "w") as fid:
+ fid.write(" ")
reload(dummy)
assert len(pyximport._test_files)==1, pyximport._test_files
reload(dummy)
- time.sleep(1) # sleep a second to get safer mtimes
- open(os.path.join(tempdir, "abc.txt"), "w").write(" ")
- print("Here goes the reolad")
+ time.sleep(1) # sleep a second to get safer mtimes
+ with open(os.path.join(tempdir, "abc.txt"), "w") as fid:
+ fid.write(" ")
+ print("Here goes the reload")
reload(dummy)
assert len(pyximport._test_files) == 1, pyximport._test_files
diff --git a/pyximport/test/test_reload.py b/pyximport/test/test_reload.py
index ba53746f9..0ba5ba13f 100644
--- a/pyximport/test/test_reload.py
+++ b/pyximport/test/test_reload.py
@@ -18,14 +18,16 @@ def test():
tempdir = test_pyximport.make_tempdir()
sys.path.append(tempdir)
hello_file = os.path.join(tempdir, "hello.pyx")
- open(hello_file, "w").write("x = 1; print x; before = 'before'\n")
+ with open(hello_file, "w") as fid:
+ fid.write("x = 1; print x; before = 'before'\n")
import hello
assert hello.x == 1
- time.sleep(1) # sleep to make sure that new "hello.pyx" has later
- # timestamp than object file.
+ time.sleep(1) # sleep to make sure that new "hello.pyx" has later
+ # timestamp than object file.
- open(hello_file, "w").write("x = 2; print x; after = 'after'\n")
+ with open(hello_file, "w") as fid:
+ fid.write("x = 2; print x; after = 'after'\n")
reload(hello)
assert hello.x == 2, "Reload should work on Python 2.3 but not 2.2"
test_pyximport.remove_tempdir(tempdir)
View Lorry Controller Status