diff options
| -rw-r--r-- | astroid/interpreter/_import/spec.py | 242 | ||||
| -rw-r--r-- | astroid/interpreter/_import/util.py | 12 | ||||
| -rw-r--r-- | astroid/manager.py | 32 | ||||
| -rw-r--r-- | astroid/modutils.py | 282 | ||||
| -rw-r--r-- | astroid/tests/unittest_modutils.py | 17 |
5 files changed, 306 insertions, 279 deletions
diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py new file mode 100644 index 00000000..6d6c228f --- /dev/null +++ b/astroid/interpreter/_import/spec.py @@ -0,0 +1,242 @@ +# Copyright (c) 2016 Claudiu Popa <pcmanticore@gmail.com> + +import abc +import collections +import enum +import imp +import os +import sys +import zipimport +try: + import importlib.machinery + _HAS_MACHINERY = True +except ImportError: + _HAS_MACHINERY = False + +from . import util + +ModuleType = enum.Enum('ModuleType', 'C_BUILTIN C_EXTENSION PKG_DIRECTORY ' + 'PY_CODERESOURCE PY_COMPILED PY_FROZEN PY_RESOURCE ' + 'PY_SOURCE PY_ZIPMODULE PY_NAMESPACE') +_ImpTypes = {imp.C_BUILTIN: ModuleType.C_BUILTIN, + imp.C_EXTENSION: ModuleType.C_EXTENSION, + imp.PKG_DIRECTORY: ModuleType.PKG_DIRECTORY, + imp.PY_COMPILED: ModuleType.PY_COMPILED, + imp.PY_FROZEN: ModuleType.PY_FROZEN, + imp.PY_SOURCE: ModuleType.PY_SOURCE, + } +if hasattr(imp, 'PY_RESOURCE'): + _ImpTypes[imp.PY_RESOURCE] = ModuleType.PY_RESOURCE +if hasattr(imp, 'PY_CODERESOURCE'): + _ImpTypes[imp.PY_CODERESOURCE] = ModuleType.PY_CODERESOURCE + +def _imp_type_to_module_type(imp_type): + return _ImpTypes[imp_type] + + + +_ModuleSpec = collections.namedtuple('_ModuleSpec', 'name type location ' + 'origin submodule_search_locations') + +class ModuleSpec(_ModuleSpec): + + def __new__(cls, name, type, location=None, origin=None, + submodule_search_locations=None): + return _ModuleSpec.__new__(cls, name=name, type=type, + location=location, origin=origin, + submodule_search_locations=submodule_search_locations) + + +class Finder(object): + + def __init__(self, path=None): + self._path = path or sys.path + + @abc.abstractmethod + def find_module(self, modname, module_parts, processed, submodule_path): + pass + + def contribute_to_path(self, filename, processed): + return None + + +class ImpFinder(Finder): + + def find_module(self, modname, _, processed, submodule_path): + try: + stream, mp_filename, mp_desc = imp.find_module(modname, submodule_path) + except ImportError: + return None + + # Close resources. + if stream: + stream.close() + + return ModuleSpec(name=modname, location=mp_filename, + type=_imp_type_to_module_type(mp_desc[2])) + + def contribute_to_path(self, spec, processed): + if spec.location is None: + # Builtin. + return None + + if _is_setuptools_namespace(spec.location): + # extend_path is called, search sys.path for module/packages + # of this name see pkgutil.extend_path documentation + path = [os.path.join(p, *processed) for p in sys.path + if os.path.isdir(os.path.join(p, *processed))] + else: + path = [spec.location] + return path + + +class DynamicImpFinder(ImpFinder): + + def find_module(self, modname, module_parts, processed, submodule_path): + if util.is_namespace(modname) and modname in sys.modules: + submodule_path = sys.modules[modname].__path__ + return ModuleSpec(name=modname, location='', + origin='namespace', + type=ModuleType.PY_NAMESPACE, + submodule_search_locations=submodule_path) + + + def contribute_to_path(self, spec, processed): + return spec.submodule_search_locations + + + +class ZipFinder(Finder): + + def __init__(self, path): + super(ZipFinder, self).__init__(path) + self._zipimporters = _precache_zipimporters(path) + + def find_module(self, modname, module_parts, processed, submodule_path): + try: + file_type, filename, path = _search_zip(module_parts, self._zipimporters) + except ImportError: + return None + + return ModuleSpec(name=modname, location=filename, + origin='egg', type=file_type, + submodule_search_locations=path) + + +class PEP420SpecFinder(Finder): + + def find_module(self, modname, module_parts, processed, submodule_path): + spec = importlib.machinery.PathFinder.find_spec(modname, path=submodule_path) + if spec: + location = spec.origin if spec.origin != 'namespace' else None + type = ModuleType.PY_NAMESPACE if spec.origin == 'namespace' else None + spec = ModuleSpec(name=spec.name, location=location, + origin=spec.origin, type=type, + submodule_search_locations=list(spec.submodule_search_locations or [])) + return spec + + def contribute_to_path(self, spec, processed): + if spec.type == ModuleType.PY_NAMESPACE: + return spec.submodule_search_locations + return None + +_SPEC_FINDERS = ( + ImpFinder, + ZipFinder, +) +if _HAS_MACHINERY and sys.version_info[:2] > (3, 3): + _SPEC_FINDERS += (PEP420SpecFinder, ) +_SPEC_FINDERS += (DynamicImpFinder, ) + + +def _is_setuptools_namespace(location): + try: + with open(os.path.join(location, '__init__.py'), 'rb') as stream: + data = stream.read(4096) + except IOError: + pass + else: + extend_path = b'pkgutil' in data and b'extend_path' in data + declare_namespace = ( + b"pkg_resources" in data + and b"declare_namespace(__name__)" in data) + return extend_path or declare_namespace + + +def _precache_zipimporters(path=None): + pic = sys.path_importer_cache + path = path or sys.path + for entry_path in path: + if entry_path not in pic: + try: + pic[entry_path] = zipimport.zipimporter(entry_path) + except zipimport.ZipImportError: + continue + return pic + + +def _search_zip(modpath, pic): + for filepath, importer in list(pic.items()): + if importer is not None: + found = importer.find_module(modpath[0]) + if found: + if not importer.find_module(os.path.sep.join(modpath)): + raise ImportError('No module named %s in %s/%s' % ( + '.'.join(modpath[1:]), filepath, modpath)) + #import code; code.interact(local=locals()) + return (ModuleType.PY_ZIPMODULE, + os.path.abspath(filepath) + os.path.sep + os.path.sep.join(modpath), + filepath) + raise ImportError('No module named %s' % '.'.join(modpath)) + + +def _find_spec_with_path(search_path, modname, module_parts, processed, submodule_path): + finders = [finder(search_path) for finder in _SPEC_FINDERS] + for finder in finders: + spec = finder.find_module(modname, module_parts, processed, submodule_path) + if spec is None: + continue + return finder, spec + + raise ImportError('No module named %s' % '.'.join(module_parts)) + + +def find_spec(modpath, path=None): + """get a module type / file path + + :type modpath: list or tuple + :param modpath: + split module's name (i.e name of a module or package split + on '.'), with leading empty strings for explicit relative import + + :type path: list or None + :param path: + optional list of path where the module or package should be + searched (use sys.path if nothing or None is given) + + + :rtype: tuple(int, str) + :return: the module type flag and the file path for a module + """ + _path = path or sys.path + + # Need a copy for not mutating the argument. + modpath = modpath[:] + + submodule_path = None + module_parts = modpath[:] + processed = [] + + while modpath: + modname = modpath.pop(0) + finder, spec = _find_spec_with_path(_path, modname, + module_parts, processed, + submodule_path or path) + processed.append(modname) + if modpath: + submodule_path = finder.contribute_to_path(spec, processed) + + if spec.type == ModuleType.PKG_DIRECTORY: + spec = spec._replace(submodule_search_locations=submodule_path) + + return spec diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py new file mode 100644 index 00000000..6dc7b029 --- /dev/null +++ b/astroid/interpreter/_import/util.py @@ -0,0 +1,12 @@ +# Copyright (c) 2016 Claudiu Popa <pcmanticore@gmail.com> + +try: + import pkg_resources +except ImportError: + pkg_resources = None + + +def is_namespace(modname): + # pylint: disable=no-member; astroid issue #290, modifying globals at runtime. + return (pkg_resources is not None + and modname in pkg_resources._namespace_packages)
\ No newline at end of file diff --git a/astroid/manager.py b/astroid/manager.py index 59623828..05512200 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -11,7 +11,6 @@ possible by providing a class responsible to get astroid representation from various source and using a cache of built modules) """ -import imp import os import sys import zipimport @@ -19,6 +18,7 @@ import zipimport import six from astroid import exceptions +from astroid.interpreter._import import spec from astroid import modutils from astroid import transforms from astroid import util @@ -111,13 +111,16 @@ class AstroidManager(object): if context_file: os.chdir(os.path.dirname(context_file)) try: - spec = self.file_from_module_name(modname, context_file) - if spec.type == modutils.ModuleType.PY_ZIPMODULE: - module = self.zip_import_data(spec.location) + found_spec = self.file_from_module_name(modname, context_file) + if found_spec.type == spec.ModuleType.PY_ZIPMODULE: + module = self.zip_import_data(found_spec.location) if module is not None: return module - elif spec.type in (modutils.ModuleType.C_BUILTIN, modutils.ModuleType.C_EXTENSION): - if spec.type == modutils.ModuleType.C_EXTENSION and not self._can_load_extension(modname): + + elif found_spec.type in (spec.ModuleType.C_BUILTIN, + spec.ModuleType.C_EXTENSION): + if (found_spec.type == spec.ModuleType.C_EXTENSION + and not self._can_load_extension(modname)): return self._build_stub_module(modname) try: module = modutils.load_module_from_name(modname) @@ -126,17 +129,22 @@ class AstroidManager(object): 'Loading {modname} failed with:\n{error}', modname=modname, path=spec.location, error=ex)) return self.ast_from_module(module, modname) - elif spec.type == modutils.ModuleType.PY_COMPILED: + + elif found_spec.type == spec.ModuleType.PY_COMPILED: raise exceptions.AstroidImportError( "Unable to load compiled module {modname}.", - modname=modname, path=spec.location) - elif spec.type == modutils.ModuleType.PY_NAMESPACE: - return self._build_namespace_module(modname, spec.submodule_search_locations) - if spec.location is None: + modname=modname, path=found_spec.location) + + elif found_spec.type == spec.ModuleType.PY_NAMESPACE: + return self._build_namespace_module(modname, + found_spec.submodule_search_locations) + + if found_spec.location is None: raise exceptions.AstroidImportError( "Can't find a file for module {modname}.", modname=modname) - return self.ast_from_file(spec.location, modname, fallback=False) + + return self.ast_from_file(found_spec.location, modname, fallback=False) except exceptions.AstroidBuildingError as e: for hook in self._failed_import_hooks: try: diff --git a/astroid/modutils.py b/astroid/modutils.py index 3334e1e3..d198be7c 100644 --- a/astroid/modutils.py +++ b/astroid/modutils.py @@ -19,10 +19,6 @@ :type BUILTIN_MODULES: dict :var BUILTIN_MODULES: dictionary with builtin module names has key """ - -import abc -import collections -import enum import imp import os import platform @@ -34,12 +30,6 @@ from distutils.errors import DistutilsPlatformError # distutils is replaced by virtualenv with a module that does # weird path manipulations in order to get to the # real distutils module. -import zipimport -try: - import importlib.machinery - _HAS_MACHINERY = True -except ImportError: - _HAS_MACHINERY = False try: import pkg_resources @@ -47,23 +37,8 @@ except ImportError: pkg_resources = None import six -ModuleType = enum.Enum('ModuleType', 'C_BUILTIN C_EXTENSION PKG_DIRECTORY ' - 'PY_CODERESOURCE PY_COMPILED PY_FROZEN PY_RESOURCE ' - 'PY_SOURCE PY_ZIPMODULE PY_NAMESPACE') -_ImpTypes = {imp.C_BUILTIN: ModuleType.C_BUILTIN, - imp.C_EXTENSION: ModuleType.C_EXTENSION, - imp.PKG_DIRECTORY: ModuleType.PKG_DIRECTORY, - imp.PY_COMPILED: ModuleType.PY_COMPILED, - imp.PY_FROZEN: ModuleType.PY_FROZEN, - imp.PY_SOURCE: ModuleType.PY_SOURCE, - } -if hasattr(imp, 'PY_RESOURCE'): - _ImpTypes[imp.PY_RESOURCE] = ModuleType.PY_RESOURCE -if hasattr(imp, 'PY_CODERESOURCE'): - _ImpTypes[imp.PY_CODERESOURCE] = ModuleType.PY_CODERESOURCE - -def _imp_type_to_module_type(imp_type): - return _ImpTypes[imp_type] +from .interpreter._import import spec +from .interpreter._import import util if sys.platform.startswith('win'): PY_SOURCE_EXTS = ('py', 'pyw') @@ -260,7 +235,7 @@ def load_module_from_modpath(parts, path=None, use_sys=1): setattr(prevmodule, part, module) _file = getattr(module, '__file__', '') prevmodule = module - if not _file and _is_namespace(curname): + if not _file and util.is_namespace(curname): continue if not _file and len(modpath) != len(parts): raise ImportError('no module in %s' % '.'.join(parts[len(modpath):])) @@ -301,7 +276,7 @@ def check_modpath_has_init(path, mod_path): modpath.append(part) path = os.path.join(path, part) if not _has_init(path): - old_namespace = _is_namespace('.'.join(modpath)) + old_namespace = util.is_namespace('.'.join(modpath)) if not old_namespace: return False return True @@ -401,7 +376,7 @@ def file_info_from_modpath(modpath, path=None, context_file=None): return _spec_from_modpath(modpath, path, context) elif modpath == ['os', 'path']: # FIXME: currently ignoring search_path... - return ModuleSpec(name='os.path', location=os.path.__file__, type=imp.PY_SOURCE) + return spec.ModuleSpec(name='os.path', location=os.path.__file__, type=imp.PY_SOURCE) return _spec_from_modpath(modpath, path, context) @@ -560,7 +535,7 @@ def is_standard_module(modname, std_path=None): # (sys and __builtin__ for instance) if filename is None: # we assume there are no namespaces in stdlib - return not _is_namespace(modname) + return not util.is_namespace(modname) filename = _normalize_path(filename) if filename.startswith(_cache_normalize_path(EXT_LIB_DIR)): return False @@ -616,241 +591,26 @@ def _spec_from_modpath(modpath, path=None, context=None): location = None if context is not None: try: - spec = _find_spec(modpath, [context]) - location = spec.location + found_spec = spec.find_spec(modpath, [context]) + location = found_spec.location except ImportError: - spec = _find_spec(modpath, path) - location = spec.location + found_spec = spec.find_spec(modpath, path) + location = found_spec.location else: - spec = _find_spec(modpath, path) - if spec.type == ModuleType.PY_COMPILED: + found_spec = spec.find_spec(modpath, path) + if found_spec.type == spec.ModuleType.PY_COMPILED: try: - location = get_source_file(spec.location) - return spec._replace(location=location, type=ModuleSpec.PY_SOURCE) + location = get_source_file(found_spec.location) + return found_spec._replace(location=location, type=spec.ModuleSpec.PY_SOURCE) except NoSourceFile: - return spec.replace(location=location) - elif spec.type == ModuleType.C_BUILTIN: + return found_spec.replace(location=location) + elif found_spec.type == spec.ModuleType.C_BUILTIN: # integrated builtin module - return spec._replace(location=None) - elif spec.type == ModuleType.PKG_DIRECTORY: - location = _has_init(spec.location) - return spec._replace(location=location, type=ModuleType.PY_SOURCE) - return spec - - -def _search_zip(modpath, pic): - for filepath, importer in list(pic.items()): - if importer is not None: - found = importer.find_module(modpath[0]) - if found: - if not importer.find_module(os.path.sep.join(modpath)): - raise ImportError('No module named %s in %s/%s' % ( - '.'.join(modpath[1:]), filepath, modpath)) - #import code; code.interact(local=locals()) - return (ModuleType.PY_ZIPMODULE, - os.path.abspath(filepath) + os.path.sep + os.path.sep.join(modpath), - filepath) - raise ImportError('No module named %s' % '.'.join(modpath)) - - -def _precache_zipimporters(path=None): - pic = sys.path_importer_cache - path = path or sys.path - for entry_path in path: - if entry_path not in pic: - try: - pic[entry_path] = zipimport.zipimporter(entry_path) - except zipimport.ZipImportError: - continue - return pic - - -def _is_namespace(modname): - # pylint: disable=no-member; astroid issue #290, modifying globals at runtime. - return (pkg_resources is not None - and modname in pkg_resources._namespace_packages) - - -def _is_setuptools_namespace(location): - try: - with open(os.path.join(location, '__init__.py'), 'rb') as stream: - data = stream.read(4096) - except IOError: - pass - else: - extend_path = b'pkgutil' in data and b'extend_path' in data - declare_namespace = ( - b"pkg_resources" in data - and b"declare_namespace(__name__)" in data) - return extend_path or declare_namespace - - -# Spec finders. - -_ModuleSpec = collections.namedtuple('_ModuleSpec', 'name type location ' - 'origin submodule_search_locations') - -class ModuleSpec(_ModuleSpec): - - def __new__(cls, name, type, location=None, origin=None, - submodule_search_locations=None): - return _ModuleSpec.__new__(cls, name=name, type=type, - location=location, origin=origin, - submodule_search_locations=submodule_search_locations) - - -class Finder(object): - - def __init__(self, path=None): - self._path = path or sys.path - - @abc.abstractmethod - def find_module(self, modname, module_parts, processed, submodule_path): - pass - - def contribute_to_path(self, filename, processed): - return None - - -class ImpFinder(Finder): - - def find_module(self, modname, _, processed, submodule_path): - try: - stream, mp_filename, mp_desc = imp.find_module(modname, submodule_path) - except ImportError: - return None - - # Close resources. - if stream: - stream.close() - - return ModuleSpec(name=modname, location=mp_filename, - type=_imp_type_to_module_type(mp_desc[2])) - - def contribute_to_path(self, spec, processed): - if spec.location is None: - # Builtin. - return None - - if _is_setuptools_namespace(spec.location): - # extend_path is called, search sys.path for module/packages - # of this name see pkgutil.extend_path documentation - path = [os.path.join(p, *processed) for p in sys.path - if os.path.isdir(os.path.join(p, *processed))] - else: - path = [spec.location] - return path - - -class DynamicImpFinder(ImpFinder): - - def find_module(self, modname, module_parts, processed, submodule_path): - if _is_namespace(modname) and modname in sys.modules: - submodule_path = sys.modules[modname].__path__ - return ModuleSpec(name=modname, location='', - origin='namespace', - type=ModuleType.PY_NAMESPACE, - submodule_search_locations=submodule_path) - - - def contribute_to_path(self, spec, processed): - return spec.submodule_search_locations - - - -class ZipFinder(Finder): - - def __init__(self, path): - super(ZipFinder, self).__init__(path) - self._zipimporters = _precache_zipimporters(path) - - def find_module(self, modname, module_parts, processed, submodule_path): - try: - file_type, filename, path = _search_zip(module_parts, self._zipimporters) - except ImportError: - return None - - return ModuleSpec(name=modname, location=filename, - origin='egg', type=file_type, - submodule_search_locations=path) - - -class PEP420SpecFinder(Finder): - - def find_module(self, modname, module_parts, processed, submodule_path): - spec = importlib.machinery.PathFinder.find_spec(modname, path=submodule_path) - if spec: - location = spec.origin if spec.origin != 'namespace' else None - type = ModuleType.PY_NAMESPACE if spec.origin == 'namespace' else None - spec = ModuleSpec(name=spec.name, location=location, - origin=spec.origin, type=type, - submodule_search_locations=list(spec.submodule_search_locations or [])) - return spec - - def contribute_to_path(self, spec, processed): - if spec.type == ModuleType.PY_NAMESPACE: - return spec.submodule_search_locations - return None - -_SPEC_FINDERS = ( - ImpFinder, - ZipFinder, -) -if _HAS_MACHINERY and sys.version_info[:2] > (3, 3): - _SPEC_FINDERS += (PEP420SpecFinder, ) -_SPEC_FINDERS += (DynamicImpFinder, ) - - -def _find_spec_with_path(search_path, modname, module_parts, processed, submodule_path): - finders = [finder(search_path) for finder in _SPEC_FINDERS] - for finder in finders: - spec = finder.find_module(modname, module_parts, processed, submodule_path) - if spec is None: - continue - return finder, spec - - raise ImportError('No module named %s' % '.'.join(module_parts)) - - -def _find_spec(modpath, path=None): - """get a module type / file path - - :type modpath: list or tuple - :param modpath: - split module's name (i.e name of a module or package split - on '.'), with leading empty strings for explicit relative import - - :type path: list or None - :param path: - optional list of path where the module or package should be - searched (use sys.path if nothing or None is given) - - - :rtype: tuple(int, str) - :return: the module type flag and the file path for a module - """ - _path = path or sys.path - - # Need a copy for not mutating the argument. - modpath = modpath[:] - - submodule_path = None - module_parts = modpath[:] - processed = [] - - while modpath: - modname = modpath.pop(0) - finder, spec = _find_spec_with_path(_path, modname, - module_parts, processed, - submodule_path or path) - processed.append(modname) - if modpath: - submodule_path = finder.contribute_to_path(spec, processed) - - if spec.type == ModuleType.PKG_DIRECTORY: - spec = spec._replace(submodule_search_locations=submodule_path) - - return spec + return found_spec._replace(location=None) + elif found_spec.type == spec.ModuleType.PKG_DIRECTORY: + location = _has_init(found_spec.location) + return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE) + return found_spec def _is_python_file(filename): diff --git a/astroid/tests/unittest_modutils.py b/astroid/tests/unittest_modutils.py index 93efa56e..f2fac031 100644 --- a/astroid/tests/unittest_modutils.py +++ b/astroid/tests/unittest_modutils.py @@ -17,6 +17,7 @@ import unittest from xml import etree import astroid +from astroid.interpreter._import import spec from astroid import modutils from astroid.tests import resources @@ -34,16 +35,20 @@ class ModuleFileTest(unittest.TestCase): del sys.path_importer_cache[k] def test_find_zipped_module(self): - spec = modutils._find_spec( + found_spec = spec.find_spec( [self.package], [resources.find('data/MyPyPa-0.1.0-py2.5.zip')]) - self.assertEqual(spec.type, modutils.ModuleType.PY_ZIPMODULE) - self.assertEqual(spec.location.split(os.sep)[-3:], ["data", "MyPyPa-0.1.0-py2.5.zip", self.package]) + self.assertEqual(found_spec.type, + spec.ModuleType.PY_ZIPMODULE) + self.assertEqual(found_spec.location.split(os.sep)[-3:], + ["data", "MyPyPa-0.1.0-py2.5.zip", self.package]) def test_find_egg_module(self): - spec = modutils._find_spec( + found_spec = spec.find_spec( [self.package], [resources.find('data/MyPyPa-0.1.0-py2.5.egg')]) - self.assertEqual(spec.type, modutils.ModuleType.PY_ZIPMODULE) - self.assertEqual(spec.location.split(os.sep)[-3:], ["data", "MyPyPa-0.1.0-py2.5.egg", self.package]) + self.assertEqual(found_spec.type, + spec.ModuleType.PY_ZIPMODULE) + self.assertEqual(found_spec.location.split(os.sep)[-3:], + ["data", "MyPyPa-0.1.0-py2.5.egg", self.package]) class LoadModuleFromNameTest(unittest.TestCase): |