diff options
Diffstat (limited to 'logilab/common/modutils.py')
| -rw-r--r-- | logilab/common/modutils.py | 182 |
1 files changed, 102 insertions, 80 deletions
diff --git a/logilab/common/modutils.py b/logilab/common/modutils.py index 76c4ac4..9ca4c81 100644 --- a/logilab/common/modutils.py +++ b/logilab/common/modutils.py @@ -32,8 +32,18 @@ __docformat__ = "restructuredtext en" import sys import os -from os.path import (splitext, join, abspath, isdir, dirname, exists, - basename, expanduser, normcase, realpath) +from os.path import ( + splitext, + join, + abspath, + isdir, + dirname, + exists, + basename, + expanduser, + normcase, + realpath, +) from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY from distutils.sysconfig import get_config_var, get_python_lib from distutils.errors import DistutilsPlatformError @@ -59,19 +69,19 @@ from logilab.common.deprecation import deprecated # # :see: `Problems with /usr/lib64 builds <http://bugs.python.org/issue1294959>`_ # :see: `FHS <http://www.pathname.com/fhs/pub/fhs-2.3.html#LIBLTQUALGTALTERNATEFORMATESSENTIAL>`_ -if sys.platform.startswith('win'): - PY_SOURCE_EXTS = ('py', 'pyw') - PY_COMPILED_EXTS = ('dll', 'pyd') +if sys.platform.startswith("win"): + PY_SOURCE_EXTS = ("py", "pyw") + PY_COMPILED_EXTS = ("dll", "pyd") else: - PY_SOURCE_EXTS = ('py',) - PY_COMPILED_EXTS = ('so',) + PY_SOURCE_EXTS = ("py",) + PY_COMPILED_EXTS = ("so",) try: STD_LIB_DIR = get_python_lib(standard_lib=True) # get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to # non-valid path, see https://bugs.pypy.org/issue1164 except DistutilsPlatformError: - STD_LIB_DIR = '//' + STD_LIB_DIR = "//" EXT_LIB_DIR = get_python_lib() @@ -83,6 +93,7 @@ class NoSourceFile(Exception): source file for a precompiled file """ + class LazyObject(object): def __init__(self, module, obj): self.module = module @@ -91,8 +102,7 @@ class LazyObject(object): def _getobj(self): if self._imported is None: - self._imported = getattr(load_module_from_name(self.module), - self.obj) + self._imported = getattr(load_module_from_name(self.module), self.obj) return self._imported def __getattribute__(self, attr): @@ -105,7 +115,9 @@ class LazyObject(object): return self._getobj()(*args, **kwargs) -def load_module_from_name(dotted_name: str, path: Optional[Any] = None, use_sys: int = True) -> ModuleType: +def load_module_from_name( + dotted_name: str, path: Optional[Any] = None, use_sys: int = True +) -> ModuleType: """Load a Python module from its name. :type dotted_name: str @@ -127,13 +139,15 @@ def load_module_from_name(dotted_name: str, path: Optional[Any] = None, use_sys: :rtype: module :return: the loaded module """ - module = load_module_from_modpath(dotted_name.split('.'), path, use_sys) + module = load_module_from_modpath(dotted_name.split("."), path, use_sys) if module is None: raise ImportError("module %s doesn't exist" % dotted_name) return module -def load_module_from_modpath(parts: List[str], path: Optional[Any] = None, use_sys: int = True) -> Optional[ModuleType]: +def load_module_from_modpath( + parts: List[str], path: Optional[Any] = None, use_sys: int = True +) -> Optional[ModuleType]: """Load a python module from its splitted name. :type parts: list(str) or tuple(str) @@ -156,14 +170,14 @@ def load_module_from_modpath(parts: List[str], path: Optional[Any] = None, use_s """ if use_sys: try: - return sys.modules['.'.join(parts)] + return sys.modules[".".join(parts)] except KeyError: pass modpath = [] prevmodule = None for part in parts: modpath.append(part) - curname = '.'.join(modpath) + curname = ".".join(modpath) module = None if len(modpath) != len(parts): # even with use_sys=False, should try to get outer packages from sys.modules @@ -180,13 +194,13 @@ def load_module_from_modpath(parts: List[str], path: Optional[Any] = None, use_s mp_file.close() if prevmodule: setattr(prevmodule, part, module) - _file = getattr(module, '__file__', '') + _file = getattr(module, "__file__", "") prevmodule = module if not _file and _is_namespace(curname): continue if not _file and len(modpath) != len(parts): - raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) ) - path = [dirname( _file )] + raise ImportError("no module in %s" % ".".join(parts[len(modpath) :])) + path = [dirname(_file)] return module @@ -222,7 +236,7 @@ def _check_init(path: str, mod_path: List[str]) -> bool: for part in mod_path: modpath.append(part) path = join(path, part) - if not _is_namespace('.'.join(modpath)) and not _has_init(path): + if not _is_namespace(".".join(modpath)) and not _has_init(path): return False return True @@ -231,8 +245,7 @@ def _canonicalize_path(path: str) -> str: return realpath(expanduser(path)) - -@deprecated('you should avoid using modpath_from_file()') +@deprecated("you should avoid using modpath_from_file()") def modpath_from_file(filename: str, extrapath: Optional[Dict[str, str]] = None) -> List[str]: """DEPRECATED: doens't play well with symlinks and sys.meta_path @@ -261,23 +274,23 @@ def modpath_from_file(filename: str, extrapath: Optional[Dict[str, str]] = None) if extrapath is not None: for path_ in map(_canonicalize_path, extrapath): path = abspath(path_) - if path and normcase(base[:len(path)]) == normcase(path): - submodpath = [pkg for pkg in base[len(path):].split(os.sep) - if pkg] + if path and normcase(base[: len(path)]) == normcase(path): + submodpath = [pkg for pkg in base[len(path) :].split(os.sep) if pkg] if _check_init(path, submodpath[:-1]): - return extrapath[path_].split('.') + submodpath + return extrapath[path_].split(".") + submodpath for path in map(_canonicalize_path, sys.path): if path and normcase(base).startswith(path): - modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg] + modpath = [pkg for pkg in base[len(path) :].split(os.sep) if pkg] if _check_init(path, modpath[:-1]): return modpath - raise ImportError('Unable to find module for %s in %s' % ( - filename, ', \n'.join(sys.path))) + raise ImportError("Unable to find module for %s in %s" % (filename, ", \n".join(sys.path))) -def file_from_modpath(modpath: List[str], path: Optional[Any] = None, context_file: Optional[str] = None) -> Optional[str]: +def file_from_modpath( + modpath: List[str], path: Optional[Any] = None, context_file: Optional[str] = None +) -> Optional[str]: """given a mod path (i.e. splitted module / package name), return the corresponding file, giving priority to source file over precompiled file if it exists @@ -312,19 +325,18 @@ def file_from_modpath(modpath: List[str], path: Optional[Any] = None, context_fi context = dirname(context_file) else: context = context_file - if modpath[0] == 'xml': + if modpath[0] == "xml": # handle _xmlplus try: - return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context) + return _file_from_modpath(["_xmlplus"] + modpath[1:], path, context) except ImportError: return _file_from_modpath(modpath, path, context) - elif modpath == ['os', 'path']: + elif modpath == ["os", "path"]: # FIXME: currently ignoring search_path... return os.path.__file__ return _file_from_modpath(modpath, path, context) - def get_module_part(dotted_name: str, context_file: Optional[str] = None) -> str: """given a dotted name return the module part of the name : @@ -352,9 +364,9 @@ def get_module_part(dotted_name: str, context_file: Optional[str] = None) -> str (see #10066) """ # os.path trick - if dotted_name.startswith('os.path'): - return 'os.path' - parts = dotted_name.split('.') + if dotted_name.startswith("os.path"): + return "os.path" + parts = dotted_name.split(".") if context_file is not None: # first check for builtin module which won't be considered latter # in that case (path != None) @@ -365,27 +377,27 @@ def get_module_part(dotted_name: str, context_file: Optional[str] = None) -> str # don't use += or insert, we want a new list to be created ! path: Optional[List] = None starti = 0 - if parts[0] == '': - assert context_file is not None, \ - 'explicit relative import, but no context_file?' - path = [] # prevent resolving the import non-relatively + if parts[0] == "": + assert context_file is not None, "explicit relative import, but no context_file?" + path = [] # prevent resolving the import non-relatively starti = 1 - while parts[starti] == '': # for all further dots: change context + while parts[starti] == "": # for all further dots: change context starti += 1 assert context_file is not None context_file = dirname(context_file) for i in range(starti, len(parts)): try: - file_from_modpath(parts[starti:i+1], - path=path, context_file=context_file) + file_from_modpath(parts[starti : i + 1], path=path, context_file=context_file) except ImportError: if not i >= max(1, len(parts) - 2): raise - return '.'.join(parts[:i]) + return ".".join(parts[:i]) return dotted_name -def get_modules(package: str, src_directory: str, blacklist: Sequence[str] = STD_BLACKLIST) -> List[str]: +def get_modules( + package: str, src_directory: str, blacklist: Sequence[str] = STD_BLACKLIST +) -> List[str]: """given a package directory return a list of all available python modules in the package and its subpackages @@ -410,21 +422,20 @@ def get_modules(package: str, src_directory: str, blacklist: Sequence[str] = STD for directory, dirnames, filenames in os.walk(src_directory): _handle_blacklist(blacklist, dirnames, filenames) # check for __init__.py - if not '__init__.py' in filenames: + if not "__init__.py" in filenames: dirnames[:] = () continue if directory != src_directory: - dir_package = directory[len(src_directory):].replace(os.sep, '.') + dir_package = directory[len(src_directory) :].replace(os.sep, ".") modules.append(package + dir_package) for filename in filenames: - if _is_python_file(filename) and filename != '__init__.py': + if _is_python_file(filename) and filename != "__init__.py": src = join(directory, filename) - module = package + src[len(src_directory):-3] - modules.append(module.replace(os.sep, '.')) + module = package + src[len(src_directory) : -3] + modules.append(module.replace(os.sep, ".")) return modules - def get_module_files(src_directory: str, blacklist: Sequence[str] = STD_BLACKLIST) -> List[str]: """given a package directory return a list of all available python module's files in the package and its subpackages @@ -447,7 +458,7 @@ def get_module_files(src_directory: str, blacklist: Sequence[str] = STD_BLACKLIS for directory, dirnames, filenames in os.walk(src_directory): _handle_blacklist(blacklist, dirnames, filenames) # check for __init__.py - if not '__init__.py' in filenames: + if not "__init__.py" in filenames: dirnames[:] = () continue for filename in filenames: @@ -473,7 +484,7 @@ def get_source_file(filename: str, include_no_ext: bool = False) -> str: """ base, orig_ext = splitext(abspath(filename)) for ext in PY_SOURCE_EXTS: - source_path = '%s.%s' % (base, ext) + source_path = "%s.%s" % (base, ext) if exists(source_path): return source_path if include_no_ext and not orig_ext and exists(base): @@ -485,7 +496,7 @@ def cleanup_sys_modules(directories): """remove submodules of `directories` from `sys.modules`""" cleaned = [] for modname, module in list(sys.modules.items()): - modfile = getattr(module, '__file__', None) + modfile = getattr(module, "__file__", None) if modfile: for directory in directories: if modfile.startswith(directory): @@ -515,7 +526,9 @@ def is_python_source(filename): return splitext(filename)[1][1:] in PY_SOURCE_EXTS -def is_standard_module(modname: str, std_path: Union[List[str], Tuple[str]] = (STD_LIB_DIR,)) -> bool: +def is_standard_module( + modname: str, std_path: Union[List[str], Tuple[str]] = (STD_LIB_DIR,) +) -> bool: """try to guess if a module is a standard python module (by default, see `std_path` parameter's description) @@ -535,7 +548,7 @@ def is_standard_module(modname: str, std_path: Union[List[str], Tuple[str]] = (S Note: this function is known to return wrong values when inside virtualenv. See https://www.logilab.org/ticket/294756. """ - modname = modname.split('.')[0] + modname = modname.split(".")[0] try: filename = file_from_modpath([modname]) except ImportError as ex: @@ -556,7 +569,6 @@ def is_standard_module(modname: str, std_path: Union[List[str], Tuple[str]] = (S return False - def is_relative(modname: str, from_file: str) -> bool: """return true if the given module name is relative to the given file name @@ -577,7 +589,7 @@ def is_relative(modname: str, from_file: str) -> bool: if from_file in sys.path: return False try: - find_module(modname.split('.')[0], [from_file]) + find_module(modname.split(".")[0], [from_file]) return True except ImportError: return False @@ -585,7 +597,10 @@ def is_relative(modname: str, from_file: str) -> bool: # internal only functions ##################################################### -def _file_from_modpath(modpath: List[str], path: Optional[Any] = None, context: Optional[str] = None) -> Optional[str]: + +def _file_from_modpath( + modpath: List[str], path: Optional[Any] = None, context: Optional[str] = None +) -> Optional[str]: """given a mod path (i.e. splitted module / package name), return the corresponding file @@ -614,15 +629,20 @@ def _file_from_modpath(modpath: List[str], path: Optional[Any] = None, context: mp_filename = _has_init(mp_filename) return mp_filename -def _search_zip(modpath: List[str], pic: Dict[str, Optional[FileFinder]]) -> Tuple[object, str, str]: + +def _search_zip( + modpath: List[str], pic: Dict[str, Optional[FileFinder]] +) -> Tuple[object, str, str]: for filepath, importer in pic.items(): if importer is not None: if importer.find_module(modpath[0]): - if not importer.find_module('/'.join(modpath)): - raise ImportError('No module named %s in %s/%s' % ( - '.'.join(modpath[1:]), filepath, modpath)) - return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath - raise ImportError('No module named %s' % '.'.join(modpath)) + if not importer.find_module("/".join(modpath)): + raise ImportError( + "No module named %s in %s/%s" % (".".join(modpath[1:]), filepath, modpath) + ) + return ZIPFILE, abspath(filepath) + "/" + "/".join(modpath), filepath + raise ImportError("No module named %s" % ".".join(modpath)) + try: import pkg_resources @@ -635,11 +655,14 @@ except ImportError: def _is_namespace(modname: str) -> bool: # mypy: Module has no attribute "_namespace_packages"; maybe "fixup_namespace_packages"?" # but is still has? or is it a failure from python3 port? - return (pkg_resources is not None - and modname in pkg_resources._namespace_packages) # type: ignore + return ( + pkg_resources is not None and modname in pkg_resources._namespace_packages + ) # type: ignore -def _module_file(modpath: List[str], path: Optional[List[str]] = None) -> Tuple[Union[int, object], Optional[str]]: +def _module_file( + modpath: List[str], path: Optional[List[str]] = None +) -> Tuple[Union[int, object], Optional[str]]: """get a module type / file path :type modpath: list or tuple @@ -670,7 +693,7 @@ def _module_file(modpath: List[str], path: Optional[List[str]] = None) -> Tuple[ except AttributeError: checkeggs = False # pkg_resources support (aka setuptools namespace packages) - if (_is_namespace(modpath[0]) and modpath[0] in sys.modules): + if _is_namespace(modpath[0]) and modpath[0] in sys.modules: # setuptools has added into sys.modules a module object with proper # __path__, get back information from there module = sys.modules[modpath.pop(0)] @@ -720,31 +743,30 @@ def _module_file(modpath: List[str], path: Optional[List[str]] = None) -> Tuple[ mtype = mp_desc[2] if modpath: if mtype != PKG_DIRECTORY: - raise ImportError('No module %s in %s' % ('.'.join(modpath), - '.'.join(imported))) + raise ImportError("No module %s in %s" % (".".join(modpath), ".".join(imported))) # XXX guess if package is using pkgutil.extend_path by looking for # those keywords in the first four Kbytes try: - with open(join(mp_filename, '__init__.py')) as stream: + with open(join(mp_filename, "__init__.py")) as stream: data = stream.read(4096) except IOError: path = [mp_filename] else: - if 'pkgutil' in data and 'extend_path' in data: + if "pkgutil" in data and "extend_path" in data: # extend_path is called, search sys.path for module/packages # of this name see pkgutil.extend_path documentation - path = [join(p, *imported) for p in sys.path - if isdir(join(p, *imported))] + path = [join(p, *imported) for p in sys.path if isdir(join(p, *imported))] else: path = [mp_filename] return mtype, mp_filename + def _is_python_file(filename: str) -> bool: """return true if the given filename should be considered as a python file .pyc and .pyo are ignored """ - for ext in ('.py', '.so', '.pyd', '.pyw'): + for ext in (".py", ".so", ".pyd", ".pyw"): if filename.endswith(ext): return True return False @@ -754,10 +776,10 @@ def _has_init(directory: str) -> Optional[str]: """if the given directory has a valid __init__ file, return its path, else return None """ - mod_or_pack = join(directory, '__init__') + mod_or_pack = join(directory, "__init__") - for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'): - if exists(mod_or_pack + '.' + ext): - return mod_or_pack + '.' + ext + for ext in PY_SOURCE_EXTS + ("pyc", "pyo"): + if exists(mod_or_pack + "." + ext): + return mod_or_pack + "." + ext return None |