diff options
Diffstat (limited to 'Lib/importlib/_bootstrap.py')
| -rw-r--r-- | Lib/importlib/_bootstrap.py | 1747 |
1 files changed, 1203 insertions, 544 deletions
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index e40ec925ce..8a0cc34aba 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -9,7 +9,7 @@ work. One should use importlib as the public-facing version of this module. # # IMPORTANT: Whenever making changes to this module, be sure to run # a top-level make in order to get the frozen version of the module -# update. Not doing so, will result in the Makefile to fail for +# update. Not doing so will result in the Makefile to fail for # all others who don't have a ./python around to freeze the module # in the early stages of compilation. # @@ -20,10 +20,6 @@ work. One should use importlib as the public-facing version of this module. # reference any injected objects! This includes not only global code but also # anything specified at the class level. -# XXX Make sure all public names have no single leading underscore and all -# others do. - - # Bootstrap-related code ###################################################### _CASE_INSENSITIVE_PLATFORMS = 'win', 'cygwin', 'darwin' @@ -41,76 +37,58 @@ def _make_relax_case(): return _relax_case -# TODO: Expose from marshal def _w_long(x): - """Convert a 32-bit integer to little-endian. + """Convert a 32-bit integer to little-endian.""" + return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little') - XXX Temporary until marshal's long functions are exposed. - - """ - x = int(x) - int_bytes = [] - int_bytes.append(x & 0xFF) - int_bytes.append((x >> 8) & 0xFF) - int_bytes.append((x >> 16) & 0xFF) - int_bytes.append((x >> 24) & 0xFF) - return bytearray(int_bytes) - -# TODO: Expose from marshal def _r_long(int_bytes): - """Convert 4 bytes in little-endian to an integer. - - XXX Temporary until marshal's long function are exposed. - - """ - x = int_bytes[0] - x |= int_bytes[1] << 8 - x |= int_bytes[2] << 16 - x |= int_bytes[3] << 24 - return x + """Convert 4 bytes in little-endian to an integer.""" + return int.from_bytes(int_bytes, 'little') def _path_join(*path_parts): """Replacement for os.path.join().""" - new_parts = [] - for part in path_parts: - if not part: - continue - new_parts.append(part) - if part[-1] not in path_separators: - new_parts.append(path_sep) - return ''.join(new_parts[:-1]) # Drop superfluous path separator. + return path_sep.join([part.rstrip(path_separators) + for part in path_parts if part]) def _path_split(path): """Replacement for os.path.split().""" + if len(path_separators) == 1: + front, _, tail = path.rpartition(path_sep) + return front, tail for x in reversed(path): if x in path_separators: - sep = x - break - else: - sep = path_sep - front, _, tail = path.rpartition(sep) - return front, tail + front, tail = path.rsplit(x, maxsplit=1) + return front, tail + return '', path + + +def _path_stat(path): + """Stat the path. + + Made a separate function to make it easier to override in experiments + (e.g. cache stat results). + + """ + return _os.stat(path) def _path_is_mode_type(path, mode): """Test whether the path is the specified mode type.""" try: - stat_info = _os.stat(path) + stat_info = _path_stat(path) except OSError: return False return (stat_info.st_mode & 0o170000) == mode -# XXX Could also expose Modules/getpath.c:isfile() def _path_isfile(path): """Replacement for os.path.isfile.""" return _path_is_mode_type(path, 0o100000) -# XXX Could also expose Modules/getpath.c:isdir() def _path_isdir(path): """Replacement for os.path.isdir.""" if not path: @@ -148,17 +126,30 @@ def _wrap(new, old): new.__dict__.update(old.__dict__) +def _new_module(name): + return type(sys)(name) + + _code_type = type(_wrap.__code__) -def new_module(name): - """Create a new module. - The module is not entered into sys.modules. +class _ManageReload: - """ - return type(_io)(name) + """Manages the possible clean-up of sys.modules for load_module().""" + + def __init__(self, name): + self._name = name + def __enter__(self): + self._is_reload = self._name in sys.modules + + def __exit__(self, *args): + if any(arg is not None for arg in args) and not self._is_reload: + try: + del sys.modules[self._name] + except KeyError: + pass # Module-level locking ######################################################## @@ -214,7 +205,7 @@ class _ModuleLock: self.count += 1 return True if self.has_deadlock(): - raise _DeadlockError("deadlock detected by %r" % self) + raise _DeadlockError('deadlock detected by %r' % self) if self.wakeup.acquire(False): self.waiters += 1 # Wait for a release() call @@ -227,7 +218,7 @@ class _ModuleLock: tid = _thread.get_ident() with self.lock: if self.owner != tid: - raise RuntimeError("cannot release un-acquired lock") + raise RuntimeError('cannot release un-acquired lock') assert self.count > 0 self.count -= 1 if self.count == 0: @@ -237,7 +228,7 @@ class _ModuleLock: self.wakeup.release() def __repr__(self): - return "_ModuleLock(%r) at %d" % (self.name, id(self)) + return '_ModuleLock({!r}) at {}'.format(self.name, id(self)) class _DummyModuleLock: @@ -254,11 +245,28 @@ class _DummyModuleLock: def release(self): if self.count == 0: - raise RuntimeError("cannot release un-acquired lock") + raise RuntimeError('cannot release un-acquired lock') self.count -= 1 def __repr__(self): - return "_DummyModuleLock(%r) at %d" % (self.name, id(self)) + return '_DummyModuleLock({!r}) at {}'.format(self.name, id(self)) + + +class _ModuleLockManager: + + def __init__(self, name): + self._name = name + self._lock = None + + def __enter__(self): + try: + self._lock = _get_module_lock(self._name) + finally: + _imp.release_lock() + self._lock.acquire() + + def __exit__(self, *args, **kwargs): + self._lock.release() # The following two functions are for consumption by Python/import.c. @@ -315,95 +323,108 @@ def _call_with_frames_removed(f, *args, **kwds): # Finder/loader utility code ############################################### -"""Magic word to reject .pyc files generated by other Python versions. -It should change for each incompatible change to the bytecode. - -The value of CR and LF is incorporated so if you ever read or write -a .pyc file in text mode the magic number will be wrong; also, the -Apple MPW compiler swaps their values, botching string constants. - -The magic numbers must be spaced apart at least 2 values, as the --U interpeter flag will cause MAGIC+1 being used. They have been -odd numbers for some time now. - -There were a variety of old schemes for setting the magic number. -The current working scheme is to increment the previous value by -10. - -Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic -number also includes a new "magic tag", i.e. a human readable string used -to represent the magic number in __pycache__ directories. When you change -the magic number, you must also set a new unique magic tag. Generally this -can be named after the Python major version of the magic number bump, but -it can really be anything, as long as it's different than anything else -that's come before. The tags are included in the following table, starting -with Python 3.2a0. - -Known values: - Python 1.5: 20121 - Python 1.5.1: 20121 - Python 1.5.2: 20121 - Python 1.6: 50428 - Python 2.0: 50823 - Python 2.0.1: 50823 - Python 2.1: 60202 - Python 2.1.1: 60202 - Python 2.1.2: 60202 - Python 2.2: 60717 - Python 2.3a0: 62011 - Python 2.3a0: 62021 - Python 2.3a0: 62011 (!) - Python 2.4a0: 62041 - Python 2.4a3: 62051 - Python 2.4b1: 62061 - Python 2.5a0: 62071 - Python 2.5a0: 62081 (ast-branch) - Python 2.5a0: 62091 (with) - Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) - Python 2.5b3: 62101 (fix wrong code: for x, in ...) - Python 2.5b3: 62111 (fix wrong code: x += yield) - Python 2.5c1: 62121 (fix wrong lnotab with for loops and - storing constants that should have been removed) - Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) - Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) - Python 2.6a1: 62161 (WITH_CLEANUP optimization) - Python 3000: 3000 - 3010 (removed UNARY_CONVERT) - 3020 (added BUILD_SET) - 3030 (added keyword-only parameters) - 3040 (added signature annotations) - 3050 (print becomes a function) - 3060 (PEP 3115 metaclass syntax) - 3061 (string literals become unicode) - 3071 (PEP 3109 raise changes) - 3081 (PEP 3137 make __file__ and __name__ unicode) - 3091 (kill str8 interning) - 3101 (merge from 2.6a0, see 62151) - 3103 (__file__ points to source file) - Python 3.0a4: 3111 (WITH_CLEANUP optimization). - Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT) - Python 3.1a0: 3141 (optimize list, set and dict comprehensions: - change LIST_APPEND and SET_ADD, add MAP_ADD) - Python 3.1a0: 3151 (optimize conditional branches: - introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) - Python 3.2a0: 3160 (add SETUP_WITH) - tag: cpython-32 - Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) - tag: cpython-32 - Python 3.2a2 3180 (add DELETE_DEREF) - Python 3.3a0 3190 __class__ super closure changed - Python 3.3a0 3200 (__qualname__ added) - 3210 (added size modulo 2**32 to the pyc header) - Python 3.3a1 3220 (changed PEP 380 implementation) - Python 3.3a4 3230 (revert changes to implicit __class__ closure) - -MAGIC must change whenever the bytecode emitted by the compiler may no -longer be understood by older implementations of the eval loop (usually -due to the addition of new opcodes). +# Magic word to reject .pyc files generated by other Python versions. +# It should change for each incompatible change to the bytecode. +# +# The value of CR and LF is incorporated so if you ever read or write +# a .pyc file in text mode the magic number will be wrong; also, the +# Apple MPW compiler swaps their values, botching string constants. +# +# The magic numbers must be spaced apart at least 2 values, as the +# -U interpeter flag will cause MAGIC+1 being used. They have been +# odd numbers for some time now. +# +# There were a variety of old schemes for setting the magic number. +# The current working scheme is to increment the previous value by +# 10. +# +# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic +# number also includes a new "magic tag", i.e. a human readable string used +# to represent the magic number in __pycache__ directories. When you change +# the magic number, you must also set a new unique magic tag. Generally this +# can be named after the Python major version of the magic number bump, but +# it can really be anything, as long as it's different than anything else +# that's come before. The tags are included in the following table, starting +# with Python 3.2a0. +# +# Known values: +# Python 1.5: 20121 +# Python 1.5.1: 20121 +# Python 1.5.2: 20121 +# Python 1.6: 50428 +# Python 2.0: 50823 +# Python 2.0.1: 50823 +# Python 2.1: 60202 +# Python 2.1.1: 60202 +# Python 2.1.2: 60202 +# Python 2.2: 60717 +# Python 2.3a0: 62011 +# Python 2.3a0: 62021 +# Python 2.3a0: 62011 (!) +# Python 2.4a0: 62041 +# Python 2.4a3: 62051 +# Python 2.4b1: 62061 +# Python 2.5a0: 62071 +# Python 2.5a0: 62081 (ast-branch) +# Python 2.5a0: 62091 (with) +# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) +# Python 2.5b3: 62101 (fix wrong code: for x, in ...) +# Python 2.5b3: 62111 (fix wrong code: x += yield) +# Python 2.5c1: 62121 (fix wrong lnotab with for loops and +# storing constants that should have been removed) +# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) +# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) +# Python 2.6a1: 62161 (WITH_CLEANUP optimization) +# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) +# Python 2.7a0: 62181 (optimize conditional branches: +# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) +# Python 2.7a0 62191 (introduce SETUP_WITH) +# Python 2.7a0 62201 (introduce BUILD_SET) +# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) +# Python 3000: 3000 +# 3010 (removed UNARY_CONVERT) +# 3020 (added BUILD_SET) +# 3030 (added keyword-only parameters) +# 3040 (added signature annotations) +# 3050 (print becomes a function) +# 3060 (PEP 3115 metaclass syntax) +# 3061 (string literals become unicode) +# 3071 (PEP 3109 raise changes) +# 3081 (PEP 3137 make __file__ and __name__ unicode) +# 3091 (kill str8 interning) +# 3101 (merge from 2.6a0, see 62151) +# 3103 (__file__ points to source file) +# Python 3.0a4: 3111 (WITH_CLEANUP optimization). +# Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT) +# Python 3.1a0: 3141 (optimize list, set and dict comprehensions: +# change LIST_APPEND and SET_ADD, add MAP_ADD) +# Python 3.1a0: 3151 (optimize conditional branches: +# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) +# Python 3.2a0: 3160 (add SETUP_WITH) +# tag: cpython-32 +# Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) +# tag: cpython-32 +# Python 3.2a2 3180 (add DELETE_DEREF) +# Python 3.3a0 3190 __class__ super closure changed +# Python 3.3a0 3200 (__qualname__ added) +# 3210 (added size modulo 2**32 to the pyc header) +# Python 3.3a1 3220 (changed PEP 380 implementation) +# Python 3.3a4 3230 (revert changes to implicit __class__ closure) +# Python 3.4a1 3250 (evaluate positional default arguments before +# keyword-only defaults) +# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override +# free vars) +# Python 3.4a1 3270 (various tweaks to the __class__ closure) +# Python 3.4a1 3280 (remove implicit class argument) +# Python 3.4a4 3290 (changes to __qualname__ computation) +# Python 3.4a4 3300 (more changes to __qualname__ computation) +# +# MAGIC must change whenever the bytecode emitted by the compiler may no +# longer be understood by older implementations of the eval loop (usually +# due to the addition of new opcodes). -""" -_RAW_MAGIC_NUMBER = 3230 | ord('\r') << 16 | ord('\n') << 24 -_MAGIC_BYTES = bytes(_RAW_MAGIC_NUMBER >> n & 0xff for n in range(0, 25, 8)) +MAGIC_NUMBER = (3300).to_bytes(2, 'little') + b'\r\n' +_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' @@ -481,6 +502,18 @@ def _get_sourcefile(bytecode_path): return source_path if _path_isfile(source_path) else bytecode_path +def _calc_mode(path): + """Calculate the mode permissions for a bytecode file.""" + try: + mode = _path_stat(path).st_mode + except OSError: + mode = 0o666 + # We always ensure write access so we can update cached files + # later even when the source files are read-only on Windows (#6074) + mode |= 0o200 + return mode + + def _verbose_message(message, *args, verbosity=1): """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" if sys.flags.verbose >= verbosity: @@ -489,85 +522,6 @@ def _verbose_message(message, *args, verbosity=1): print(message.format(*args), file=sys.stderr) -def set_package(fxn): - """Set __package__ on the returned module.""" - def set_package_wrapper(*args, **kwargs): - module = fxn(*args, **kwargs) - if getattr(module, '__package__', None) is None: - module.__package__ = module.__name__ - if not hasattr(module, '__path__'): - module.__package__ = module.__package__.rpartition('.')[0] - return module - _wrap(set_package_wrapper, fxn) - return set_package_wrapper - - -def set_loader(fxn): - """Set __loader__ on the returned module.""" - def set_loader_wrapper(self, *args, **kwargs): - module = fxn(self, *args, **kwargs) - if not hasattr(module, '__loader__'): - module.__loader__ = self - return module - _wrap(set_loader_wrapper, fxn) - return set_loader_wrapper - - -def module_for_loader(fxn): - """Decorator to handle selecting the proper module for loaders. - - The decorated function is passed the module to use instead of the module - name. The module passed in to the function is either from sys.modules if - it already exists or is a new module. If the module is new, then __name__ - is set the first argument to the method, __loader__ is set to self, and - __package__ is set accordingly (if self.is_package() is defined) will be set - before it is passed to the decorated function (if self.is_package() does - not work for the module it will be set post-load). - - If an exception is raised and the decorator created the module it is - subsequently removed from sys.modules. - - The decorator assumes that the decorated function takes the module name as - the second argument. - - """ - def module_for_loader_wrapper(self, fullname, *args, **kwargs): - module = sys.modules.get(fullname) - is_reload = module is not None - if not is_reload: - # This must be done before open() is called as the 'io' module - # implicitly imports 'locale' and would otherwise trigger an - # infinite loop. - module = new_module(fullname) - # This must be done before putting the module in sys.modules - # (otherwise an optimization shortcut in import.c becomes wrong) - module.__initializing__ = True - sys.modules[fullname] = module - module.__loader__ = self - try: - is_package = self.is_package(fullname) - except (ImportError, AttributeError): - pass - else: - if is_package: - module.__package__ = fullname - else: - module.__package__ = fullname.rpartition('.')[0] - else: - module.__initializing__ = True - try: - # If __package__ was not set above, __import__() will do it later. - return fxn(self, module, *args, **kwargs) - except: - if not is_reload: - del sys.modules[fullname] - raise - finally: - module.__initializing__ = False - _wrap(module_for_loader_wrapper, fxn) - return module_for_loader_wrapper - - def _check_name(method): """Decorator to verify that the module being requested matches the one the loader can handle. @@ -580,7 +534,7 @@ def _check_name(method): if name is None: name = self.name elif self.name != name: - raise ImportError("loader cannot handle %s" % name, name=name) + raise ImportError('loader cannot handle %s' % name, name=name) return method(self, name, *args, **kwargs) _wrap(_check_name_wrapper, method) return _check_name_wrapper @@ -590,7 +544,7 @@ def _requires_builtin(fxn): """Decorator to verify the named module is built-in.""" def _requires_builtin_wrapper(self, fullname): if fullname not in sys.builtin_module_names: - raise ImportError("{} is not a built-in module".format(fullname), + raise ImportError('{!r} is not a built-in module'.format(fullname), name=fullname) return fxn(self, fullname) _wrap(_requires_builtin_wrapper, fxn) @@ -601,7 +555,7 @@ def _requires_frozen(fxn): """Decorator to verify the named module is frozen.""" def _requires_frozen_wrapper(self, fullname): if not _imp.is_frozen(fullname): - raise ImportError("{} is not a frozen module".format(fullname), + raise ImportError('{!r} is not a frozen module'.format(fullname), name=fullname) return fxn(self, fullname) _wrap(_requires_frozen_wrapper, fxn) @@ -610,17 +564,659 @@ def _requires_frozen(fxn): def _find_module_shim(self, fullname): """Try to find a loader for the specified module by delegating to - self.find_loader().""" + self.find_loader(). + + This method is deprecated in favor of finder.find_spec(). + + """ # Call find_loader(). If it returns a string (indicating this # is a namespace package portion), generate a warning and # return None. loader, portions = self.find_loader(fullname) if loader is None and len(portions): - msg = "Not importing directory {}: missing __init__" + msg = 'Not importing directory {}: missing __init__' _warnings.warn(msg.format(portions[0]), ImportWarning) return loader +def _load_module_shim(self, fullname): + """Load the specified module into sys.modules and return it. + + This method is deprecated. Use loader.exec_module instead. + + """ + spec = spec_from_loader(fullname, self) + methods = _SpecMethods(spec) + if fullname in sys.modules: + module = sys.modules[fullname] + methods.exec(module) + return sys.modules[fullname] + else: + return methods.load() + + +def _validate_bytecode_header(data, source_stats=None, name=None, path=None): + """Validate the header of the passed-in bytecode against source_stats (if + given) and returning the bytecode that can be compiled by compile(). + + All other arguments are used to enhance error reporting. + + ImportError is raised when the magic number is incorrect or the bytecode is + found to be stale. EOFError is raised when the data is found to be + truncated. + + """ + exc_details = {} + if name is not None: + exc_details['name'] = name + else: + # To prevent having to make all messages have a conditional name. + name = '<bytecode>' + if path is not None: + exc_details['path'] = path + magic = data[:4] + raw_timestamp = data[4:8] + raw_size = data[8:12] + if magic != MAGIC_NUMBER: + message = 'bad magic number in {!r}: {!r}'.format(name, magic) + _verbose_message(message) + raise ImportError(message, **exc_details) + elif len(raw_timestamp) != 4: + message = 'reached EOF while reading timestamp in {!r}'.format(name) + _verbose_message(message) + raise EOFError(message) + elif len(raw_size) != 4: + message = 'reached EOF while reading size of source in {!r}'.format(name) + _verbose_message(message) + raise EOFError(message) + if source_stats is not None: + try: + source_mtime = int(source_stats['mtime']) + except KeyError: + pass + else: + if _r_long(raw_timestamp) != source_mtime: + message = 'bytecode is stale for {!r}'.format(name) + _verbose_message(message) + raise ImportError(message, **exc_details) + try: + source_size = source_stats['size'] & 0xFFFFFFFF + except KeyError: + pass + else: + if _r_long(raw_size) != source_size: + raise ImportError('bytecode is stale for {!r}'.format(name), + **exc_details) + return data[12:] + + +def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None): + """Compile bytecode as returned by _validate_bytecode_header().""" + code = marshal.loads(data) + if isinstance(code, _code_type): + _verbose_message('code object from {!r}', bytecode_path) + if source_path is not None: + _imp._fix_co_filename(code, source_path) + return code + else: + raise ImportError('Non-code object in {!r}'.format(bytecode_path), + name=name, path=bytecode_path) + +def _code_to_bytecode(code, mtime=0, source_size=0): + """Compile a code object into bytecode for writing out to a byte-compiled + file.""" + data = bytearray(MAGIC_NUMBER) + data.extend(_w_long(mtime)) + data.extend(_w_long(source_size)) + data.extend(marshal.dumps(code)) + return data + + +def decode_source(source_bytes): + """Decode bytes representing source code and return the string. + + Universal newline support is used in the decoding. + """ + import tokenize # To avoid bootstrap issues. + source_bytes_readline = _io.BytesIO(source_bytes).readline + encoding = tokenize.detect_encoding(source_bytes_readline) + newline_decoder = _io.IncrementalNewlineDecoder(None, True) + return newline_decoder.decode(source_bytes.decode(encoding[0])) + + +# Module specifications ####################################################### + +def _module_repr(module): + # The implementation of ModuleType__repr__(). + loader = getattr(module, '__loader__', None) + if hasattr(loader, 'module_repr'): + # As soon as BuiltinImporter, FrozenImporter, and NamespaceLoader + # drop their implementations for module_repr. we can add a + # deprecation warning here. + try: + return loader.module_repr(module) + except Exception: + pass + try: + spec = module.__spec__ + except AttributeError: + pass + else: + if spec is not None: + return _SpecMethods(spec).module_repr() + + # We could use module.__class__.__name__ instead of 'module' in the + # various repr permutations. + try: + name = module.__name__ + except AttributeError: + name = '?' + try: + filename = module.__file__ + except AttributeError: + if loader is None: + return '<module {!r}>'.format(name) + else: + return '<module {!r} ({!r})>'.format(name, loader) + else: + return '<module {!r} from {!r}>'.format(name, filename) + + +class _installed_safely: + + def __init__(self, module): + self._module = module + self._spec = module.__spec__ + + def __enter__(self): + # This must be done before putting the module in sys.modules + # (otherwise an optimization shortcut in import.c becomes + # wrong) + self._spec._initializing = True + sys.modules[self._spec.name] = self._module + + def __exit__(self, *args): + try: + spec = self._spec + if any(arg is not None for arg in args): + try: + del sys.modules[spec.name] + except KeyError: + pass + else: + _verbose_message('import {!r} # {!r}', spec.name, spec.loader) + finally: + self._spec._initializing = False + + +class ModuleSpec: + """The specification for a module, used for loading. + + A module's spec is the source for information about the module. For + data associated with the module, including source, use the spec's + loader. + + `name` is the absolute name of the module. `loader` is the loader + to use when loading the module. `parent` is the name of the + package the module is in. The parent is derived from the name. + + `is_package` determines if the module is considered a package or + not. On modules this is reflected by the `__path__` attribute. + + `origin` is the specific location used by the loader from which to + load the module, if that information is available. When filename is + set, origin will match. + + `has_location` indicates that a spec's "origin" reflects a location. + When this is True, `__file__` attribute of the module is set. + + `cached` is the location of the cached bytecode file, if any. It + corresponds to the `__cached__` attribute. + + `submodule_search_locations` is the sequence of path entries to + search when importing submodules. If set, is_package should be + True--and False otherwise. + + Packages are simply modules that (may) have submodules. If a spec + has a non-None value in `submodule_search_locations`, the import + system will consider modules loaded from the spec as packages. + + Only finders (see importlib.abc.MetaPathFinder and + importlib.abc.PathEntryFinder) should modify ModuleSpec instances. + + """ + + def __init__(self, name, loader, *, origin=None, loader_state=None, + is_package=None): + self.name = name + self.loader = loader + self.origin = origin + self.loader_state = loader_state + self.submodule_search_locations = [] if is_package else None + + # file-location attributes + self._set_fileattr = False + self._cached = None + + def __repr__(self): + args = ['name={!r}'.format(self.name), + 'loader={!r}'.format(self.loader)] + if self.origin is not None: + args.append('origin={!r}'.format(self.origin)) + if self.submodule_search_locations is not None: + args.append('submodule_search_locations={}' + .format(self.submodule_search_locations)) + return '{}({})'.format(self.__class__.__name__, ', '.join(args)) + + def __eq__(self, other): + smsl = self.submodule_search_locations + try: + return (self.name == other.name and + self.loader == other.loader and + self.origin == other.origin and + smsl == other.submodule_search_locations and + self.cached == other.cached and + self.has_location == other.has_location) + except AttributeError: + return False + + @property + def cached(self): + if self._cached is None: + if self.origin is not None and self._set_fileattr: + filename = self.origin + if filename.endswith(tuple(SOURCE_SUFFIXES)): + try: + self._cached = cache_from_source(filename) + except NotImplementedError: + pass + elif filename.endswith(tuple(BYTECODE_SUFFIXES)): + self._cached = filename + return self._cached + + @cached.setter + def cached(self, cached): + self._cached = cached + + @property + def parent(self): + """The name of the module's parent.""" + if self.submodule_search_locations is None: + return self.name.rpartition('.')[0] + else: + return self.name + + @property + def has_location(self): + return self._set_fileattr + + @has_location.setter + def has_location(self, value): + self._set_fileattr = bool(value) + + +def spec_from_loader(name, loader, *, origin=None, is_package=None): + """Return a module spec based on various loader methods.""" + if hasattr(loader, 'get_filename'): + if is_package is None: + return spec_from_file_location(name, loader=loader) + search = [] if is_package else None + return spec_from_file_location(name, loader=loader, + submodule_search_locations=search) + + if is_package is None: + if hasattr(loader, 'is_package'): + try: + is_package = loader.is_package(name) + except ImportError: + is_package = None # aka, undefined + else: + # the default + is_package = False + + return ModuleSpec(name, loader, origin=origin, is_package=is_package) + + +_POPULATE = object() + + +def spec_from_file_location(name, location=None, *, loader=None, + submodule_search_locations=_POPULATE): + """Return a module spec based on a file location. + + To indicate that the module is a package, set + submodule_search_locations to a list of directory paths. An + empty list is sufficient, though its not otherwise useful to the + import system. + + The loader must take a spec as its only __init__() arg. + + """ + if location is None: + # The caller may simply want a partially populated location- + # oriented spec. So we set the location to a bogus value and + # fill in as much as we can. + location = '<unknown>' + if hasattr(loader, 'get_filename'): + # ExecutionLoader + try: + location = loader.get_filename(name) + except ImportError: + pass + + # If the location is on the filesystem, but doesn't actually exist, + # we could return None here, indicating that the location is not + # valid. However, we don't have a good way of testing since an + # indirect location (e.g. a zip file or URL) will look like a + # non-existent file relative to the filesystem. + + spec = ModuleSpec(name, loader, origin=location) + spec._set_fileattr = True + + # Pick a loader if one wasn't provided. + if loader is None: + for loader_class, suffixes in _get_supported_file_loaders(): + if location.endswith(tuple(suffixes)): + loader = loader_class(name, location) + spec.loader = loader + break + else: + return None + + # Set submodule_search_paths appropriately. + if submodule_search_locations is _POPULATE: + # Check the loader. + if hasattr(loader, 'is_package'): + try: + is_package = loader.is_package(name) + except ImportError: + pass + else: + if is_package: + spec.submodule_search_locations = [] + else: + spec.submodule_search_locations = submodule_search_locations + if spec.submodule_search_locations == []: + if location: + dirname = _path_split(location)[0] + spec.submodule_search_locations.append(dirname) + + return spec + + +def _spec_from_module(module, loader=None, origin=None): + # This function is meant for use in _setup(). + try: + spec = module.__spec__ + except AttributeError: + pass + else: + if spec is not None: + return spec + + name = module.__name__ + if loader is None: + try: + loader = module.__loader__ + except AttributeError: + # loader will stay None. + pass + try: + location = module.__file__ + except AttributeError: + location = None + if origin is None: + if location is None: + try: + origin = loader._ORIGIN + except AttributeError: + origin = None + else: + origin = location + try: + cached = module.__cached__ + except AttributeError: + cached = None + try: + submodule_search_locations = list(module.__path__) + except AttributeError: + submodule_search_locations = None + + spec = ModuleSpec(name, loader, origin=origin) + spec._set_fileattr = False if location is None else True + spec.cached = cached + spec.submodule_search_locations = submodule_search_locations + return spec + + +class _SpecMethods: + + """Convenience wrapper around spec objects to provide spec-specific + methods.""" + + # The various spec_from_* functions could be made factory methods here. + + def __init__(self, spec): + self.spec = spec + + def module_repr(self): + """Return the repr to use for the module.""" + # We mostly replicate _module_repr() using the spec attributes. + spec = self.spec + name = '?' if spec.name is None else spec.name + if spec.origin is None: + if spec.loader is None: + return '<module {!r}>'.format(name) + else: + return '<module {!r} ({!r})>'.format(name, spec.loader) + else: + if spec.has_location: + return '<module {!r} from {!r}>'.format(name, spec.origin) + else: + return '<module {!r} ({})>'.format(spec.name, spec.origin) + + def init_module_attrs(self, module, *, _override=False, _force_name=True): + """Set the module's attributes. + + All missing import-related module attributes will be set. Here + is how the spec attributes map onto the module: + + spec.name -> module.__name__ + spec.loader -> module.__loader__ + spec.parent -> module.__package__ + spec -> module.__spec__ + + Optional: + spec.origin -> module.__file__ (if spec.set_fileattr is true) + spec.cached -> module.__cached__ (if __file__ also set) + spec.submodule_search_locations -> module.__path__ (if set) + + """ + spec = self.spec + + # The passed in module may be not support attribute assignment, + # in which case we simply don't set the attributes. + + # __name__ + if (_override or _force_name or + getattr(module, '__name__', None) is None): + try: + module.__name__ = spec.name + except AttributeError: + pass + + # __loader__ + if _override or getattr(module, '__loader__', None) is None: + loader = spec.loader + if loader is None: + # A backward compatibility hack. + if spec.submodule_search_locations is not None: + loader = _NamespaceLoader.__new__(_NamespaceLoader) + loader._path = spec.submodule_search_locations + try: + module.__loader__ = loader + except AttributeError: + pass + + # __package__ + if _override or getattr(module, '__package__', None) is None: + try: + module.__package__ = spec.parent + except AttributeError: + pass + + # __spec__ + try: + module.__spec__ = spec + except AttributeError: + pass + + # __path__ + if _override or getattr(module, '__path__', None) is None: + if spec.submodule_search_locations is not None: + try: + module.__path__ = spec.submodule_search_locations + except AttributeError: + pass + + if spec.has_location: + # __file__ + if _override or getattr(module, '__file__', None) is None: + try: + module.__file__ = spec.origin + except AttributeError: + pass + + # __cached__ + if _override or getattr(module, '__cached__', None) is None: + if spec.cached is not None: + try: + module.__cached__ = spec.cached + except AttributeError: + pass + + def create(self): + """Return a new module to be loaded. + + The import-related module attributes are also set with the + appropriate values from the spec. + + """ + spec = self.spec + # Typically loaders will not implement create_module(). + if hasattr(spec.loader, 'create_module'): + # If create_module() returns `None` it means the default + # module creation should be used. + module = spec.loader.create_module(spec) + else: + module = None + if module is None: + # This must be done before open() is ever called as the 'io' + # module implicitly imports 'locale' and would otherwise + # trigger an infinite loop. + module = _new_module(spec.name) + self.init_module_attrs(module) + return module + + def _exec(self, module): + """Do everything necessary to execute the module. + + The namespace of `module` is used as the target of execution. + This method uses the loader's `exec_module()` method. + + """ + self.spec.loader.exec_module(module) + + # Used by importlib.reload() and _load_module_shim(). + def exec(self, module): + """Execute the spec in an existing module's namespace.""" + name = self.spec.name + _imp.acquire_lock() + with _ModuleLockManager(name): + if sys.modules.get(name) is not module: + msg = 'module {!r} not in sys.modules'.format(name) + raise ImportError(msg, name=name) + if self.spec.loader is None: + if self.spec.submodule_search_locations is None: + raise ImportError('missing loader', name=self.spec.name) + # namespace package + self.init_module_attrs(module, _override=True) + return module + self.init_module_attrs(module, _override=True) + if not hasattr(self.spec.loader, 'exec_module'): + # (issue19713) Once BuiltinImporter and ExtensionFileLoader + # have exec_module() implemented, we can add a deprecation + # warning here. + self.spec.loader.load_module(name) + else: + self._exec(module) + return sys.modules[name] + + def _load_backward_compatible(self): + # (issue19713) Once BuiltinImporter and ExtensionFileLoader + # have exec_module() implemented, we can add a deprecation + # warning here. + spec = self.spec + spec.loader.load_module(spec.name) + # The module must be in sys.modules at this point! + module = sys.modules[spec.name] + if getattr(module, '__loader__', None) is None: + try: + module.__loader__ = spec.loader + except AttributeError: + pass + if getattr(module, '__package__', None) is None: + try: + # Since module.__path__ may not line up with + # spec.submodule_search_paths, we can't necessarily rely + # on spec.parent here. + module.__package__ = module.__name__ + if not hasattr(module, '__path__'): + module.__package__ = spec.name.rpartition('.')[0] + except AttributeError: + pass + if getattr(module, '__spec__', None) is None: + try: + module.__spec__ = spec + except AttributeError: + pass + return module + + def _load_unlocked(self): + # A helper for direct use by the import system. + if self.spec.loader is not None: + # not a namespace package + if not hasattr(self.spec.loader, 'exec_module'): + return self._load_backward_compatible() + + module = self.create() + with _installed_safely(module): + if self.spec.loader is None: + if self.spec.submodule_search_locations is None: + raise ImportError('missing loader', name=self.spec.name) + # A namespace package so do nothing. + else: + self._exec(module) + + # We don't ensure that the import-related module attributes get + # set in the sys.modules replacement case. Such modules are on + # their own. + return sys.modules[self.spec.name] + + # A method used during testing of _load_unlocked() and by + # _load_module_shim(). + def load(self): + """Return a new module object, loaded by the spec's loader. + + The module is not added to its parent. + + If a module is already in sys.modules, that existing module gets + clobbered. + + """ + _imp.acquire_lock() + with _ModuleLockManager(self.spec.name): + return self._load_unlocked() # Loaders ##################################################################### @@ -634,9 +1230,23 @@ class BuiltinImporter: """ + @staticmethod + def module_repr(module): + """Return repr for the module. + + The method is deprecated. The import machinery does the job itself. + + """ + return '<module {!r} (built-in)>'.format(module.__name__) + @classmethod - def module_repr(cls, module): - return "<module '{}' (built-in)>".format(module.__name__) + def find_spec(cls, fullname, path=None, target=None): + if path is not None: + return None + if _imp.is_builtin(fullname): + return spec_from_loader(fullname, cls, origin='built-in') + else: + return None @classmethod def find_module(cls, fullname, path=None): @@ -644,24 +1254,23 @@ class BuiltinImporter: If 'path' is ever specified then the search is considered a failure. + This method is deprecated. Use find_spec() instead. + """ - if path is not None: - return None - return cls if _imp.is_builtin(fullname) else None + spec = cls.find_spec(fullname, path) + return spec.loader if spec is not None else None @classmethod - @set_package - @set_loader @_requires_builtin def load_module(cls, fullname): """Load a built-in module.""" - is_reload = fullname in sys.modules - try: - return _call_with_frames_removed(_imp.init_builtin, fullname) - except: - if not is_reload and fullname in sys.modules: - del sys.modules[fullname] - raise + # Once an exec_module() implementation is added we can also + # add a deprecation warning here. + with _ManageReload(fullname): + module = _call_with_frames_removed(_imp.init_builtin, fullname) + module.__loader__ = cls + module.__package__ = '' + return module @classmethod @_requires_builtin @@ -691,31 +1300,48 @@ class FrozenImporter: """ + @staticmethod + def module_repr(m): + """Return repr for the module. + + The method is deprecated. The import machinery does the job itself. + + """ + return '<module {!r} (frozen)>'.format(m.__name__) + @classmethod - def module_repr(cls, m): - return "<module '{}' (frozen)>".format(m.__name__) + def find_spec(cls, fullname, path=None, target=None): + if _imp.is_frozen(fullname): + return spec_from_loader(fullname, cls, origin='frozen') + else: + return None @classmethod def find_module(cls, fullname, path=None): - """Find a frozen module.""" + """Find a frozen module. + + This method is deprecated. Use find_spec() instead. + + """ return cls if _imp.is_frozen(fullname) else None + @staticmethod + def exec_module(module): + name = module.__spec__.name + if not _imp.is_frozen(name): + raise ImportError('{!r} is not a frozen module'.format(name), + name=name) + code = _call_with_frames_removed(_imp.get_frozen_object, name) + exec(code, module.__dict__) + @classmethod - @set_package - @set_loader - @_requires_frozen def load_module(cls, fullname): - """Load a frozen module.""" - is_reload = fullname in sys.modules - try: - m = _call_with_frames_removed(_imp.init_frozen, fullname) - # Let our own module_repr() method produce a suitable repr. - del m.__file__ - return m - except: - if not is_reload and fullname in sys.modules: - del sys.modules[fullname] - raise + """Load a frozen module. + + This method is deprecated. Use exec_module() instead. + + """ + return _load_module_shim(cls, fullname) @classmethod @_requires_frozen @@ -738,22 +1364,21 @@ class FrozenImporter: class WindowsRegistryFinder: - """Meta path finder for modules declared in the Windows registry. - """ + """Meta path finder for modules declared in the Windows registry.""" REGISTRY_KEY = ( - "Software\\Python\\PythonCore\\{sys_version}" - "\\Modules\\{fullname}") + 'Software\\Python\\PythonCore\\{sys_version}' + '\\Modules\\{fullname}') REGISTRY_KEY_DEBUG = ( - "Software\\Python\\PythonCore\\{sys_version}" - "\\Modules\\{fullname}\\Debug") + 'Software\\Python\\PythonCore\\{sys_version}' + '\\Modules\\{fullname}\\Debug') DEBUG_BUILD = False # Changed in _setup() @classmethod def _open_registry(cls, key): try: return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key) - except WindowsError: + except OSError: return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key) @classmethod @@ -766,24 +1391,38 @@ class WindowsRegistryFinder: sys_version=sys.version[:3]) try: with cls._open_registry(key) as hkey: - filepath = _winreg.QueryValue(hkey, "") - except WindowsError: + filepath = _winreg.QueryValue(hkey, '') + except OSError: return None return filepath @classmethod - def find_module(cls, fullname, path=None): - """Find module named in the registry.""" + def find_spec(cls, fullname, path=None, target=None): filepath = cls._search_registry(fullname) if filepath is None: return None try: - _os.stat(filepath) + _path_stat(filepath) except OSError: return None for loader, suffixes in _get_supported_file_loaders(): if filepath.endswith(tuple(suffixes)): - return loader(fullname, filepath) + spec = spec_from_loader(fullname, loader(fullname, filepath), + origin=filepath) + return spec + + @classmethod + def find_module(cls, fullname, path=None): + """Find module named in the registry. + + This method is deprecated. Use exec_module() instead. + + """ + spec = cls.find_spec(fullname, path) + if spec is not None: + return spec.loader + else: + return None class _LoaderBasics: @@ -799,74 +1438,15 @@ class _LoaderBasics: tail_name = fullname.rpartition('.')[2] return filename_base == '__init__' and tail_name != '__init__' - def _bytes_from_bytecode(self, fullname, data, bytecode_path, source_stats): - """Return the marshalled bytes from bytecode, verifying the magic - number, timestamp and source size along the way. + def exec_module(self, module): + """Execute the module.""" + code = self.get_code(module.__name__) + if code is None: + raise ImportError('cannot load module {!r} when get_code() ' + 'returns None'.format(module.__name__)) + _call_with_frames_removed(exec, code, module.__dict__) - If source_stats is None then skip the timestamp check. - - """ - magic = data[:4] - raw_timestamp = data[4:8] - raw_size = data[8:12] - if magic != _MAGIC_BYTES: - msg = 'bad magic number in {!r}: {!r}'.format(fullname, magic) - _verbose_message(msg) - raise ImportError(msg, name=fullname, path=bytecode_path) - elif len(raw_timestamp) != 4: - message = 'bad timestamp in {}'.format(fullname) - _verbose_message(message) - raise EOFError(message) - elif len(raw_size) != 4: - message = 'bad size in {}'.format(fullname) - _verbose_message(message) - raise EOFError(message) - if source_stats is not None: - try: - source_mtime = int(source_stats['mtime']) - except KeyError: - pass - else: - if _r_long(raw_timestamp) != source_mtime: - message = 'bytecode is stale for {}'.format(fullname) - _verbose_message(message) - raise ImportError(message, name=fullname, - path=bytecode_path) - try: - source_size = source_stats['size'] & 0xFFFFFFFF - except KeyError: - pass - else: - if _r_long(raw_size) != source_size: - raise ImportError( - "bytecode is stale for {}".format(fullname), - name=fullname, path=bytecode_path) - # Can't return the code object as errors from marshal loading need to - # propagate even when source is available. - return data[12:] - - @module_for_loader - def _load_module(self, module, *, sourceless=False): - """Helper for load_module able to handle either source or sourceless - loading.""" - name = module.__name__ - code_object = self.get_code(name) - module.__file__ = self.get_filename(name) - if not sourceless: - try: - module.__cached__ = cache_from_source(module.__file__) - except NotImplementedError: - module.__cached__ = module.__file__ - else: - module.__cached__ = module.__file__ - module.__package__ = name - if self.is_package(name): - module.__path__ = [_path_split(module.__file__)[0]] - else: - module.__package__ = module.__package__.rpartition('.')[0] - module.__loader__ = self - _call_with_frames_removed(exec, code_object, module.__dict__) - return module + load_module = _load_module_shim class SourceLoader(_LoaderBasics): @@ -874,8 +1454,10 @@ class SourceLoader(_LoaderBasics): def path_mtime(self, path): """Optional method that returns the modification time (an int) for the specified path, where path is a str. + + Raises IOError when the path cannot be handled. """ - raise NotImplementedError + raise IOError def path_stats(self, path): """Optional method returning a metadata dict for the specified path @@ -886,6 +1468,7 @@ class SourceLoader(_LoaderBasics): - 'size' (optional) is the size in bytes of the source code. Implementing this method allows the loader to read bytecode files. + Raises IOError when the path cannot be handled. """ return {'mtime': self.path_mtime(path)} @@ -903,32 +1486,26 @@ class SourceLoader(_LoaderBasics): """Optional method which writes data (bytes) to a file path (a str). Implementing this method allows for the writing of bytecode files. - """ - raise NotImplementedError def get_source(self, fullname): """Concrete implementation of InspectLoader.get_source.""" - import tokenize path = self.get_filename(fullname) try: source_bytes = self.get_data(path) - except IOError as exc: - raise ImportError("source not available through get_data()", - name=fullname) from exc - readsource = _io.BytesIO(source_bytes).readline - try: - encoding = tokenize.detect_encoding(readsource) - except SyntaxError as exc: - raise ImportError("Failed to detect encoding", - name=fullname) from exc - newline_decoder = _io.IncrementalNewlineDecoder(None, True) - try: - return newline_decoder.decode(source_bytes.decode(encoding[0])) - except UnicodeDecodeError as exc: - raise ImportError("Failed to decode source file", + except OSError as exc: + raise ImportError('source not available through get_data()', name=fullname) from exc + return decode_source(source_bytes) + + def source_to_code(self, data, path, *, _optimize=-1): + """Return the code object compiled from source. + + The 'data' argument can be any object type that compile() supports. + """ + return _call_with_frames_removed(compile, data, path, 'exec', + dont_inherit=True, optimize=_optimize) def get_code(self, fullname): """Concrete implementation of InspectLoader.get_code. @@ -946,45 +1523,34 @@ class SourceLoader(_LoaderBasics): else: try: st = self.path_stats(source_path) - except NotImplementedError: + except IOError: pass else: source_mtime = int(st['mtime']) try: data = self.get_data(bytecode_path) - except IOError: + except OSError: pass else: try: - bytes_data = self._bytes_from_bytecode(fullname, data, - bytecode_path, - st) + bytes_data = _validate_bytecode_header(data, + source_stats=st, name=fullname, + path=bytecode_path) except (ImportError, EOFError): pass else: _verbose_message('{} matches {}', bytecode_path, source_path) - found = marshal.loads(bytes_data) - if isinstance(found, _code_type): - _imp._fix_co_filename(found, source_path) - _verbose_message('code object from {}', - bytecode_path) - return found - else: - msg = "Non-code object in {}" - raise ImportError(msg.format(bytecode_path), - name=fullname, path=bytecode_path) + return _compile_bytecode(bytes_data, name=fullname, + bytecode_path=bytecode_path, + source_path=source_path) source_bytes = self.get_data(source_path) - code_object = _call_with_frames_removed(compile, - source_bytes, source_path, 'exec', - dont_inherit=True) + code_object = self.source_to_code(source_bytes, source_path) _verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and - source_mtime is not None): - data = bytearray(_MAGIC_BYTES) - data.extend(_w_long(source_mtime)) - data.extend(_w_long(len(source_bytes))) - data.extend(marshal.dumps(code_object)) + source_mtime is not None): + data = _code_to_bytecode(code_object, source_mtime, + len(source_bytes)) try: self._cache_bytecode(source_path, bytecode_path, data) _verbose_message('wrote {!r}', bytecode_path) @@ -992,16 +1558,6 @@ class SourceLoader(_LoaderBasics): pass return code_object - def load_module(self, fullname): - """Concrete implementation of Loader.load_module. - - Requires ExecutionLoader.get_filename and ResourceLoader.get_data to be - implemented to load source code. Use of bytecode is dictated by whether - get_code uses/writes bytecode. - - """ - return self._load_module(fullname) - class FileLoader: @@ -1014,10 +1570,22 @@ class FileLoader: self.name = fullname self.path = path + def __eq__(self, other): + return (self.__class__ == other.__class__ and + self.__dict__ == other.__dict__) + + def __hash__(self): + return hash(self.name) ^ hash(self.path) + @_check_name def load_module(self, fullname): - """Load a module from a file.""" - # Issue #14857: Avoid the zero-argument form so the implementation + """Load a module from a file. + + This method is deprecated. Use exec_module() instead. + + """ + # The only reason for this method is for the name check. + # Issue #14857: Avoid the zero-argument form of super so the implementation # of that form can be updated without breaking the frozen module return super(FileLoader, self).load_module(fullname) @@ -1038,18 +1606,12 @@ class SourceFileLoader(FileLoader, SourceLoader): def path_stats(self, path): """Return the metadata for the path.""" - st = _os.stat(path) + st = _path_stat(path) return {'mtime': st.st_mtime, 'size': st.st_size} def _cache_bytecode(self, source_path, bytecode_path, data): # Adapt between the two APIs - try: - mode = _os.stat(source_path).st_mode - except OSError: - mode = 0o666 - # We always ensure write access so we can update cached files - # later even when the source files are read-only on Windows (#6074) - mode |= 0o200 + mode = _calc_mode(source_path) return self.set_data(bytecode_path, data, _mode=mode) def set_data(self, path, data, *, _mode=0o666): @@ -1085,20 +1647,11 @@ class SourcelessFileLoader(FileLoader, _LoaderBasics): """Loader which handles sourceless file imports.""" - def load_module(self, fullname): - return self._load_module(fullname, sourceless=True) - def get_code(self, fullname): path = self.get_filename(fullname) data = self.get_data(path) - bytes_data = self._bytes_from_bytecode(fullname, data, path, None) - found = marshal.loads(bytes_data) - if isinstance(found, _code_type): - _verbose_message('code object from {!r}', path) - return found - else: - raise ImportError("Non-code object in {}".format(path), - name=fullname, path=path) + bytes_data = _validate_bytecode_header(data, name=fullname, path=path) + return _compile_bytecode(bytes_data, name=fullname, bytecode_path=path) def get_source(self, fullname): """Return None as there is no source code.""" @@ -1121,23 +1674,30 @@ class ExtensionFileLoader: self.name = name self.path = path + def __eq__(self, other): + return (self.__class__ == other.__class__ and + self.__dict__ == other.__dict__) + + def __hash__(self): + return hash(self.name) ^ hash(self.path) + @_check_name - @set_package - @set_loader def load_module(self, fullname): """Load an extension module.""" - is_reload = fullname in sys.modules - try: + # Once an exec_module() implementation is added we can also + # add a deprecation warning here. + with _ManageReload(fullname): module = _call_with_frames_removed(_imp.load_dynamic, fullname, self.path) - _verbose_message('extension module loaded from {!r}', self.path) - if self.is_package(fullname) and not hasattr(module, '__path__'): - module.__path__ = [_path_split(self.path)[0]] - return module - except: - if not is_reload and fullname in sys.modules: - del sys.modules[fullname] - raise + _verbose_message('extension module loaded from {!r}', self.path) + is_package = self.is_package(fullname) + if is_package and not hasattr(module, '__path__'): + module.__path__ = [_path_split(self.path)[0]] + module.__loader__ = self + module.__package__ = module.__name__ + if not is_package: + module.__package__ = module.__package__.rpartition('.')[0] + return module def is_package(self, fullname): """Return True if the extension module is a package.""" @@ -1153,6 +1713,11 @@ class ExtensionFileLoader: """Return None as extension modules have no source code.""" return None + @_check_name + def get_filename(self, fullname): + """Return the path to the source file as found by the finder.""" + return self.path + class _NamespacePath: """Represents a namespace package's path. It uses the module name @@ -1185,11 +1750,12 @@ class _NamespacePath: # If the parent's path has changed, recalculate _path parent_path = tuple(self._get_parent_path()) # Make a copy if parent_path != self._last_parent_path: - loader, new_path = self._path_finder(self._name, parent_path) + spec = self._path_finder(self._name, parent_path) # Note that no changes are made if a loader is returned, but we # do remember the new parent path - if loader is None: - self._path = new_path + if spec is not None and spec.loader is None: + if spec.submodule_search_locations: + self._path = spec.submodule_search_locations self._last_parent_path = parent_path # Save the copy return self._path @@ -1200,7 +1766,7 @@ class _NamespacePath: return len(self._recalculate()) def __repr__(self): - return "_NamespacePath({!r})".format(self._path) + return '_NamespacePath({!r})'.format(self._path) def __contains__(self, item): return item in self._recalculate() @@ -1209,20 +1775,41 @@ class _NamespacePath: self._path.append(item) -class NamespaceLoader: +# We use this exclusively in init_module_attrs() for backward-compatibility. +class _NamespaceLoader: def __init__(self, name, path, path_finder): self._path = _NamespacePath(name, path, path_finder) @classmethod def module_repr(cls, module): - return "<module '{}' (namespace)>".format(module.__name__) + """Return repr for the module. + + The method is deprecated. The import machinery does the job itself. + + """ + return '<module {!r} (namespace)>'.format(module.__name__) + + def is_package(self, fullname): + return True + + def get_source(self, fullname): + return '' + + def get_code(self, fullname): + return compile('', '<string>', 'exec', dont_inherit=True) - @module_for_loader - def load_module(self, module): - """Load a namespace module.""" + def exec_module(self, module): + pass + + def load_module(self, fullname): + """Load a namespace module. + + This method is deprecated. Use exec_module() instead. + + """ + # The import system never calls this method. _verbose_message('namespace module loaded with path {!r}', self._path) - module.__path__ = self._path - return module + return _load_module_shim(self, fullname) # Finders ##################################################################### @@ -1265,7 +1852,7 @@ class PathFinder: """ if path == '': - path = '.' + path = _os.getcwd() try: finder = sys.path_importer_cache[path] except KeyError: @@ -1274,7 +1861,22 @@ class PathFinder: return finder @classmethod - def _get_loader(cls, fullname, path): + def _legacy_get_spec(cls, fullname, finder): + # This would be a good place for a DeprecationWarning if + # we ended up going that route. + if hasattr(finder, 'find_loader'): + loader, portions = finder.find_loader(fullname) + else: + loader = finder.find_module(fullname) + portions = None + if loader is not None: + return spec_from_loader(fullname, loader) + spec = ModuleSpec(fullname, None) + spec.submodule_search_locations = portions + return spec + + @classmethod + def _get_spec(cls, fullname, path, target=None): """Find the loader or namespace_path for this module/package name.""" # If this ends up being a namespace package, namespace_path is # the list of paths that will become its __path__ @@ -1284,38 +1886,61 @@ class PathFinder: continue finder = cls._path_importer_cache(entry) if finder is not None: - if hasattr(finder, 'find_loader'): - loader, portions = finder.find_loader(fullname) + if hasattr(finder, 'find_spec'): + spec = finder.find_spec(fullname, target) else: - loader = finder.find_module(fullname) - portions = [] - if loader is not None: - # We found a loader: return it immediately. - return loader, namespace_path + spec = cls._legacy_get_spec(fullname, finder) + if spec is None: + continue + if spec.loader is not None: + return spec + portions = spec.submodule_search_locations + if portions is None: + raise ImportError('spec missing loader') # This is possibly part of a namespace package. # Remember these path entries (if any) for when we # create a namespace package, and continue iterating # on path. namespace_path.extend(portions) else: - return None, namespace_path + spec = ModuleSpec(fullname, None) + spec.submodule_search_locations = namespace_path + return spec @classmethod - def find_module(cls, fullname, path=None): - """Find the module on sys.path or 'path' based on sys.path_hooks and + def find_spec(cls, fullname, path=None, target=None): + """find the module on sys.path or 'path' based on sys.path_hooks and sys.path_importer_cache.""" if path is None: path = sys.path - loader, namespace_path = cls._get_loader(fullname, path) - if loader is not None: - return loader - else: + spec = cls._get_spec(fullname, path, target) + if spec is None: + return None + elif spec.loader is None: + namespace_path = spec.submodule_search_locations if namespace_path: # We found at least one namespace path. Return a - # loader which can create the namespace package. - return NamespaceLoader(fullname, namespace_path, cls._get_loader) + # spec which can create the namespace package. + spec.origin = 'namespace' + spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec) + return spec else: return None + else: + return spec + + @classmethod + def find_module(cls, fullname, path=None): + """find the module on sys.path or 'path' based on sys.path_hooks and + sys.path_importer_cache. + + This method is deprecated. Use find_spec() instead. + + """ + spec = cls.find_spec(fullname, path) + if spec is None: + return None + return spec.loader class FileFinder: @@ -1349,11 +1974,28 @@ class FileFinder: def find_loader(self, fullname): """Try to find a loader for the specified module, or the namespace + package portions. Returns (loader, list-of-portions). + + This method is deprecated. Use find_spec() instead. + + """ + spec = self.find_spec(fullname) + if spec is None: + return None, [] + return spec.loader, spec.submodule_search_locations or [] + + def _get_spec(self, loader_class, fullname, path, smsl, target): + loader = loader_class(fullname, path) + return spec_from_file_location(fullname, path, loader=loader, + submodule_search_locations=smsl) + + def find_spec(self, fullname, target=None): + """Try to find a loader for the specified module, or the namespace package portions. Returns (loader, list-of-portions).""" is_namespace = False tail_module = fullname.rpartition('.')[2] try: - mtime = _os.stat(self.path).st_mtime + mtime = _path_stat(self.path or _os.getcwd()).st_mtime except OSError: mtime = -1 if mtime != self._path_mtime: @@ -1369,33 +2011,34 @@ class FileFinder: # Check if the module is the name of a directory (and thus a package). if cache_module in cache: base_path = _path_join(self.path, tail_module) - if _path_isdir(base_path): - for suffix, loader in self._loaders: - init_filename = '__init__' + suffix - full_path = _path_join(base_path, init_filename) - if _path_isfile(full_path): - return (loader(fullname, full_path), [base_path]) - else: - # A namespace package, return the path if we don't also - # find a module in the next section. - is_namespace = True + for suffix, loader_class in self._loaders: + init_filename = '__init__' + suffix + full_path = _path_join(base_path, init_filename) + if _path_isfile(full_path): + return self._get_spec(loader_class, fullname, full_path, [base_path], target) + else: + # If a namespace package, return the path if we don't + # find a module in the next section. + is_namespace = _path_isdir(base_path) # Check for a file w/ a proper suffix exists. - for suffix, loader in self._loaders: + for suffix, loader_class in self._loaders: full_path = _path_join(self.path, tail_module + suffix) _verbose_message('trying {}'.format(full_path), verbosity=2) if cache_module + suffix in cache: if _path_isfile(full_path): - return (loader(fullname, full_path), []) + return self._get_spec(loader_class, fullname, full_path, None, target) if is_namespace: _verbose_message('possible namespace for {}'.format(base_path)) - return (None, [base_path]) - return (None, []) + spec = ModuleSpec(fullname, None) + spec.submodule_search_locations = [base_path] + return spec + return None def _fill_cache(self): """Fill the cache of potential modules and packages for this directory.""" path = self.path try: - contents = _os.listdir(path) + contents = _os.listdir(path or _os.getcwd()) except (FileNotFoundError, PermissionError, NotADirectoryError): # Directory has either been removed, turned into a file, or made # unreadable. @@ -1420,7 +2063,7 @@ class FileFinder: lower_suffix_contents.add(new_name) self._path_cache = lower_suffix_contents if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): - self._relaxed_path_cache = set(fn.lower() for fn in contents) + self._relaxed_path_cache = {fn.lower() for fn in contents} @classmethod def path_hook(cls, *loader_details): @@ -1435,13 +2078,13 @@ class FileFinder: def path_hook_for_FileFinder(path): """Path hook for importlib.machinery.FileFinder.""" if not _path_isdir(path): - raise ImportError("only directories are supported", path=path) + raise ImportError('only directories are supported', path=path) return cls(path, *loader_details) return path_hook_for_FileFinder def __repr__(self): - return "FileFinder(%r)" % (self.path,) + return 'FileFinder({!r})'.format(self.path) # Import itself ############################################################### @@ -1468,19 +2111,51 @@ def _resolve_name(name, package, level): return '{}.{}'.format(base, name) if name else base -def _find_module(name, path): +def _find_spec_legacy(finder, name, path): + # This would be a good place for a DeprecationWarning if + # we ended up going that route. + loader = finder.find_module(name, path) + if loader is None: + return None + return spec_from_loader(name, loader) + + +def _find_spec(name, path, target=None): """Find a module's loader.""" if not sys.meta_path: _warnings.warn('sys.meta_path is empty', ImportWarning) + # We check sys.modules here for the reload case. While a passed-in + # target will usually indicate a reload there is no guarantee, whereas + # sys.modules provides one. + is_reload = name in sys.modules for finder in sys.meta_path: with _ImportLockContext(): - loader = finder.find_module(name, path) - if loader is not None: + try: + find_spec = finder.find_spec + except AttributeError: + spec = _find_spec_legacy(finder, name, path) + if spec is None: + continue + else: + spec = find_spec(name, path, target) + if spec is not None: # The parent import may have already imported this module. - if name not in sys.modules: - return loader + if not is_reload and name in sys.modules: + module = sys.modules[name] + try: + __spec__ = module.__spec__ + except AttributeError: + # We use the found spec since that is the one that + # we would have used if the parent module hadn't + # beaten us to the punch. + return spec + else: + if __spec__ is None: + return spec + else: + return __spec__ else: - return sys.modules[name].__loader__ + return spec else: return None @@ -1488,21 +2163,22 @@ def _find_module(name, path): def _sanity_check(name, package, level): """Verify arguments are "sane".""" if not isinstance(name, str): - raise TypeError("module name must be str, not {}".format(type(name))) + raise TypeError('module name must be str, not {}'.format(type(name))) if level < 0: raise ValueError('level must be >= 0') if package: if not isinstance(package, str): - raise TypeError("__package__ not set to a string") + raise TypeError('__package__ not set to a string') elif package not in sys.modules: - msg = ("Parent module {!r} not loaded, cannot perform relative " - "import") + msg = ('Parent module {!r} not loaded, cannot perform relative ' + 'import') raise SystemError(msg.format(package)) if not name and level == 0: - raise ValueError("Empty module name") + raise ValueError('Empty module name') -_ERR_MSG = 'No module named {!r}' +_ERR_MSG_PREFIX = 'No module named ' +_ERR_MSG = _ERR_MSG_PREFIX + '{!r}' def _find_and_load_unlocked(name, import_): path = None @@ -1513,58 +2189,28 @@ def _find_and_load_unlocked(name, import_): # Crazy side-effects! if name in sys.modules: return sys.modules[name] - # Backwards-compatibility; be nicer to skip the dict lookup. parent_module = sys.modules[parent] try: path = parent_module.__path__ except AttributeError: - msg = (_ERR_MSG + '; {} is not a package').format(name, parent) + msg = (_ERR_MSG + '; {!r} is not a package').format(name, parent) raise ImportError(msg, name=name) - loader = _find_module(name, path) - if loader is None: - exc = ImportError(_ERR_MSG.format(name), name=name) - # TODO(brett): switch to a proper ModuleNotFound exception in Python - # 3.4. - exc._not_found = True - raise exc - elif name not in sys.modules: - # The parent import may have already imported this module. - loader.load_module(name) - _verbose_message('import {!r} # {!r}', name, loader) - # Backwards-compatibility; be nicer to skip the dict lookup. - module = sys.modules[name] + spec = _find_spec(name, path) + if spec is None: + raise ImportError(_ERR_MSG.format(name), name=name) + else: + module = _SpecMethods(spec)._load_unlocked() if parent: # Set the module as an attribute on its parent. parent_module = sys.modules[parent] setattr(parent_module, name.rpartition('.')[2], module) - # Set __package__ if the loader did not. - if getattr(module, '__package__', None) is None: - try: - module.__package__ = module.__name__ - if not hasattr(module, '__path__'): - module.__package__ = module.__package__.rpartition('.')[0] - except AttributeError: - pass - # Set loader if need be. - if not hasattr(module, '__loader__'): - try: - module.__loader__ = loader - except AttributeError: - pass return module def _find_and_load(name, import_): """Find and load the module, and release the import lock.""" - try: - lock = _get_module_lock(name) - finally: - _imp.release_lock() - lock.acquire() - try: + with _ModuleLockManager(name): return _find_and_load_unlocked(name, import_) - finally: - lock.release() def _gcd_import(name, package=None, level=0): @@ -1585,8 +2231,8 @@ def _gcd_import(name, package=None, level=0): module = sys.modules[name] if module is None: _imp.release_lock() - message = ("import of {} halted; " - "None in sys.modules".format(name)) + message = ('import of {} halted; ' + 'None in sys.modules'.format(name)) raise ImportError(message, name=name) _lock_unlock_module(name) return module @@ -1616,9 +2262,7 @@ def _handle_fromlist(module, fromlist, import_): # Backwards-compatibility dictates we ignore failed # imports triggered by fromlist for modules that don't # exist. - # TODO(brett): In Python 3.4, have import raise - # ModuleNotFound and catch that. - if getattr(exc, '_not_found', False): + if str(exc).startswith(_ERR_MSG_PREFIX): if exc.name == from_name: continue raise @@ -1686,6 +2330,13 @@ def __import__(name, globals=None, locals=None, fromlist=(), level=0): return _handle_fromlist(module, fromlist, _gcd_import) +def _builtin_from_name(name): + spec = BuiltinImporter.find_spec(name) + if spec is None: + raise ImportError('no built-in module named ' + name) + methods = _SpecMethods(spec) + return methods._load_unlocked() + def _setup(sys_module, _imp_module): """Setup importlib by importing needed built-in modules and injecting them @@ -1704,24 +2355,31 @@ def _setup(sys_module, _imp_module): else: BYTECODE_SUFFIXES = DEBUG_BYTECODE_SUFFIXES + # Set up the spec for existing builtin/frozen modules. module_type = type(sys) for name, module in sys.modules.items(): if isinstance(module, module_type): - if not hasattr(module, '__loader__'): - if name in sys.builtin_module_names: - module.__loader__ = BuiltinImporter - elif _imp.is_frozen(name): - module.__loader__ = FrozenImporter + if name in sys.builtin_module_names: + loader = BuiltinImporter + elif _imp.is_frozen(name): + loader = FrozenImporter + else: + continue + spec = _spec_from_module(module, loader) + methods = _SpecMethods(spec) + methods.init_module_attrs(module) + # Directly load built-in modules needed during bootstrap. self_module = sys.modules[__name__] for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'): if builtin_name not in sys.modules: - builtin_module = BuiltinImporter.load_module(builtin_name) + builtin_module = _builtin_from_name(builtin_name) else: builtin_module = sys.modules[builtin_name] setattr(self_module, builtin_name, builtin_module) - os_details = ('posix', ['/']), ('nt', ['\\', '/']), ('os2', ['\\', '/']) + # Directly load the os module (needed during bootstrap). + os_details = ('posix', ['/']), ('nt', ['\\', '/']) for builtin_os, path_separators in os_details: # Assumption made in _path_join() assert all(len(sep) == 1 for sep in path_separators) @@ -1731,32 +2389,33 @@ def _setup(sys_module, _imp_module): break else: try: - os_module = BuiltinImporter.load_module(builtin_os) - # TODO: rip out os2 code after 3.3 is released as per PEP 11 - if builtin_os == 'os2' and 'EMX GCC' in sys.version: - path_sep = path_separators[1] + os_module = _builtin_from_name(builtin_os) break except ImportError: continue else: raise ImportError('importlib requires posix or nt') + setattr(self_module, '_os', os_module) + setattr(self_module, 'path_sep', path_sep) + setattr(self_module, 'path_separators', ''.join(path_separators)) + # Directly load the _thread module (needed during bootstrap). try: - thread_module = BuiltinImporter.load_module('_thread') + thread_module = _builtin_from_name('_thread') except ImportError: # Python was built without threads thread_module = None - weakref_module = BuiltinImporter.load_module('_weakref') + setattr(self_module, '_thread', thread_module) + # Directly load the _weakref module (needed during bootstrap). + weakref_module = _builtin_from_name('_weakref') + setattr(self_module, '_weakref', weakref_module) + + # Directly load the winreg module (needed during bootstrap). if builtin_os == 'nt': - winreg_module = BuiltinImporter.load_module('winreg') + winreg_module = _builtin_from_name('winreg') setattr(self_module, '_winreg', winreg_module) - setattr(self_module, '_os', os_module) - setattr(self_module, '_thread', thread_module) - setattr(self_module, '_weakref', weakref_module) - setattr(self_module, 'path_sep', path_sep) - setattr(self_module, 'path_separators', set(path_separators)) # Constants setattr(self_module, '_relax_case', _make_relax_case()) EXTENSION_SUFFIXES.extend(_imp.extension_suffixes()) |