summaryrefslogtreecommitdiff
path: root/Lib/test/test_importlib/source
diff options
context:
space:
mode:
Diffstat (limited to 'Lib/test/test_importlib/source')
-rw-r--r--Lib/test/test_importlib/source/__init__.py13
-rw-r--r--Lib/test/test_importlib/source/test_abc_loader.py906
-rw-r--r--Lib/test/test_importlib/source/test_case_sensitivity.py70
-rw-r--r--Lib/test/test_importlib/source/test_file_loader.py484
-rw-r--r--Lib/test/test_importlib/source/test_finder.py191
-rw-r--r--Lib/test/test_importlib/source/test_path_hook.py32
-rw-r--r--Lib/test/test_importlib/source/test_source_encoding.py123
-rw-r--r--Lib/test/test_importlib/source/util.py97
8 files changed, 1916 insertions, 0 deletions
diff --git a/Lib/test/test_importlib/source/__init__.py b/Lib/test/test_importlib/source/__init__.py
new file mode 100644
index 0000000000..3ef97f3aa0
--- /dev/null
+++ b/Lib/test/test_importlib/source/__init__.py
@@ -0,0 +1,13 @@
+from .. import test_suite
+import os.path
+import unittest
+
+
+def test_suite():
+ directory = os.path.dirname(__file__)
+ return test.test_suite('importlib.test.source', directory)
+
+
+if __name__ == '__main__':
+ from test.support import run_unittest
+ run_unittest(test_suite())
diff --git a/Lib/test/test_importlib/source/test_abc_loader.py b/Lib/test/test_importlib/source/test_abc_loader.py
new file mode 100644
index 0000000000..0d912b6469
--- /dev/null
+++ b/Lib/test/test_importlib/source/test_abc_loader.py
@@ -0,0 +1,906 @@
+import importlib
+from importlib import abc
+
+from .. import abc as testing_abc
+from .. import util
+from . import util as source_util
+
+import imp
+import inspect
+import io
+import marshal
+import os
+import sys
+import types
+import unittest
+import warnings
+
+
+class SourceOnlyLoaderMock(abc.SourceLoader):
+
+ # Globals that should be defined for all modules.
+ source = (b"_ = '::'.join([__name__, __file__, __cached__, __package__, "
+ b"repr(__loader__)])")
+
+ def __init__(self, path):
+ self.path = path
+
+ def get_data(self, path):
+ assert self.path == path
+ return self.source
+
+ def get_filename(self, fullname):
+ return self.path
+
+ def module_repr(self, module):
+ return '<module>'
+
+
+class SourceLoaderMock(SourceOnlyLoaderMock):
+
+ source_mtime = 1
+
+ def __init__(self, path, magic=imp.get_magic()):
+ super().__init__(path)
+ self.bytecode_path = imp.cache_from_source(self.path)
+ self.source_size = len(self.source)
+ data = bytearray(magic)
+ data.extend(importlib._w_long(self.source_mtime))
+ data.extend(importlib._w_long(self.source_size))
+ code_object = compile(self.source, self.path, 'exec',
+ dont_inherit=True)
+ data.extend(marshal.dumps(code_object))
+ self.bytecode = bytes(data)
+ self.written = {}
+
+ def get_data(self, path):
+ if path == self.path:
+ return super().get_data(path)
+ elif path == self.bytecode_path:
+ return self.bytecode
+ else:
+ raise IOError
+
+ def path_stats(self, path):
+ assert path == self.path
+ return {'mtime': self.source_mtime, 'size': self.source_size}
+
+ def set_data(self, path, data):
+ self.written[path] = bytes(data)
+ return path == self.bytecode_path
+
+
+class PyLoaderMock(abc.PyLoader):
+
+ # Globals that should be defined for all modules.
+ source = (b"_ = '::'.join([__name__, __file__, __package__, "
+ b"repr(__loader__)])")
+
+ def __init__(self, data):
+ """Take a dict of 'module_name: path' pairings.
+
+ Paths should have no file extension, allowing packages to be denoted by
+ ending in '__init__'.
+
+ """
+ self.module_paths = data
+ self.path_to_module = {val:key for key,val in data.items()}
+
+ def get_data(self, path):
+ if path not in self.path_to_module:
+ raise IOError
+ return self.source
+
+ def is_package(self, name):
+ filename = os.path.basename(self.get_filename(name))
+ return os.path.splitext(filename)[0] == '__init__'
+
+ def source_path(self, name):
+ try:
+ return self.module_paths[name]
+ except KeyError:
+ raise ImportError
+
+ def get_filename(self, name):
+ """Silence deprecation warning."""
+ with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter("always")
+ path = super().get_filename(name)
+ assert len(w) == 1
+ assert issubclass(w[0].category, DeprecationWarning)
+ return path
+
+ def module_repr(self):
+ return '<module>'
+
+
+class PyLoaderCompatMock(PyLoaderMock):
+
+ """Mock that matches what is suggested to have a loader that is compatible
+ from Python 3.1 onwards."""
+
+ def get_filename(self, fullname):
+ try:
+ return self.module_paths[fullname]
+ except KeyError:
+ raise ImportError
+
+ def source_path(self, fullname):
+ try:
+ return self.get_filename(fullname)
+ except ImportError:
+ return None
+
+
+class PyPycLoaderMock(abc.PyPycLoader, PyLoaderMock):
+
+ default_mtime = 1
+
+ def __init__(self, source, bc={}):
+ """Initialize mock.
+
+ 'bc' is a dict keyed on a module's name. The value is dict with
+ possible keys of 'path', 'mtime', 'magic', and 'bc'. Except for 'path',
+ each of those keys control if any part of created bytecode is to
+ deviate from default values.
+
+ """
+ super().__init__(source)
+ self.module_bytecode = {}
+ self.path_to_bytecode = {}
+ self.bytecode_to_path = {}
+ for name, data in bc.items():
+ self.path_to_bytecode[data['path']] = name
+ self.bytecode_to_path[name] = data['path']
+ magic = data.get('magic', imp.get_magic())
+ mtime = importlib._w_long(data.get('mtime', self.default_mtime))
+ source_size = importlib._w_long(len(self.source) & 0xFFFFFFFF)
+ if 'bc' in data:
+ bc = data['bc']
+ else:
+ bc = self.compile_bc(name)
+ self.module_bytecode[name] = magic + mtime + source_size + bc
+
+ def compile_bc(self, name):
+ source_path = self.module_paths.get(name, '<test>') or '<test>'
+ code = compile(self.source, source_path, 'exec')
+ return marshal.dumps(code)
+
+ def source_mtime(self, name):
+ if name in self.module_paths:
+ return self.default_mtime
+ elif name in self.module_bytecode:
+ return None
+ else:
+ raise ImportError
+
+ def bytecode_path(self, name):
+ try:
+ return self.bytecode_to_path[name]
+ except KeyError:
+ if name in self.module_paths:
+ return None
+ else:
+ raise ImportError
+
+ def write_bytecode(self, name, bytecode):
+ self.module_bytecode[name] = bytecode
+ return True
+
+ def get_data(self, path):
+ if path in self.path_to_module:
+ return super().get_data(path)
+ elif path in self.path_to_bytecode:
+ name = self.path_to_bytecode[path]
+ return self.module_bytecode[name]
+ else:
+ raise IOError
+
+ def is_package(self, name):
+ try:
+ return super().is_package(name)
+ except TypeError:
+ return '__init__' in self.bytecode_to_path[name]
+
+ def get_code(self, name):
+ with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter("always")
+ code_object = super().get_code(name)
+ assert len(w) == 1
+ assert issubclass(w[0].category, DeprecationWarning)
+ return code_object
+
+class PyLoaderTests(testing_abc.LoaderTests):
+
+ """Tests for importlib.abc.PyLoader."""
+
+ mocker = PyLoaderMock
+
+ def eq_attrs(self, ob, **kwargs):
+ for attr, val in kwargs.items():
+ found = getattr(ob, attr)
+ self.assertEqual(found, val,
+ "{} attribute: {} != {}".format(attr, found, val))
+
+ def test_module(self):
+ name = '<module>'
+ path = os.path.join('', 'path', 'to', 'module')
+ mock = self.mocker({name: path})
+ with util.uncache(name):
+ module = mock.load_module(name)
+ self.assertIn(name, sys.modules)
+ self.eq_attrs(module, __name__=name, __file__=path, __package__='',
+ __loader__=mock)
+ self.assertTrue(not hasattr(module, '__path__'))
+ return mock, name
+
+ def test_package(self):
+ name = '<pkg>'
+ path = os.path.join('path', 'to', name, '__init__')
+ mock = self.mocker({name: path})
+ with util.uncache(name):
+ module = mock.load_module(name)
+ self.assertIn(name, sys.modules)
+ self.eq_attrs(module, __name__=name, __file__=path,
+ __path__=[os.path.dirname(path)], __package__=name,
+ __loader__=mock)
+ return mock, name
+
+ def test_lacking_parent(self):
+ name = 'pkg.mod'
+ path = os.path.join('path', 'to', 'pkg', 'mod')
+ mock = self.mocker({name: path})
+ with util.uncache(name):
+ module = mock.load_module(name)
+ self.assertIn(name, sys.modules)
+ self.eq_attrs(module, __name__=name, __file__=path, __package__='pkg',
+ __loader__=mock)
+ self.assertFalse(hasattr(module, '__path__'))
+ return mock, name
+
+ def test_module_reuse(self):
+ name = 'mod'
+ path = os.path.join('path', 'to', 'mod')
+ module = imp.new_module(name)
+ mock = self.mocker({name: path})
+ with util.uncache(name):
+ sys.modules[name] = module
+ loaded_module = mock.load_module(name)
+ self.assertIs(loaded_module, module)
+ self.assertIs(sys.modules[name], module)
+ return mock, name
+
+ def test_state_after_failure(self):
+ name = "mod"
+ module = imp.new_module(name)
+ module.blah = None
+ mock = self.mocker({name: os.path.join('path', 'to', 'mod')})
+ mock.source = b"1/0"
+ with util.uncache(name):
+ sys.modules[name] = module
+ with self.assertRaises(ZeroDivisionError):
+ mock.load_module(name)
+ self.assertIs(sys.modules[name], module)
+ self.assertTrue(hasattr(module, 'blah'))
+ return mock
+
+ def test_unloadable(self):
+ name = "mod"
+ mock = self.mocker({name: os.path.join('path', 'to', 'mod')})
+ mock.source = b"1/0"
+ with util.uncache(name):
+ with self.assertRaises(ZeroDivisionError):
+ mock.load_module(name)
+ self.assertNotIn(name, sys.modules)
+ return mock
+
+
+class PyLoaderCompatTests(PyLoaderTests):
+
+ """Test that the suggested code to make a loader that is compatible from
+ Python 3.1 forward works."""
+
+ mocker = PyLoaderCompatMock
+
+
+class PyLoaderInterfaceTests(unittest.TestCase):
+
+ """Tests for importlib.abc.PyLoader to make sure that when source_path()
+ doesn't return a path everything works as expected."""
+
+ def test_no_source_path(self):
+ # No source path should lead to ImportError.
+ name = 'mod'
+ mock = PyLoaderMock({})
+ with util.uncache(name), self.assertRaises(ImportError):
+ mock.load_module(name)
+
+ def test_source_path_is_None(self):
+ name = 'mod'
+ mock = PyLoaderMock({name: None})
+ with util.uncache(name), self.assertRaises(ImportError):
+ mock.load_module(name)
+
+ def test_get_filename_with_source_path(self):
+ # get_filename() should return what source_path() returns.
+ name = 'mod'
+ path = os.path.join('path', 'to', 'source')
+ mock = PyLoaderMock({name: path})
+ with util.uncache(name):
+ self.assertEqual(mock.get_filename(name), path)
+
+ def test_get_filename_no_source_path(self):
+ # get_filename() should raise ImportError if source_path returns None.
+ name = 'mod'
+ mock = PyLoaderMock({name: None})
+ with util.uncache(name), self.assertRaises(ImportError):
+ mock.get_filename(name)
+
+
+class PyPycLoaderTests(PyLoaderTests):
+
+ """Tests for importlib.abc.PyPycLoader."""
+
+ mocker = PyPycLoaderMock
+
+ @source_util.writes_bytecode_files
+ def verify_bytecode(self, mock, name):
+ assert name in mock.module_paths
+ self.assertIn(name, mock.module_bytecode)
+ magic = mock.module_bytecode[name][:4]
+ self.assertEqual(magic, imp.get_magic())
+ mtime = importlib._r_long(mock.module_bytecode[name][4:8])
+ self.assertEqual(mtime, 1)
+ source_size = mock.module_bytecode[name][8:12]
+ self.assertEqual(len(mock.source) & 0xFFFFFFFF,
+ importlib._r_long(source_size))
+ bc = mock.module_bytecode[name][12:]
+ self.assertEqual(bc, mock.compile_bc(name))
+
+ def test_module(self):
+ mock, name = super().test_module()
+ self.verify_bytecode(mock, name)
+
+ def test_package(self):
+ mock, name = super().test_package()
+ self.verify_bytecode(mock, name)
+
+ def test_lacking_parent(self):
+ mock, name = super().test_lacking_parent()
+ self.verify_bytecode(mock, name)
+
+ def test_module_reuse(self):
+ mock, name = super().test_module_reuse()
+ self.verify_bytecode(mock, name)
+
+ def test_state_after_failure(self):
+ super().test_state_after_failure()
+
+ def test_unloadable(self):
+ super().test_unloadable()
+
+
+class PyPycLoaderInterfaceTests(unittest.TestCase):
+
+ """Test for the interface of importlib.abc.PyPycLoader."""
+
+ def get_filename_check(self, src_path, bc_path, expect):
+ name = 'mod'
+ mock = PyPycLoaderMock({name: src_path}, {name: {'path': bc_path}})
+ with util.uncache(name):
+ assert mock.source_path(name) == src_path
+ assert mock.bytecode_path(name) == bc_path
+ self.assertEqual(mock.get_filename(name), expect)
+
+ def test_filename_with_source_bc(self):
+ # When source and bytecode paths present, return the source path.
+ self.get_filename_check('source_path', 'bc_path', 'source_path')
+
+ def test_filename_with_source_no_bc(self):
+ # With source but no bc, return source path.
+ self.get_filename_check('source_path', None, 'source_path')
+
+ def test_filename_with_no_source_bc(self):
+ # With not source but bc, return the bc path.
+ self.get_filename_check(None, 'bc_path', 'bc_path')
+
+ def test_filename_with_no_source_or_bc(self):
+ # With no source or bc, raise ImportError.
+ name = 'mod'
+ mock = PyPycLoaderMock({name: None}, {name: {'path': None}})
+ with util.uncache(name), self.assertRaises(ImportError):
+ mock.get_filename(name)
+
+
+class SkipWritingBytecodeTests(unittest.TestCase):
+
+ """Test that bytecode is properly handled based on
+ sys.dont_write_bytecode."""
+
+ @source_util.writes_bytecode_files
+ def run_test(self, dont_write_bytecode):
+ name = 'mod'
+ mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')})
+ sys.dont_write_bytecode = dont_write_bytecode
+ with util.uncache(name):
+ mock.load_module(name)
+ self.assertIsNot(name in mock.module_bytecode, dont_write_bytecode)
+
+ def test_no_bytecode_written(self):
+ self.run_test(True)
+
+ def test_bytecode_written(self):
+ self.run_test(False)
+
+
+class RegeneratedBytecodeTests(unittest.TestCase):
+
+ """Test that bytecode is regenerated as expected."""
+
+ @source_util.writes_bytecode_files
+ def test_different_magic(self):
+ # A different magic number should lead to new bytecode.
+ name = 'mod'
+ bad_magic = b'\x00\x00\x00\x00'
+ assert bad_magic != imp.get_magic()
+ mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')},
+ {name: {'path': os.path.join('path', 'to',
+ 'mod.bytecode'),
+ 'magic': bad_magic}})
+ with util.uncache(name):
+ mock.load_module(name)
+ self.assertIn(name, mock.module_bytecode)
+ magic = mock.module_bytecode[name][:4]
+ self.assertEqual(magic, imp.get_magic())
+
+ @source_util.writes_bytecode_files
+ def test_old_mtime(self):
+ # Bytecode with an older mtime should be regenerated.
+ name = 'mod'
+ old_mtime = PyPycLoaderMock.default_mtime - 1
+ mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')},
+ {name: {'path': 'path/to/mod.bytecode', 'mtime': old_mtime}})
+ with util.uncache(name):
+ mock.load_module(name)
+ self.assertIn(name, mock.module_bytecode)
+ mtime = importlib._r_long(mock.module_bytecode[name][4:8])
+ self.assertEqual(mtime, PyPycLoaderMock.default_mtime)
+
+
+class BadBytecodeFailureTests(unittest.TestCase):
+
+ """Test import failures when there is no source and parts of the bytecode
+ is bad."""
+
+ def test_bad_magic(self):
+ # A bad magic number should lead to an ImportError.
+ name = 'mod'
+ bad_magic = b'\x00\x00\x00\x00'
+ bc = {name:
+ {'path': os.path.join('path', 'to', 'mod'),
+ 'magic': bad_magic}}
+ mock = PyPycLoaderMock({name: None}, bc)
+ with util.uncache(name), self.assertRaises(ImportError) as cm:
+ mock.load_module(name)
+ self.assertEqual(cm.exception.name, name)
+
+ def test_no_bytecode(self):
+ # Missing code object bytecode should lead to an EOFError.
+ name = 'mod'
+ bc = {name: {'path': os.path.join('path', 'to', 'mod'), 'bc': b''}}
+ mock = PyPycLoaderMock({name: None}, bc)
+ with util.uncache(name), self.assertRaises(EOFError):
+ mock.load_module(name)
+
+ def test_bad_bytecode(self):
+ # Malformed code object bytecode should lead to a ValueError.
+ name = 'mod'
+ bc = {name: {'path': os.path.join('path', 'to', 'mod'), 'bc': b'1234'}}
+ mock = PyPycLoaderMock({name: None}, bc)
+ with util.uncache(name), self.assertRaises(ValueError):
+ mock.load_module(name)
+
+
+def raise_ImportError(*args, **kwargs):
+ raise ImportError
+
+class MissingPathsTests(unittest.TestCase):
+
+ """Test what happens when a source or bytecode path does not exist (either
+ from *_path returning None or raising ImportError)."""
+
+ def test_source_path_None(self):
+ # Bytecode should be used when source_path returns None, along with
+ # __file__ being set to the bytecode path.
+ name = 'mod'
+ bytecode_path = 'path/to/mod'
+ mock = PyPycLoaderMock({name: None}, {name: {'path': bytecode_path}})
+ with util.uncache(name):
+ module = mock.load_module(name)
+ self.assertEqual(module.__file__, bytecode_path)
+
+ # Testing for bytecode_path returning None handled by all tests where no
+ # bytecode initially exists.
+
+ def test_all_paths_None(self):
+ # If all *_path methods return None, raise ImportError.
+ name = 'mod'
+ mock = PyPycLoaderMock({name: None})
+ with util.uncache(name), self.assertRaises(ImportError) as cm:
+ mock.load_module(name)
+ self.assertEqual(cm.exception.name, name)
+
+ def test_source_path_ImportError(self):
+ # An ImportError from source_path should trigger an ImportError.
+ name = 'mod'
+ mock = PyPycLoaderMock({}, {name: {'path': os.path.join('path', 'to',
+ 'mod')}})
+ with util.uncache(name), self.assertRaises(ImportError):
+ mock.load_module(name)
+
+ def test_bytecode_path_ImportError(self):
+ # An ImportError from bytecode_path should trigger an ImportError.
+ name = 'mod'
+ mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')})
+ bad_meth = types.MethodType(raise_ImportError, mock)
+ mock.bytecode_path = bad_meth
+ with util.uncache(name), self.assertRaises(ImportError) as cm:
+ mock.load_module(name)
+
+
+class SourceLoaderTestHarness(unittest.TestCase):
+
+ def setUp(self, *, is_package=True, **kwargs):
+ self.package = 'pkg'
+ if is_package:
+ self.path = os.path.join(self.package, '__init__.py')
+ self.name = self.package
+ else:
+ module_name = 'mod'
+ self.path = os.path.join(self.package, '.'.join(['mod', 'py']))
+ self.name = '.'.join([self.package, module_name])
+ self.cached = imp.cache_from_source(self.path)
+ self.loader = self.loader_mock(self.path, **kwargs)
+
+ def verify_module(self, module):
+ self.assertEqual(module.__name__, self.name)
+ self.assertEqual(module.__file__, self.path)
+ self.assertEqual(module.__cached__, self.cached)
+ self.assertEqual(module.__package__, self.package)
+ self.assertEqual(module.__loader__, self.loader)
+ values = module._.split('::')
+ self.assertEqual(values[0], self.name)
+ self.assertEqual(values[1], self.path)
+ self.assertEqual(values[2], self.cached)
+ self.assertEqual(values[3], self.package)
+ self.assertEqual(values[4], repr(self.loader))
+
+ def verify_code(self, code_object):
+ module = imp.new_module(self.name)
+ module.__file__ = self.path
+ module.__cached__ = self.cached
+ module.__package__ = self.package
+ module.__loader__ = self.loader
+ module.__path__ = []
+ exec(code_object, module.__dict__)
+ self.verify_module(module)
+
+
+class SourceOnlyLoaderTests(SourceLoaderTestHarness):
+
+ """Test importlib.abc.SourceLoader for source-only loading.
+
+ Reload testing is subsumed by the tests for
+ importlib.util.module_for_loader.
+
+ """
+
+ loader_mock = SourceOnlyLoaderMock
+
+ def test_get_source(self):
+ # Verify the source code is returned as a string.
+ # If an IOError is raised by get_data then raise ImportError.
+ expected_source = self.loader.source.decode('utf-8')
+ self.assertEqual(self.loader.get_source(self.name), expected_source)
+ def raise_IOError(path):
+ raise IOError
+ self.loader.get_data = raise_IOError
+ with self.assertRaises(ImportError) as cm:
+ self.loader.get_source(self.name)
+ self.assertEqual(cm.exception.name, self.name)
+
+ def test_is_package(self):
+ # Properly detect when loading a package.
+ self.setUp(is_package=False)
+ self.assertFalse(self.loader.is_package(self.name))
+ self.setUp(is_package=True)
+ self.assertTrue(self.loader.is_package(self.name))
+ self.assertFalse(self.loader.is_package(self.name + '.__init__'))
+
+ def test_get_code(self):
+ # Verify the code object is created.
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object)
+
+ def test_load_module(self):
+ # Loading a module should set __name__, __loader__, __package__,
+ # __path__ (for packages), __file__, and __cached__.
+ # The module should also be put into sys.modules.
+ with util.uncache(self.name):
+ module = self.loader.load_module(self.name)
+ self.verify_module(module)
+ self.assertEqual(module.__path__, [os.path.dirname(self.path)])
+ self.assertIn(self.name, sys.modules)
+
+ def test_package_settings(self):
+ # __package__ needs to be set, while __path__ is set on if the module
+ # is a package.
+ # Testing the values for a package are covered by test_load_module.
+ self.setUp(is_package=False)
+ with util.uncache(self.name):
+ module = self.loader.load_module(self.name)
+ self.verify_module(module)
+ self.assertTrue(not hasattr(module, '__path__'))
+
+ def test_get_source_encoding(self):
+ # Source is considered encoded in UTF-8 by default unless otherwise
+ # specified by an encoding line.
+ source = "_ = 'ü'"
+ self.loader.source = source.encode('utf-8')
+ returned_source = self.loader.get_source(self.name)
+ self.assertEqual(returned_source, source)
+ source = "# coding: latin-1\n_ = ü"
+ self.loader.source = source.encode('latin-1')
+ returned_source = self.loader.get_source(self.name)
+ self.assertEqual(returned_source, source)
+
+
+@unittest.skipIf(sys.dont_write_bytecode, "sys.dont_write_bytecode is true")
+class SourceLoaderBytecodeTests(SourceLoaderTestHarness):
+
+ """Test importlib.abc.SourceLoader's use of bytecode.
+
+ Source-only testing handled by SourceOnlyLoaderTests.
+
+ """
+
+ loader_mock = SourceLoaderMock
+
+ def verify_code(self, code_object, *, bytecode_written=False):
+ super().verify_code(code_object)
+ if bytecode_written:
+ self.assertIn(self.cached, self.loader.written)
+ data = bytearray(imp.get_magic())
+ data.extend(importlib._w_long(self.loader.source_mtime))
+ data.extend(importlib._w_long(self.loader.source_size))
+ data.extend(marshal.dumps(code_object))
+ self.assertEqual(self.loader.written[self.cached], bytes(data))
+
+ def test_code_with_everything(self):
+ # When everything should work.
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object)
+
+ def test_no_bytecode(self):
+ # If no bytecode exists then move on to the source.
+ self.loader.bytecode_path = "<does not exist>"
+ # Sanity check
+ with self.assertRaises(IOError):
+ bytecode_path = imp.cache_from_source(self.path)
+ self.loader.get_data(bytecode_path)
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object, bytecode_written=True)
+
+ def test_code_bad_timestamp(self):
+ # Bytecode is only used when the timestamp matches the source EXACTLY.
+ for source_mtime in (0, 2):
+ assert source_mtime != self.loader.source_mtime
+ original = self.loader.source_mtime
+ self.loader.source_mtime = source_mtime
+ # If bytecode is used then EOFError would be raised by marshal.
+ self.loader.bytecode = self.loader.bytecode[8:]
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object, bytecode_written=True)
+ self.loader.source_mtime = original
+
+ def test_code_bad_magic(self):
+ # Skip over bytecode with a bad magic number.
+ self.setUp(magic=b'0000')
+ # If bytecode is used then EOFError would be raised by marshal.
+ self.loader.bytecode = self.loader.bytecode[8:]
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object, bytecode_written=True)
+
+ def test_dont_write_bytecode(self):
+ # Bytecode is not written if sys.dont_write_bytecode is true.
+ # Can assume it is false already thanks to the skipIf class decorator.
+ try:
+ sys.dont_write_bytecode = True
+ self.loader.bytecode_path = "<does not exist>"
+ code_object = self.loader.get_code(self.name)
+ self.assertNotIn(self.cached, self.loader.written)
+ finally:
+ sys.dont_write_bytecode = False
+
+ def test_no_set_data(self):
+ # If set_data is not defined, one can still read bytecode.
+ self.setUp(magic=b'0000')
+ original_set_data = self.loader.__class__.set_data
+ try:
+ del self.loader.__class__.set_data
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object)
+ finally:
+ self.loader.__class__.set_data = original_set_data
+
+ def test_set_data_raises_exceptions(self):
+ # Raising NotImplementedError or IOError is okay for set_data.
+ def raise_exception(exc):
+ def closure(*args, **kwargs):
+ raise exc
+ return closure
+
+ self.setUp(magic=b'0000')
+ self.loader.set_data = raise_exception(NotImplementedError)
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object)
+
+
+class SourceLoaderGetSourceTests(unittest.TestCase):
+
+ """Tests for importlib.abc.SourceLoader.get_source()."""
+
+ def test_default_encoding(self):
+ # Should have no problems with UTF-8 text.
+ name = 'mod'
+ mock = SourceOnlyLoaderMock('mod.file')
+ source = 'x = "ü"'
+ mock.source = source.encode('utf-8')
+ returned_source = mock.get_source(name)
+ self.assertEqual(returned_source, source)
+
+ def test_decoded_source(self):
+ # Decoding should work.
+ name = 'mod'
+ mock = SourceOnlyLoaderMock("mod.file")
+ source = "# coding: Latin-1\nx='ü'"
+ assert source.encode('latin-1') != source.encode('utf-8')
+ mock.source = source.encode('latin-1')
+ returned_source = mock.get_source(name)
+ self.assertEqual(returned_source, source)
+
+ def test_universal_newlines(self):
+ # PEP 302 says universal newlines should be used.
+ name = 'mod'
+ mock = SourceOnlyLoaderMock('mod.file')
+ source = "x = 42\r\ny = -13\r\n"
+ mock.source = source.encode('utf-8')
+ expect = io.IncrementalNewlineDecoder(None, True).decode(source)
+ self.assertEqual(mock.get_source(name), expect)
+
+
+class AbstractMethodImplTests(unittest.TestCase):
+
+ """Test the concrete abstractmethod implementations."""
+
+ class MetaPathFinder(abc.MetaPathFinder):
+ def find_module(self, fullname, path):
+ super().find_module(fullname, path)
+
+ class PathEntryFinder(abc.PathEntryFinder):
+ def find_module(self, _):
+ super().find_module(_)
+
+ def find_loader(self, _):
+ super().find_loader(_)
+
+ class Finder(abc.Finder):
+ def find_module(self, fullname, path):
+ super().find_module(fullname, path)
+
+ class Loader(abc.Loader):
+ def load_module(self, fullname):
+ super().load_module(fullname)
+ def module_repr(self, module):
+ super().module_repr(module)
+
+ class ResourceLoader(Loader, abc.ResourceLoader):
+ def get_data(self, _):
+ super().get_data(_)
+
+ class InspectLoader(Loader, abc.InspectLoader):
+ def is_package(self, _):
+ super().is_package(_)
+
+ def get_code(self, _):
+ super().get_code(_)
+
+ def get_source(self, _):
+ super().get_source(_)
+
+ class ExecutionLoader(InspectLoader, abc.ExecutionLoader):
+ def get_filename(self, _):
+ super().get_filename(_)
+
+ class SourceLoader(ResourceLoader, ExecutionLoader, abc.SourceLoader):
+ pass
+
+ class PyLoader(ResourceLoader, InspectLoader, abc.PyLoader):
+ def source_path(self, _):
+ super().source_path(_)
+
+ class PyPycLoader(PyLoader, abc.PyPycLoader):
+ def bytecode_path(self, _):
+ super().bytecode_path(_)
+
+ def source_mtime(self, _):
+ super().source_mtime(_)
+
+ def write_bytecode(self, _, _2):
+ super().write_bytecode(_, _2)
+
+ def raises_NotImplementedError(self, ins, *args):
+ for method_name in args:
+ method = getattr(ins, method_name)
+ arg_count = len(inspect.getfullargspec(method)[0]) - 1
+ args = [''] * arg_count
+ try:
+ method(*args)
+ except NotImplementedError:
+ pass
+ else:
+ msg = "{}.{} did not raise NotImplementedError"
+ self.fail(msg.format(ins.__class__.__name__, method_name))
+
+ def test_Loader(self):
+ self.raises_NotImplementedError(self.Loader(), 'load_module')
+
+ # XXX misplaced; should be somewhere else
+ def test_Finder(self):
+ self.raises_NotImplementedError(self.Finder(), 'find_module')
+
+ def test_ResourceLoader(self):
+ self.raises_NotImplementedError(self.ResourceLoader(), 'load_module',
+ 'get_data')
+
+ def test_InspectLoader(self):
+ self.raises_NotImplementedError(self.InspectLoader(), 'load_module',
+ 'is_package', 'get_code', 'get_source')
+
+ def test_ExecutionLoader(self):
+ self.raises_NotImplementedError(self.ExecutionLoader(), 'load_module',
+ 'is_package', 'get_code', 'get_source',
+ 'get_filename')
+
+ def test_SourceLoader(self):
+ ins = self.SourceLoader()
+ # Required abstractmethods.
+ self.raises_NotImplementedError(ins, 'get_filename', 'get_data')
+ # Optional abstractmethods.
+ self.raises_NotImplementedError(ins,'path_stats', 'set_data')
+
+ def test_PyLoader(self):
+ self.raises_NotImplementedError(self.PyLoader(), 'source_path',
+ 'get_data', 'is_package')
+
+ def test_PyPycLoader(self):
+ self.raises_NotImplementedError(self.PyPycLoader(), 'source_path',
+ 'source_mtime', 'bytecode_path',
+ 'write_bytecode')
+
+
+def test_main():
+ from test.support import run_unittest
+ run_unittest(PyLoaderTests, PyLoaderCompatTests,
+ PyLoaderInterfaceTests,
+ PyPycLoaderTests, PyPycLoaderInterfaceTests,
+ SkipWritingBytecodeTests, RegeneratedBytecodeTests,
+ BadBytecodeFailureTests, MissingPathsTests,
+ SourceOnlyLoaderTests,
+ SourceLoaderBytecodeTests,
+ SourceLoaderGetSourceTests,
+ AbstractMethodImplTests)
+
+
+if __name__ == '__main__':
+ test_main()
diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py
new file mode 100644
index 0000000000..241173fb44
--- /dev/null
+++ b/Lib/test/test_importlib/source/test_case_sensitivity.py
@@ -0,0 +1,70 @@
+"""Test case-sensitivity (PEP 235)."""
+from importlib import _bootstrap
+from importlib import machinery
+from .. import util
+from . import util as source_util
+import imp
+import os
+import sys
+from test import support as test_support
+import unittest
+
+
+@util.case_insensitive_tests
+class CaseSensitivityTest(unittest.TestCase):
+
+ """PEP 235 dictates that on case-preserving, case-insensitive file systems
+ that imports are case-sensitive unless the PYTHONCASEOK environment
+ variable is set."""
+
+ name = 'MoDuLe'
+ assert name != name.lower()
+
+ def find(self, path):
+ finder = machinery.FileFinder(path,
+ (machinery.SourceFileLoader,
+ machinery.SOURCE_SUFFIXES),
+ (machinery.SourcelessFileLoader,
+ machinery.BYTECODE_SUFFIXES))
+ return finder.find_module(self.name)
+
+ def sensitivity_test(self):
+ """Look for a module with matching and non-matching sensitivity."""
+ sensitive_pkg = 'sensitive.{0}'.format(self.name)
+ insensitive_pkg = 'insensitive.{0}'.format(self.name.lower())
+ context = source_util.create_modules(insensitive_pkg, sensitive_pkg)
+ with context as mapping:
+ sensitive_path = os.path.join(mapping['.root'], 'sensitive')
+ insensitive_path = os.path.join(mapping['.root'], 'insensitive')
+ return self.find(sensitive_path), self.find(insensitive_path)
+
+ def test_sensitive(self):
+ with test_support.EnvironmentVarGuard() as env:
+ env.unset('PYTHONCASEOK')
+ if b'PYTHONCASEOK' in _bootstrap._os.environ:
+ self.skipTest('os.environ changes not reflected in '
+ '_os.environ')
+ sensitive, insensitive = self.sensitivity_test()
+ self.assertTrue(hasattr(sensitive, 'load_module'))
+ self.assertIn(self.name, sensitive.get_filename(self.name))
+ self.assertIsNone(insensitive)
+
+ def test_insensitive(self):
+ with test_support.EnvironmentVarGuard() as env:
+ env.set('PYTHONCASEOK', '1')
+ if b'PYTHONCASEOK' not in _bootstrap._os.environ:
+ self.skipTest('os.environ changes not reflected in '
+ '_os.environ')
+ sensitive, insensitive = self.sensitivity_test()
+ self.assertTrue(hasattr(sensitive, 'load_module'))
+ self.assertIn(self.name, sensitive.get_filename(self.name))
+ self.assertTrue(hasattr(insensitive, 'load_module'))
+ self.assertIn(self.name, insensitive.get_filename(self.name))
+
+
+def test_main():
+ test_support.run_unittest(CaseSensitivityTest)
+
+
+if __name__ == '__main__':
+ test_main()
diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py
new file mode 100644
index 0000000000..90f9d30129
--- /dev/null
+++ b/Lib/test/test_importlib/source/test_file_loader.py
@@ -0,0 +1,484 @@
+from importlib import machinery
+import importlib
+import importlib.abc
+from .. import abc
+from .. import util
+from . import util as source_util
+
+import errno
+import imp
+import marshal
+import os
+import py_compile
+import shutil
+import stat
+import sys
+import unittest
+
+from test.support import make_legacy_pyc
+
+
+class SimpleTest(unittest.TestCase):
+
+ """Should have no issue importing a source module [basic]. And if there is
+ a syntax error, it should raise a SyntaxError [syntax error].
+
+ """
+
+ def test_load_module_API(self):
+ # If fullname is not specified that assume self.name is desired.
+ class TesterMixin(importlib.abc.Loader):
+ def load_module(self, fullname): return fullname
+ def module_repr(self, module): return '<module>'
+
+ class Tester(importlib.abc.FileLoader, TesterMixin):
+ def get_code(self, _): pass
+ def get_source(self, _): pass
+ def is_package(self, _): pass
+
+ name = 'mod_name'
+ loader = Tester(name, 'some_path')
+ self.assertEqual(name, loader.load_module())
+ self.assertEqual(name, loader.load_module(None))
+ self.assertEqual(name, loader.load_module(name))
+ with self.assertRaises(ImportError):
+ loader.load_module(loader.name + 'XXX')
+
+ def test_get_filename_API(self):
+ # If fullname is not set then assume self.path is desired.
+ class Tester(importlib.abc.FileLoader):
+ def get_code(self, _): pass
+ def get_source(self, _): pass
+ def is_package(self, _): pass
+ def module_repr(self, _): pass
+
+ path = 'some_path'
+ name = 'some_name'
+ loader = Tester(name, path)
+ self.assertEqual(path, loader.get_filename(name))
+ self.assertEqual(path, loader.get_filename())
+ self.assertEqual(path, loader.get_filename(None))
+ with self.assertRaises(ImportError):
+ loader.get_filename(name + 'XXX')
+
+ # [basic]
+ def test_module(self):
+ with source_util.create_modules('_temp') as mapping:
+ loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
+ module = loader.load_module('_temp')
+ self.assertIn('_temp', sys.modules)
+ check = {'__name__': '_temp', '__file__': mapping['_temp'],
+ '__package__': ''}
+ for attr, value in check.items():
+ self.assertEqual(getattr(module, attr), value)
+
+ def test_package(self):
+ with source_util.create_modules('_pkg.__init__') as mapping:
+ loader = machinery.SourceFileLoader('_pkg',
+ mapping['_pkg.__init__'])
+ module = loader.load_module('_pkg')
+ self.assertIn('_pkg', sys.modules)
+ check = {'__name__': '_pkg', '__file__': mapping['_pkg.__init__'],
+ '__path__': [os.path.dirname(mapping['_pkg.__init__'])],
+ '__package__': '_pkg'}
+ for attr, value in check.items():
+ self.assertEqual(getattr(module, attr), value)
+
+
+ def test_lacking_parent(self):
+ with source_util.create_modules('_pkg.__init__', '_pkg.mod')as mapping:
+ loader = machinery.SourceFileLoader('_pkg.mod',
+ mapping['_pkg.mod'])
+ module = loader.load_module('_pkg.mod')
+ self.assertIn('_pkg.mod', sys.modules)
+ check = {'__name__': '_pkg.mod', '__file__': mapping['_pkg.mod'],
+ '__package__': '_pkg'}
+ for attr, value in check.items():
+ self.assertEqual(getattr(module, attr), value)
+
+ def fake_mtime(self, fxn):
+ """Fake mtime to always be higher than expected."""
+ return lambda name: fxn(name) + 1
+
+ def test_module_reuse(self):
+ with source_util.create_modules('_temp') as mapping:
+ loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
+ module = loader.load_module('_temp')
+ module_id = id(module)
+ module_dict_id = id(module.__dict__)
+ with open(mapping['_temp'], 'w') as file:
+ file.write("testing_var = 42\n")
+ module = loader.load_module('_temp')
+ self.assertIn('testing_var', module.__dict__,
+ "'testing_var' not in "
+ "{0}".format(list(module.__dict__.keys())))
+ self.assertEqual(module, sys.modules['_temp'])
+ self.assertEqual(id(module), module_id)
+ self.assertEqual(id(module.__dict__), module_dict_id)
+
+ def test_state_after_failure(self):
+ # A failed reload should leave the original module intact.
+ attributes = ('__file__', '__path__', '__package__')
+ value = '<test>'
+ name = '_temp'
+ with source_util.create_modules(name) as mapping:
+ orig_module = imp.new_module(name)
+ for attr in attributes:
+ setattr(orig_module, attr, value)
+ with open(mapping[name], 'w') as file:
+ file.write('+++ bad syntax +++')
+ loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
+ with self.assertRaises(SyntaxError):
+ loader.load_module(name)
+ for attr in attributes:
+ self.assertEqual(getattr(orig_module, attr), value)
+
+ # [syntax error]
+ def test_bad_syntax(self):
+ with source_util.create_modules('_temp') as mapping:
+ with open(mapping['_temp'], 'w') as file:
+ file.write('=')
+ loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
+ with self.assertRaises(SyntaxError):
+ loader.load_module('_temp')
+ self.assertNotIn('_temp', sys.modules)
+
+ def test_file_from_empty_string_dir(self):
+ # Loading a module found from an empty string entry on sys.path should
+ # not only work, but keep all attributes relative.
+ file_path = '_temp.py'
+ with open(file_path, 'w') as file:
+ file.write("# test file for importlib")
+ try:
+ with util.uncache('_temp'):
+ loader = machinery.SourceFileLoader('_temp', file_path)
+ mod = loader.load_module('_temp')
+ self.assertEqual(file_path, mod.__file__)
+ self.assertEqual(imp.cache_from_source(file_path),
+ mod.__cached__)
+ finally:
+ os.unlink(file_path)
+ pycache = os.path.dirname(imp.cache_from_source(file_path))
+ shutil.rmtree(pycache)
+
+ def test_timestamp_overflow(self):
+ # When a modification timestamp is larger than 2**32, it should be
+ # truncated rather than raise an OverflowError.
+ with source_util.create_modules('_temp') as mapping:
+ source = mapping['_temp']
+ compiled = imp.cache_from_source(source)
+ with open(source, 'w') as f:
+ f.write("x = 5")
+ try:
+ os.utime(source, (2 ** 33 - 5, 2 ** 33 - 5))
+ except OverflowError:
+ self.skipTest("cannot set modification time to large integer")
+ except OSError as e:
+ if e.errno != getattr(errno, 'EOVERFLOW', None):
+ raise
+ self.skipTest("cannot set modification time to large integer ({})".format(e))
+ loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
+ mod = loader.load_module('_temp')
+ # Sanity checks.
+ self.assertEqual(mod.__cached__, compiled)
+ self.assertEqual(mod.x, 5)
+ # The pyc file was created.
+ os.stat(compiled)
+
+
+class BadBytecodeTest(unittest.TestCase):
+
+ def import_(self, file, module_name):
+ loader = self.loader(module_name, file)
+ module = loader.load_module(module_name)
+ self.assertIn(module_name, sys.modules)
+
+ def manipulate_bytecode(self, name, mapping, manipulator, *,
+ del_source=False):
+ """Manipulate the bytecode of a module by passing it into a callable
+ that returns what to use as the new bytecode."""
+ try:
+ del sys.modules['_temp']
+ except KeyError:
+ pass
+ py_compile.compile(mapping[name])
+ if not del_source:
+ bytecode_path = imp.cache_from_source(mapping[name])
+ else:
+ os.unlink(mapping[name])
+ bytecode_path = make_legacy_pyc(mapping[name])
+ if manipulator:
+ with open(bytecode_path, 'rb') as file:
+ bc = file.read()
+ new_bc = manipulator(bc)
+ with open(bytecode_path, 'wb') as file:
+ if new_bc is not None:
+ file.write(new_bc)
+ return bytecode_path
+
+ def _test_empty_file(self, test, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: b'',
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
+ @source_util.writes_bytecode_files
+ def _test_partial_magic(self, test, *, del_source=False):
+ # When their are less than 4 bytes to a .pyc, regenerate it if
+ # possible, else raise ImportError.
+ with source_util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:3],
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
+ def _test_magic_only(self, test, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:4],
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
+ def _test_partial_timestamp(self, test, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:7],
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
+ def _test_partial_size(self, test, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:11],
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
+ def _test_no_marshal(self, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:12],
+ del_source=del_source)
+ file_path = mapping['_temp'] if not del_source else bc_path
+ with self.assertRaises(EOFError):
+ self.import_(file_path, '_temp')
+
+ def _test_non_code_marshal(self, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bytecode_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:12] + marshal.dumps(b'abcd'),
+ del_source=del_source)
+ file_path = mapping['_temp'] if not del_source else bytecode_path
+ with self.assertRaises(ImportError) as cm:
+ self.import_(file_path, '_temp')
+ self.assertEqual(cm.exception.name, '_temp')
+ self.assertEqual(cm.exception.path, bytecode_path)
+
+ def _test_bad_marshal(self, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bytecode_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:12] + b'<test>',
+ del_source=del_source)
+ file_path = mapping['_temp'] if not del_source else bytecode_path
+ with self.assertRaises(EOFError):
+ self.import_(file_path, '_temp')
+
+ def _test_bad_magic(self, test, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: b'\x00\x00\x00\x00' + bc[4:])
+ test('_temp', mapping, bc_path)
+
+
+class SourceLoaderBadBytecodeTest(BadBytecodeTest):
+
+ loader = machinery.SourceFileLoader
+
+ @source_util.writes_bytecode_files
+ def test_empty_file(self):
+ # When a .pyc is empty, regenerate it if possible, else raise
+ # ImportError.
+ def test(name, mapping, bytecode_path):
+ self.import_(mapping[name], name)
+ with open(bytecode_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 12)
+
+ self._test_empty_file(test)
+
+ def test_partial_magic(self):
+ def test(name, mapping, bytecode_path):
+ self.import_(mapping[name], name)
+ with open(bytecode_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 12)
+
+ self._test_partial_magic(test)
+
+ @source_util.writes_bytecode_files
+ def test_magic_only(self):
+ # When there is only the magic number, regenerate the .pyc if possible,
+ # else raise EOFError.
+ def test(name, mapping, bytecode_path):
+ self.import_(mapping[name], name)
+ with open(bytecode_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 12)
+
+ self._test_magic_only(test)
+
+ @source_util.writes_bytecode_files
+ def test_bad_magic(self):
+ # When the magic number is different, the bytecode should be
+ # regenerated.
+ def test(name, mapping, bytecode_path):
+ self.import_(mapping[name], name)
+ with open(bytecode_path, 'rb') as bytecode_file:
+ self.assertEqual(bytecode_file.read(4), imp.get_magic())
+
+ self._test_bad_magic(test)
+
+ @source_util.writes_bytecode_files
+ def test_partial_timestamp(self):
+ # When the timestamp is partial, regenerate the .pyc, else
+ # raise EOFError.
+ def test(name, mapping, bc_path):
+ self.import_(mapping[name], name)
+ with open(bc_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 12)
+
+ self._test_partial_timestamp(test)
+
+ @source_util.writes_bytecode_files
+ def test_partial_size(self):
+ # When the size is partial, regenerate the .pyc, else
+ # raise EOFError.
+ def test(name, mapping, bc_path):
+ self.import_(mapping[name], name)
+ with open(bc_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 12)
+
+ self._test_partial_size(test)
+
+ @source_util.writes_bytecode_files
+ def test_no_marshal(self):
+ # When there is only the magic number and timestamp, raise EOFError.
+ self._test_no_marshal()
+
+ @source_util.writes_bytecode_files
+ def test_non_code_marshal(self):
+ self._test_non_code_marshal()
+ # XXX ImportError when sourceless
+
+ # [bad marshal]
+ @source_util.writes_bytecode_files
+ def test_bad_marshal(self):
+ # Bad marshal data should raise a ValueError.
+ self._test_bad_marshal()
+
+ # [bad timestamp]
+ @source_util.writes_bytecode_files
+ def test_old_timestamp(self):
+ # When the timestamp is older than the source, bytecode should be
+ # regenerated.
+ zeros = b'\x00\x00\x00\x00'
+ with source_util.create_modules('_temp') as mapping:
+ py_compile.compile(mapping['_temp'])
+ bytecode_path = imp.cache_from_source(mapping['_temp'])
+ with open(bytecode_path, 'r+b') as bytecode_file:
+ bytecode_file.seek(4)
+ bytecode_file.write(zeros)
+ self.import_(mapping['_temp'], '_temp')
+ source_mtime = os.path.getmtime(mapping['_temp'])
+ source_timestamp = importlib._w_long(source_mtime)
+ with open(bytecode_path, 'rb') as bytecode_file:
+ bytecode_file.seek(4)
+ self.assertEqual(bytecode_file.read(4), source_timestamp)
+
+ # [bytecode read-only]
+ @source_util.writes_bytecode_files
+ def test_read_only_bytecode(self):
+ # When bytecode is read-only but should be rewritten, fail silently.
+ with source_util.create_modules('_temp') as mapping:
+ # Create bytecode that will need to be re-created.
+ py_compile.compile(mapping['_temp'])
+ bytecode_path = imp.cache_from_source(mapping['_temp'])
+ with open(bytecode_path, 'r+b') as bytecode_file:
+ bytecode_file.seek(0)
+ bytecode_file.write(b'\x00\x00\x00\x00')
+ # Make the bytecode read-only.
+ os.chmod(bytecode_path,
+ stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
+ try:
+ # Should not raise IOError!
+ self.import_(mapping['_temp'], '_temp')
+ finally:
+ # Make writable for eventual clean-up.
+ os.chmod(bytecode_path, stat.S_IWUSR)
+
+
+class SourcelessLoaderBadBytecodeTest(BadBytecodeTest):
+
+ loader = machinery.SourcelessFileLoader
+
+ def test_empty_file(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(ImportError) as cm:
+ self.import_(bytecode_path, name)
+ self.assertEqual(cm.exception.name, name)
+ self.assertEqual(cm.exception.path, bytecode_path)
+
+ self._test_empty_file(test, del_source=True)
+
+ def test_partial_magic(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(ImportError) as cm:
+ self.import_(bytecode_path, name)
+ self.assertEqual(cm.exception.name, name)
+ self.assertEqual(cm.exception.path, bytecode_path)
+ self._test_partial_magic(test, del_source=True)
+
+ def test_magic_only(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(EOFError):
+ self.import_(bytecode_path, name)
+
+ self._test_magic_only(test, del_source=True)
+
+ def test_bad_magic(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(ImportError) as cm:
+ self.import_(bytecode_path, name)
+ self.assertEqual(cm.exception.name, name)
+ self.assertEqual(cm.exception.path, bytecode_path)
+
+ self._test_bad_magic(test, del_source=True)
+
+ def test_partial_timestamp(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(EOFError):
+ self.import_(bytecode_path, name)
+
+ self._test_partial_timestamp(test, del_source=True)
+
+ def test_partial_size(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(EOFError):
+ self.import_(bytecode_path, name)
+
+ self._test_partial_size(test, del_source=True)
+
+ def test_no_marshal(self):
+ self._test_no_marshal(del_source=True)
+
+ def test_non_code_marshal(self):
+ self._test_non_code_marshal(del_source=True)
+
+
+def test_main():
+ from test.support import run_unittest
+ run_unittest(SimpleTest,
+ SourceLoaderBadBytecodeTest,
+ SourcelessLoaderBadBytecodeTest
+ )
+
+
+if __name__ == '__main__':
+ test_main()
diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py
new file mode 100644
index 0000000000..8e4986835d
--- /dev/null
+++ b/Lib/test/test_importlib/source/test_finder.py
@@ -0,0 +1,191 @@
+from .. import abc
+from . import util as source_util
+
+from importlib import machinery
+import errno
+import imp
+import os
+import py_compile
+import stat
+import sys
+import tempfile
+from test.support import make_legacy_pyc
+import unittest
+import warnings
+
+
+class FinderTests(abc.FinderTests):
+
+ """For a top-level module, it should just be found directly in the
+ directory being searched. This is true for a directory with source
+ [top-level source], bytecode [top-level bc], or both [top-level both].
+ There is also the possibility that it is a package [top-level package], in
+ which case there will be a directory with the module name and an
+ __init__.py file. If there is a directory without an __init__.py an
+ ImportWarning is returned [empty dir].
+
+ For sub-modules and sub-packages, the same happens as above but only use
+ the tail end of the name [sub module] [sub package] [sub empty].
+
+ When there is a conflict between a package and module having the same name
+ in the same directory, the package wins out [package over module]. This is
+ so that imports of modules within the package can occur rather than trigger
+ an import error.
+
+ When there is a package and module with the same name, always pick the
+ package over the module [package over module]. This is so that imports from
+ the package have the possibility of succeeding.
+
+ """
+
+ def get_finder(self, root):
+ loader_details = [(machinery.SourceFileLoader,
+ machinery.SOURCE_SUFFIXES),
+ (machinery.SourcelessFileLoader,
+ machinery.BYTECODE_SUFFIXES)]
+ return machinery.FileFinder(root, *loader_details)
+
+ def import_(self, root, module):
+ return self.get_finder(root).find_module(module)
+
+ def run_test(self, test, create=None, *, compile_=None, unlink=None):
+ """Test the finding of 'test' with the creation of modules listed in
+ 'create'.
+
+ Any names listed in 'compile_' are byte-compiled. Modules
+ listed in 'unlink' have their source files deleted.
+
+ """
+ if create is None:
+ create = {test}
+ with source_util.create_modules(*create) as mapping:
+ if compile_:
+ for name in compile_:
+ py_compile.compile(mapping[name])
+ if unlink:
+ for name in unlink:
+ os.unlink(mapping[name])
+ try:
+ make_legacy_pyc(mapping[name])
+ except OSError as error:
+ # Some tests do not set compile_=True so the source
+ # module will not get compiled and there will be no
+ # PEP 3147 pyc file to rename.
+ if error.errno != errno.ENOENT:
+ raise
+ loader = self.import_(mapping['.root'], test)
+ self.assertTrue(hasattr(loader, 'load_module'))
+ return loader
+
+ def test_module(self):
+ # [top-level source]
+ self.run_test('top_level')
+ # [top-level bc]
+ self.run_test('top_level', compile_={'top_level'},
+ unlink={'top_level'})
+ # [top-level both]
+ self.run_test('top_level', compile_={'top_level'})
+
+ # [top-level package]
+ def test_package(self):
+ # Source.
+ self.run_test('pkg', {'pkg.__init__'})
+ # Bytecode.
+ self.run_test('pkg', {'pkg.__init__'}, compile_={'pkg.__init__'},
+ unlink={'pkg.__init__'})
+ # Both.
+ self.run_test('pkg', {'pkg.__init__'}, compile_={'pkg.__init__'})
+
+ # [sub module]
+ def test_module_in_package(self):
+ with source_util.create_modules('pkg.__init__', 'pkg.sub') as mapping:
+ pkg_dir = os.path.dirname(mapping['pkg.__init__'])
+ loader = self.import_(pkg_dir, 'pkg.sub')
+ self.assertTrue(hasattr(loader, 'load_module'))
+
+ # [sub package]
+ def test_package_in_package(self):
+ context = source_util.create_modules('pkg.__init__', 'pkg.sub.__init__')
+ with context as mapping:
+ pkg_dir = os.path.dirname(mapping['pkg.__init__'])
+ loader = self.import_(pkg_dir, 'pkg.sub')
+ self.assertTrue(hasattr(loader, 'load_module'))
+
+ # [package over modules]
+ def test_package_over_module(self):
+ name = '_temp'
+ loader = self.run_test(name, {'{0}.__init__'.format(name), name})
+ self.assertIn('__init__', loader.get_filename(name))
+
+ def test_failure(self):
+ with source_util.create_modules('blah') as mapping:
+ nothing = self.import_(mapping['.root'], 'sdfsadsadf')
+ self.assertIsNone(nothing)
+
+ def test_empty_string_for_dir(self):
+ # The empty string from sys.path means to search in the cwd.
+ finder = machinery.FileFinder('', (machinery.SourceFileLoader,
+ machinery.SOURCE_SUFFIXES))
+ with open('mod.py', 'w') as file:
+ file.write("# test file for importlib")
+ try:
+ loader = finder.find_module('mod')
+ self.assertTrue(hasattr(loader, 'load_module'))
+ finally:
+ os.unlink('mod.py')
+
+ def test_invalidate_caches(self):
+ # invalidate_caches() should reset the mtime.
+ finder = machinery.FileFinder('', (machinery.SourceFileLoader,
+ machinery.SOURCE_SUFFIXES))
+ finder._path_mtime = 42
+ finder.invalidate_caches()
+ self.assertEqual(finder._path_mtime, -1)
+
+ # Regression test for http://bugs.python.org/issue14846
+ def test_dir_removal_handling(self):
+ mod = 'mod'
+ with source_util.create_modules(mod) as mapping:
+ finder = self.get_finder(mapping['.root'])
+ self.assertIsNotNone(finder.find_module(mod))
+ self.assertIsNone(finder.find_module(mod))
+
+ @unittest.skipUnless(sys.platform != 'win32',
+ 'os.chmod() does not support the needed arguments under Windows')
+ def test_no_read_directory(self):
+ # Issue #16730
+ tempdir = tempfile.TemporaryDirectory()
+ original_mode = os.stat(tempdir.name).st_mode
+ def cleanup(tempdir):
+ """Cleanup function for the temporary directory.
+
+ Since we muck with the permissions, we want to set them back to
+ their original values to make sure the directory can be properly
+ cleaned up.
+
+ """
+ os.chmod(tempdir.name, original_mode)
+ # If this is not explicitly called then the __del__ method is used,
+ # but since already mucking around might as well explicitly clean
+ # up.
+ tempdir.__exit__(None, None, None)
+ self.addCleanup(cleanup, tempdir)
+ os.chmod(tempdir.name, stat.S_IWUSR | stat.S_IXUSR)
+ finder = self.get_finder(tempdir.name)
+ self.assertEqual((None, []), finder.find_loader('doesnotexist'))
+
+ def test_ignore_file(self):
+ # If a directory got changed to a file from underneath us, then don't
+ # worry about looking for submodules.
+ with tempfile.NamedTemporaryFile() as file_obj:
+ finder = self.get_finder(file_obj.name)
+ self.assertEqual((None, []), finder.find_loader('doesnotexist'))
+
+
+def test_main():
+ from test.support import run_unittest
+ run_unittest(FinderTests)
+
+
+if __name__ == '__main__':
+ test_main()
diff --git a/Lib/test/test_importlib/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py
new file mode 100644
index 0000000000..6a78792f07
--- /dev/null
+++ b/Lib/test/test_importlib/source/test_path_hook.py
@@ -0,0 +1,32 @@
+from . import util as source_util
+
+from importlib import machinery
+import imp
+import unittest
+
+
+class PathHookTest(unittest.TestCase):
+
+ """Test the path hook for source."""
+
+ def path_hook(self):
+ return machinery.FileFinder.path_hook((machinery.SourceFileLoader,
+ machinery.SOURCE_SUFFIXES))
+
+ def test_success(self):
+ with source_util.create_modules('dummy') as mapping:
+ self.assertTrue(hasattr(self.path_hook()(mapping['.root']),
+ 'find_module'))
+
+ def test_empty_string(self):
+ # The empty string represents the cwd.
+ self.assertTrue(hasattr(self.path_hook()(''), 'find_module'))
+
+
+def test_main():
+ from test.support import run_unittest
+ run_unittest(PathHookTest)
+
+
+if __name__ == '__main__':
+ test_main()
diff --git a/Lib/test/test_importlib/source/test_source_encoding.py b/Lib/test/test_importlib/source/test_source_encoding.py
new file mode 100644
index 0000000000..0ca5195439
--- /dev/null
+++ b/Lib/test/test_importlib/source/test_source_encoding.py
@@ -0,0 +1,123 @@
+from . import util as source_util
+
+from importlib import _bootstrap
+import codecs
+import re
+import sys
+# Because sys.path gets essentially blanked, need to have unicodedata already
+# imported for the parser to use.
+import unicodedata
+import unittest
+
+
+CODING_RE = re.compile(r'coding[:=]\s*([-\w.]+)')
+
+
+class EncodingTest(unittest.TestCase):
+
+ """PEP 3120 makes UTF-8 the default encoding for source code
+ [default encoding].
+
+ PEP 263 specifies how that can change on a per-file basis. Either the first
+ or second line can contain the encoding line [encoding first line]
+ encoding second line]. If the file has the BOM marker it is considered UTF-8
+ implicitly [BOM]. If any encoding is specified it must be UTF-8, else it is
+ an error [BOM and utf-8][BOM conflict].
+
+ """
+
+ variable = '\u00fc'
+ character = '\u00c9'
+ source_line = "{0} = '{1}'\n".format(variable, character)
+ module_name = '_temp'
+
+ def run_test(self, source):
+ with source_util.create_modules(self.module_name) as mapping:
+ with open(mapping[self.module_name], 'wb') as file:
+ file.write(source)
+ loader = _bootstrap.SourceFileLoader(self.module_name,
+ mapping[self.module_name])
+ return loader.load_module(self.module_name)
+
+ def create_source(self, encoding):
+ encoding_line = "# coding={0}".format(encoding)
+ assert CODING_RE.search(encoding_line)
+ source_lines = [encoding_line.encode('utf-8')]
+ source_lines.append(self.source_line.encode(encoding))
+ return b'\n'.join(source_lines)
+
+ def test_non_obvious_encoding(self):
+ # Make sure that an encoding that has never been a standard one for
+ # Python works.
+ encoding_line = "# coding=koi8-r"
+ assert CODING_RE.search(encoding_line)
+ source = "{0}\na=42\n".format(encoding_line).encode("koi8-r")
+ self.run_test(source)
+
+ # [default encoding]
+ def test_default_encoding(self):
+ self.run_test(self.source_line.encode('utf-8'))
+
+ # [encoding first line]
+ def test_encoding_on_first_line(self):
+ encoding = 'Latin-1'
+ source = self.create_source(encoding)
+ self.run_test(source)
+
+ # [encoding second line]
+ def test_encoding_on_second_line(self):
+ source = b"#/usr/bin/python\n" + self.create_source('Latin-1')
+ self.run_test(source)
+
+ # [BOM]
+ def test_bom(self):
+ self.run_test(codecs.BOM_UTF8 + self.source_line.encode('utf-8'))
+
+ # [BOM and utf-8]
+ def test_bom_and_utf_8(self):
+ source = codecs.BOM_UTF8 + self.create_source('utf-8')
+ self.run_test(source)
+
+ # [BOM conflict]
+ def test_bom_conflict(self):
+ source = codecs.BOM_UTF8 + self.create_source('latin-1')
+ with self.assertRaises(SyntaxError):
+ self.run_test(source)
+
+
+class LineEndingTest(unittest.TestCase):
+
+ r"""Source written with the three types of line endings (\n, \r\n, \r)
+ need to be readable [cr][crlf][lf]."""
+
+ def run_test(self, line_ending):
+ module_name = '_temp'
+ source_lines = [b"a = 42", b"b = -13", b'']
+ source = line_ending.join(source_lines)
+ with source_util.create_modules(module_name) as mapping:
+ with open(mapping[module_name], 'wb') as file:
+ file.write(source)
+ loader = _bootstrap.SourceFileLoader(module_name,
+ mapping[module_name])
+ return loader.load_module(module_name)
+
+ # [cr]
+ def test_cr(self):
+ self.run_test(b'\r')
+
+ # [crlf]
+ def test_crlf(self):
+ self.run_test(b'\r\n')
+
+ # [lf]
+ def test_lf(self):
+ self.run_test(b'\n')
+
+
+def test_main():
+ from test.support import run_unittest
+ run_unittest(EncodingTest, LineEndingTest)
+
+
+if __name__ == '__main__':
+ test_main()
diff --git a/Lib/test/test_importlib/source/util.py b/Lib/test/test_importlib/source/util.py
new file mode 100644
index 0000000000..ae65663a67
--- /dev/null
+++ b/Lib/test/test_importlib/source/util.py
@@ -0,0 +1,97 @@
+from .. import util
+import contextlib
+import errno
+import functools
+import imp
+import os
+import os.path
+import sys
+import tempfile
+from test import support
+
+
+def writes_bytecode_files(fxn):
+ """Decorator to protect sys.dont_write_bytecode from mutation and to skip
+ tests that require it to be set to False."""
+ if sys.dont_write_bytecode:
+ return lambda *args, **kwargs: None
+ @functools.wraps(fxn)
+ def wrapper(*args, **kwargs):
+ original = sys.dont_write_bytecode
+ sys.dont_write_bytecode = False
+ try:
+ to_return = fxn(*args, **kwargs)
+ finally:
+ sys.dont_write_bytecode = original
+ return to_return
+ return wrapper
+
+
+def ensure_bytecode_path(bytecode_path):
+ """Ensure that the __pycache__ directory for PEP 3147 pyc file exists.
+
+ :param bytecode_path: File system path to PEP 3147 pyc file.
+ """
+ try:
+ os.mkdir(os.path.dirname(bytecode_path))
+ except OSError as error:
+ if error.errno != errno.EEXIST:
+ raise
+
+
+@contextlib.contextmanager
+def create_modules(*names):
+ """Temporarily create each named module with an attribute (named 'attr')
+ that contains the name passed into the context manager that caused the
+ creation of the module.
+
+ All files are created in a temporary directory returned by
+ tempfile.mkdtemp(). This directory is inserted at the beginning of
+ sys.path. When the context manager exits all created files (source and
+ bytecode) are explicitly deleted.
+
+ No magic is performed when creating packages! This means that if you create
+ a module within a package you must also create the package's __init__ as
+ well.
+
+ """
+ source = 'attr = {0!r}'
+ created_paths = []
+ mapping = {}
+ state_manager = None
+ uncache_manager = None
+ try:
+ temp_dir = tempfile.mkdtemp()
+ mapping['.root'] = temp_dir
+ import_names = set()
+ for name in names:
+ if not name.endswith('__init__'):
+ import_name = name
+ else:
+ import_name = name[:-len('.__init__')]
+ import_names.add(import_name)
+ if import_name in sys.modules:
+ del sys.modules[import_name]
+ name_parts = name.split('.')
+ file_path = temp_dir
+ for directory in name_parts[:-1]:
+ file_path = os.path.join(file_path, directory)
+ if not os.path.exists(file_path):
+ os.mkdir(file_path)
+ created_paths.append(file_path)
+ file_path = os.path.join(file_path, name_parts[-1] + '.py')
+ with open(file_path, 'w') as file:
+ file.write(source.format(name))
+ created_paths.append(file_path)
+ mapping[name] = file_path
+ uncache_manager = util.uncache(*import_names)
+ uncache_manager.__enter__()
+ state_manager = util.import_state(path=[temp_dir])
+ state_manager.__enter__()
+ yield mapping
+ finally:
+ if state_manager is not None:
+ state_manager.__exit__(None, None, None)
+ if uncache_manager is not None:
+ uncache_manager.__exit__(None, None, None)
+ support.rmtree(temp_dir)
View Lorry Controller Status