diff options
| author | Serhiy Storchaka <storchaka@gmail.com> | 2013-10-14 20:06:04 +0300 |
|---|---|---|
| committer | Serhiy Storchaka <storchaka@gmail.com> | 2013-10-14 20:06:04 +0300 |
| commit | 0238c23966a751574824604df5dbb612c21737db (patch) | |
| tree | 24b018c43e3a00ac3efa559b2b6a07a079c0a884 /Lib/test | |
| parent | 872e3b5f3130fa414717525ce5e315cc89284d94 (diff) | |
| parent | 2417e2346f8b14c2f0de2524a57ca65c40e3c2e2 (diff) | |
| download | cpython-0238c23966a751574824604df5dbb612c21737db.tar.gz | |
Issue #18919: Check warnings messages in the aifc module tests.
Diffstat (limited to 'Lib/test')
235 files changed, 14380 insertions, 4789 deletions
diff --git a/Lib/test/__main__.py b/Lib/test/__main__.py index ce5615b889..d5fbe159d7 100644 --- a/Lib/test/__main__.py +++ b/Lib/test/__main__.py @@ -1,13 +1,3 @@ -from test import regrtest, support +from test import regrtest - -TEMPDIR, TESTCWD = regrtest._make_temp_dir_for_build(regrtest.TEMPDIR) -regrtest.TEMPDIR = TEMPDIR -regrtest.TESTCWD = TESTCWD - -# Run the tests in a context manager that temporary changes the CWD to a -# temporary and writable directory. If it's not possible to create or -# change the CWD, the original CWD will be used. The original CWD is -# available from support.SAVEDCWD. -with support.temp_cwd(TESTCWD, quiet=True): - regrtest.main() +regrtest.main_in_temp_cwd() diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 2b3ddcfcda..0324afa7ae 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -43,7 +43,7 @@ from multiprocessing import util try: from multiprocessing import reduction - HAS_REDUCTION = True + HAS_REDUCTION = reduction.HAVE_SEND_HANDLE except ImportError: HAS_REDUCTION = False @@ -99,6 +99,9 @@ try: except: MAXFD = 256 +# To speed up tests when using the forkserver, we can preload these: +PRELOAD = ['__main__', 'test.test_multiprocessing_forkserver'] + # # Some tests require ctypes # @@ -342,7 +345,6 @@ class _TestProcess(BaseTestCase): @classmethod def _test_recursion(cls, wconn, id): - from multiprocessing import forking wconn.send(id) if len(id) < 2: for i in range(2): @@ -390,7 +392,7 @@ class _TestProcess(BaseTestCase): self.assertFalse(wait_for_handle(sentinel, timeout=0.0)) event.set() p.join() - self.assertTrue(wait_for_handle(sentinel, timeout=DELTA)) + self.assertTrue(wait_for_handle(sentinel, timeout=1)) # # @@ -1769,6 +1771,35 @@ class _TestPool(BaseTestCase): self.assertEqual(r.get(), expected) self.assertRaises(ValueError, p.map_async, sqr, L) + @classmethod + def _test_traceback(cls): + raise RuntimeError(123) # some comment + + def test_traceback(self): + # We want ensure that the traceback from the child process is + # contained in the traceback raised in the main process. + if self.TYPE == 'processes': + with self.Pool(1) as p: + try: + p.apply(self._test_traceback) + except Exception as e: + exc = e + else: + raise AssertionError('expected RuntimeError') + self.assertIs(type(exc), RuntimeError) + self.assertEqual(exc.args, (123,)) + cause = exc.__cause__ + self.assertIs(type(cause), multiprocessing.pool.RemoteTraceback) + self.assertIn('raise RuntimeError(123) # some comment', cause.tb) + + with test.support.captured_stderr() as f1: + try: + raise exc + except RuntimeError: + sys.excepthook(*sys.exc_info()) + self.assertIn('raise RuntimeError(123) # some comment', + f1.getvalue()) + def raising(): raise KeyError("key") @@ -2476,7 +2507,7 @@ class _TestPicklingConnections(BaseTestCase): @classmethod def tearDownClass(cls): - from multiprocessing.reduction import resource_sharer + from multiprocessing import resource_sharer resource_sharer.stop(timeout=5) @classmethod @@ -2790,30 +2821,40 @@ class _TestFinalize(BaseTestCase): # Test that from ... import * works for each module # -class _TestImportStar(BaseTestCase): +class _TestImportStar(unittest.TestCase): - ALLOWED_TYPES = ('processes',) + def get_module_names(self): + import glob + folder = os.path.dirname(multiprocessing.__file__) + pattern = os.path.join(folder, '*.py') + files = glob.glob(pattern) + modules = [os.path.splitext(os.path.split(f)[1])[0] for f in files] + modules = ['multiprocessing.' + m for m in modules] + modules.remove('multiprocessing.__init__') + modules.append('multiprocessing') + return modules def test_import(self): - modules = [ - 'multiprocessing', 'multiprocessing.connection', - 'multiprocessing.heap', 'multiprocessing.managers', - 'multiprocessing.pool', 'multiprocessing.process', - 'multiprocessing.synchronize', 'multiprocessing.util' - ] - - if HAS_REDUCTION: - modules.append('multiprocessing.reduction') + modules = self.get_module_names() + if sys.platform == 'win32': + modules.remove('multiprocessing.popen_fork') + modules.remove('multiprocessing.popen_forkserver') + modules.remove('multiprocessing.popen_spawn_posix') + else: + modules.remove('multiprocessing.popen_spawn_win32') + if not HAS_REDUCTION: + modules.remove('multiprocessing.popen_forkserver') - if c_int is not None: + if c_int is None: # This module requires _ctypes - modules.append('multiprocessing.sharedctypes') + modules.remove('multiprocessing.sharedctypes') for name in modules: __import__(name) mod = sys.modules[name] + self.assertTrue(hasattr(mod, '__all__'), name) - for attr in getattr(mod, '__all__', ()): + for attr in mod.__all__: self.assertTrue( hasattr(mod, attr), '%r does not have attribute %r' % (mod, attr) @@ -2896,7 +2937,7 @@ class _TestPollEintr(BaseTestCase): @classmethod def _killer(cls, pid): - time.sleep(0.5) + time.sleep(0.1) os.kill(pid, signal.SIGUSR1) @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') @@ -2909,12 +2950,14 @@ class _TestPollEintr(BaseTestCase): try: killer = self.Process(target=self._killer, args=(pid,)) killer.start() - p = self.Process(target=time.sleep, args=(1,)) - p.start() - p.join() + try: + p = self.Process(target=time.sleep, args=(2,)) + p.start() + p.join() + finally: + killer.join() self.assertTrue(got_signal[0]) self.assertEqual(p.exitcode, 0) - killer.join() finally: signal.signal(signal.SIGUSR1, oldhandler) @@ -2927,8 +2970,11 @@ class TestInvalidHandle(unittest.TestCase): @unittest.skipIf(WIN32, "skipped on Windows") def test_invalid_handles(self): conn = multiprocessing.connection.Connection(44977608) + # check that poll() doesn't crash try: - self.assertRaises((ValueError, OSError), conn.poll) + conn.poll() + except (ValueError, OSError): + pass finally: # Hack private attribute _handle to avoid printing an error # in conn.__del__ @@ -2936,131 +2982,6 @@ class TestInvalidHandle(unittest.TestCase): self.assertRaises((ValueError, OSError), multiprocessing.connection.Connection, -1) -# -# Functions used to create test cases from the base ones in this module -# - -def create_test_cases(Mixin, type): - result = {} - glob = globals() - Type = type.capitalize() - ALL_TYPES = {'processes', 'threads', 'manager'} - - for name in list(glob.keys()): - if name.startswith('_Test'): - base = glob[name] - assert set(base.ALLOWED_TYPES) <= ALL_TYPES, set(base.ALLOWED_TYPES) - if type in base.ALLOWED_TYPES: - newname = 'With' + Type + name[1:] - class Temp(base, Mixin, unittest.TestCase): - pass - result[newname] = Temp - Temp.__name__ = Temp.__qualname__ = newname - Temp.__module__ = Mixin.__module__ - return result - -# -# Create test cases -# - -class ProcessesMixin(object): - TYPE = 'processes' - Process = multiprocessing.Process - connection = multiprocessing.connection - current_process = staticmethod(multiprocessing.current_process) - active_children = staticmethod(multiprocessing.active_children) - Pool = staticmethod(multiprocessing.Pool) - Pipe = staticmethod(multiprocessing.Pipe) - Queue = staticmethod(multiprocessing.Queue) - JoinableQueue = staticmethod(multiprocessing.JoinableQueue) - Lock = staticmethod(multiprocessing.Lock) - RLock = staticmethod(multiprocessing.RLock) - Semaphore = staticmethod(multiprocessing.Semaphore) - BoundedSemaphore = staticmethod(multiprocessing.BoundedSemaphore) - Condition = staticmethod(multiprocessing.Condition) - Event = staticmethod(multiprocessing.Event) - Barrier = staticmethod(multiprocessing.Barrier) - Value = staticmethod(multiprocessing.Value) - Array = staticmethod(multiprocessing.Array) - RawValue = staticmethod(multiprocessing.RawValue) - RawArray = staticmethod(multiprocessing.RawArray) - -testcases_processes = create_test_cases(ProcessesMixin, type='processes') -globals().update(testcases_processes) - - -class ManagerMixin(object): - TYPE = 'manager' - Process = multiprocessing.Process - Queue = property(operator.attrgetter('manager.Queue')) - JoinableQueue = property(operator.attrgetter('manager.JoinableQueue')) - Lock = property(operator.attrgetter('manager.Lock')) - RLock = property(operator.attrgetter('manager.RLock')) - Semaphore = property(operator.attrgetter('manager.Semaphore')) - BoundedSemaphore = property(operator.attrgetter('manager.BoundedSemaphore')) - Condition = property(operator.attrgetter('manager.Condition')) - Event = property(operator.attrgetter('manager.Event')) - Barrier = property(operator.attrgetter('manager.Barrier')) - Value = property(operator.attrgetter('manager.Value')) - Array = property(operator.attrgetter('manager.Array')) - list = property(operator.attrgetter('manager.list')) - dict = property(operator.attrgetter('manager.dict')) - Namespace = property(operator.attrgetter('manager.Namespace')) - - @classmethod - def Pool(cls, *args, **kwds): - return cls.manager.Pool(*args, **kwds) - - @classmethod - def setUpClass(cls): - cls.manager = multiprocessing.Manager() - - @classmethod - def tearDownClass(cls): - # only the manager process should be returned by active_children() - # but this can take a bit on slow machines, so wait a few seconds - # if there are other children too (see #17395) - t = 0.01 - while len(multiprocessing.active_children()) > 1 and t < 5: - time.sleep(t) - t *= 2 - gc.collect() # do garbage collection - if cls.manager._number_of_objects() != 0: - # This is not really an error since some tests do not - # ensure that all processes which hold a reference to a - # managed object have been joined. - print('Shared objects which still exist at manager shutdown:') - print(cls.manager._debug_info()) - cls.manager.shutdown() - cls.manager.join() - cls.manager = None - -testcases_manager = create_test_cases(ManagerMixin, type='manager') -globals().update(testcases_manager) - - -class ThreadsMixin(object): - TYPE = 'threads' - Process = multiprocessing.dummy.Process - connection = multiprocessing.dummy.connection - current_process = staticmethod(multiprocessing.dummy.current_process) - active_children = staticmethod(multiprocessing.dummy.active_children) - Pool = staticmethod(multiprocessing.Pool) - Pipe = staticmethod(multiprocessing.dummy.Pipe) - Queue = staticmethod(multiprocessing.dummy.Queue) - JoinableQueue = staticmethod(multiprocessing.dummy.JoinableQueue) - Lock = staticmethod(multiprocessing.dummy.Lock) - RLock = staticmethod(multiprocessing.dummy.RLock) - Semaphore = staticmethod(multiprocessing.dummy.Semaphore) - BoundedSemaphore = staticmethod(multiprocessing.dummy.BoundedSemaphore) - Condition = staticmethod(multiprocessing.dummy.Condition) - Event = staticmethod(multiprocessing.dummy.Event) - Barrier = staticmethod(multiprocessing.dummy.Barrier) - Value = staticmethod(multiprocessing.dummy.Value) - Array = staticmethod(multiprocessing.dummy.Array) - -testcases_threads = create_test_cases(ThreadsMixin, type='threads') -globals().update(testcases_threads) class OtherTest(unittest.TestCase): @@ -3410,7 +3331,7 @@ class TestFlags(unittest.TestCase): def test_flags(self): import json, subprocess # start child process using unusual flags - prog = ('from test.test_multiprocessing import TestFlags; ' + + prog = ('from test._test_multiprocessing import TestFlags; ' + 'TestFlags.run_in_child()') data = subprocess.check_output( [sys.executable, '-E', '-S', '-O', '-c', prog]) @@ -3457,13 +3378,14 @@ class TestTimeouts(unittest.TestCase): class TestNoForkBomb(unittest.TestCase): def test_noforkbomb(self): + sm = multiprocessing.get_start_method() name = os.path.join(os.path.dirname(__file__), 'mp_fork_bomb.py') - if WIN32: - rc, out, err = test.script_helper.assert_python_failure(name) + if sm != 'fork': + rc, out, err = test.script_helper.assert_python_failure(name, sm) self.assertEqual('', out.decode('ascii')) self.assertIn('RuntimeError', err.decode('ascii')) else: - rc, out, err = test.script_helper.assert_python_ok(name) + rc, out, err = test.script_helper.assert_python_ok(name, sm) self.assertEqual('123', out.decode('ascii').rstrip()) self.assertEqual('', err.decode('ascii')) @@ -3481,7 +3403,8 @@ class TestForkAwareThreadLock(unittest.TestCase): if n > 1: p = multiprocessing.Process(target=cls.child, args=(n-1, conn)) p.start() - p.join() + conn.close() + p.join(timeout=5) else: conn.send(len(util._afterfork_registry)) conn.close() @@ -3492,11 +3415,78 @@ class TestForkAwareThreadLock(unittest.TestCase): old_size = len(util._afterfork_registry) p = multiprocessing.Process(target=self.child, args=(5, w)) p.start() + w.close() new_size = r.recv() - p.join() + p.join(timeout=5) self.assertLessEqual(new_size, old_size) # +# Check that non-forked child processes do not inherit unneeded fds/handles +# + +class TestCloseFds(unittest.TestCase): + + def get_high_socket_fd(self): + if WIN32: + # The child process will not have any socket handles, so + # calling socket.fromfd() should produce WSAENOTSOCK even + # if there is a handle of the same number. + return socket.socket().detach() + else: + # We want to produce a socket with an fd high enough that a + # freshly created child process will not have any fds as high. + fd = socket.socket().detach() + to_close = [] + while fd < 50: + to_close.append(fd) + fd = os.dup(fd) + for x in to_close: + os.close(x) + return fd + + def close(self, fd): + if WIN32: + socket.socket(fileno=fd).close() + else: + os.close(fd) + + @classmethod + def _test_closefds(cls, conn, fd): + try: + s = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) + except Exception as e: + conn.send(e) + else: + s.close() + conn.send(None) + + def test_closefd(self): + if not HAS_REDUCTION: + raise unittest.SkipTest('requires fd pickling') + + reader, writer = multiprocessing.Pipe() + fd = self.get_high_socket_fd() + try: + p = multiprocessing.Process(target=self._test_closefds, + args=(writer, fd)) + p.start() + writer.close() + e = reader.recv() + p.join(timeout=5) + finally: + self.close(fd) + writer.close() + reader.close() + + if multiprocessing.get_start_method() == 'fork': + self.assertIs(e, None) + else: + WSAENOTSOCK = 10038 + self.assertIsInstance(e, OSError) + self.assertTrue(e.errno == errno.EBADF or + e.winerror == WSAENOTSOCK, e) + +# # Issue #17097: EINTR should be ignored by recv(), send(), accept() etc # @@ -3540,10 +3530,10 @@ class TestIgnoreEINTR(unittest.TestCase): def handler(signum, frame): pass signal.signal(signal.SIGUSR1, handler) - l = multiprocessing.connection.Listener() - conn.send(l.address) - a = l.accept() - a.send('welcome') + with multiprocessing.connection.Listener() as l: + conn.send(l.address) + a = l.accept() + a.send('welcome') @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') def test_ignore_listener(self): @@ -3564,26 +3554,235 @@ class TestIgnoreEINTR(unittest.TestCase): finally: conn.close() +class TestStartMethod(unittest.TestCase): + def test_set_get(self): + multiprocessing.set_forkserver_preload(PRELOAD) + count = 0 + old_method = multiprocessing.get_start_method() + try: + for method in ('fork', 'spawn', 'forkserver'): + try: + multiprocessing.set_start_method(method) + except ValueError: + continue + self.assertEqual(multiprocessing.get_start_method(), method) + count += 1 + finally: + multiprocessing.set_start_method(old_method) + self.assertGreaterEqual(count, 1) + + def test_get_all(self): + methods = multiprocessing.get_all_start_methods() + if sys.platform == 'win32': + self.assertEqual(methods, ['spawn']) + else: + self.assertTrue(methods == ['fork', 'spawn'] or + methods == ['fork', 'spawn', 'forkserver']) + +# +# Check that killing process does not leak named semaphores +# + +@unittest.skipIf(sys.platform == "win32", + "test semantics don't make sense on Windows") +class TestSemaphoreTracker(unittest.TestCase): + def test_semaphore_tracker(self): + import subprocess + cmd = '''if 1: + import multiprocessing as mp, time, os + mp.set_start_method("spawn") + lock1 = mp.Lock() + lock2 = mp.Lock() + os.write(%d, lock1._semlock.name.encode("ascii") + b"\\n") + os.write(%d, lock2._semlock.name.encode("ascii") + b"\\n") + time.sleep(10) + ''' + r, w = os.pipe() + p = subprocess.Popen([sys.executable, + '-c', cmd % (w, w)], + pass_fds=[w], + stderr=subprocess.PIPE) + os.close(w) + with open(r, 'rb', closefd=True) as f: + name1 = f.readline().rstrip().decode('ascii') + name2 = f.readline().rstrip().decode('ascii') + _multiprocessing.sem_unlink(name1) + p.terminate() + p.wait() + time.sleep(1.0) + with self.assertRaises(OSError) as ctx: + _multiprocessing.sem_unlink(name2) + # docs say it should be ENOENT, but OSX seems to give EINVAL + self.assertIn(ctx.exception.errno, (errno.ENOENT, errno.EINVAL)) + err = p.stderr.read().decode('utf-8') + p.stderr.close() + expected = 'semaphore_tracker: There appear to be 2 leaked semaphores' + self.assertRegex(err, expected) + self.assertRegex(err, 'semaphore_tracker: %r: \[Errno' % name1) + # -# +# Mixins # -def setUpModule(): - if sys.platform.startswith("linux"): - try: - lock = multiprocessing.RLock() - except OSError: - raise unittest.SkipTest("OSError raises on RLock creation, " - "see issue 3111!") - check_enough_semaphores() - util.get_temp_dir() # creates temp directory for use by all processes - multiprocessing.get_logger().setLevel(LOG_LEVEL) +class ProcessesMixin(object): + TYPE = 'processes' + Process = multiprocessing.Process + connection = multiprocessing.connection + current_process = staticmethod(multiprocessing.current_process) + active_children = staticmethod(multiprocessing.active_children) + Pool = staticmethod(multiprocessing.Pool) + Pipe = staticmethod(multiprocessing.Pipe) + Queue = staticmethod(multiprocessing.Queue) + JoinableQueue = staticmethod(multiprocessing.JoinableQueue) + Lock = staticmethod(multiprocessing.Lock) + RLock = staticmethod(multiprocessing.RLock) + Semaphore = staticmethod(multiprocessing.Semaphore) + BoundedSemaphore = staticmethod(multiprocessing.BoundedSemaphore) + Condition = staticmethod(multiprocessing.Condition) + Event = staticmethod(multiprocessing.Event) + Barrier = staticmethod(multiprocessing.Barrier) + Value = staticmethod(multiprocessing.Value) + Array = staticmethod(multiprocessing.Array) + RawValue = staticmethod(multiprocessing.RawValue) + RawArray = staticmethod(multiprocessing.RawArray) + + +class ManagerMixin(object): + TYPE = 'manager' + Process = multiprocessing.Process + Queue = property(operator.attrgetter('manager.Queue')) + JoinableQueue = property(operator.attrgetter('manager.JoinableQueue')) + Lock = property(operator.attrgetter('manager.Lock')) + RLock = property(operator.attrgetter('manager.RLock')) + Semaphore = property(operator.attrgetter('manager.Semaphore')) + BoundedSemaphore = property(operator.attrgetter('manager.BoundedSemaphore')) + Condition = property(operator.attrgetter('manager.Condition')) + Event = property(operator.attrgetter('manager.Event')) + Barrier = property(operator.attrgetter('manager.Barrier')) + Value = property(operator.attrgetter('manager.Value')) + Array = property(operator.attrgetter('manager.Array')) + list = property(operator.attrgetter('manager.list')) + dict = property(operator.attrgetter('manager.dict')) + Namespace = property(operator.attrgetter('manager.Namespace')) + @classmethod + def Pool(cls, *args, **kwds): + return cls.manager.Pool(*args, **kwds) -def tearDownModule(): - # pause a bit so we don't get warning about dangling threads/processes - time.sleep(0.5) + @classmethod + def setUpClass(cls): + cls.manager = multiprocessing.Manager() + + @classmethod + def tearDownClass(cls): + # only the manager process should be returned by active_children() + # but this can take a bit on slow machines, so wait a few seconds + # if there are other children too (see #17395) + t = 0.01 + while len(multiprocessing.active_children()) > 1 and t < 5: + time.sleep(t) + t *= 2 + gc.collect() # do garbage collection + if cls.manager._number_of_objects() != 0: + # This is not really an error since some tests do not + # ensure that all processes which hold a reference to a + # managed object have been joined. + print('Shared objects which still exist at manager shutdown:') + print(cls.manager._debug_info()) + cls.manager.shutdown() + cls.manager.join() + cls.manager = None -if __name__ == '__main__': - unittest.main() +class ThreadsMixin(object): + TYPE = 'threads' + Process = multiprocessing.dummy.Process + connection = multiprocessing.dummy.connection + current_process = staticmethod(multiprocessing.dummy.current_process) + active_children = staticmethod(multiprocessing.dummy.active_children) + Pool = staticmethod(multiprocessing.Pool) + Pipe = staticmethod(multiprocessing.dummy.Pipe) + Queue = staticmethod(multiprocessing.dummy.Queue) + JoinableQueue = staticmethod(multiprocessing.dummy.JoinableQueue) + Lock = staticmethod(multiprocessing.dummy.Lock) + RLock = staticmethod(multiprocessing.dummy.RLock) + Semaphore = staticmethod(multiprocessing.dummy.Semaphore) + BoundedSemaphore = staticmethod(multiprocessing.dummy.BoundedSemaphore) + Condition = staticmethod(multiprocessing.dummy.Condition) + Event = staticmethod(multiprocessing.dummy.Event) + Barrier = staticmethod(multiprocessing.dummy.Barrier) + Value = staticmethod(multiprocessing.dummy.Value) + Array = staticmethod(multiprocessing.dummy.Array) + +# +# Functions used to create test cases from the base ones in this module +# + +def install_tests_in_module_dict(remote_globs, start_method): + __module__ = remote_globs['__name__'] + local_globs = globals() + ALL_TYPES = {'processes', 'threads', 'manager'} + + for name, base in local_globs.items(): + if not isinstance(base, type): + continue + if issubclass(base, BaseTestCase): + if base is BaseTestCase: + continue + assert set(base.ALLOWED_TYPES) <= ALL_TYPES, base.ALLOWED_TYPES + for type_ in base.ALLOWED_TYPES: + newname = 'With' + type_.capitalize() + name[1:] + Mixin = local_globs[type_.capitalize() + 'Mixin'] + class Temp(base, Mixin, unittest.TestCase): + pass + Temp.__name__ = Temp.__qualname__ = newname + Temp.__module__ = __module__ + remote_globs[newname] = Temp + elif issubclass(base, unittest.TestCase): + class Temp(base, object): + pass + Temp.__name__ = Temp.__qualname__ = name + Temp.__module__ = __module__ + remote_globs[name] = Temp + + dangling = [None, None] + old_start_method = [None] + + def setUpModule(): + multiprocessing.set_forkserver_preload(PRELOAD) + multiprocessing.process._cleanup() + dangling[0] = multiprocessing.process._dangling.copy() + dangling[1] = threading._dangling.copy() + old_start_method[0] = multiprocessing.get_start_method() + try: + multiprocessing.set_start_method(start_method) + except ValueError: + raise unittest.SkipTest(start_method + + ' start method not supported') + + if sys.platform.startswith("linux"): + try: + lock = multiprocessing.RLock() + except OSError: + raise unittest.SkipTest("OSError raises on RLock creation, " + "see issue 3111!") + check_enough_semaphores() + util.get_temp_dir() # creates temp directory + multiprocessing.get_logger().setLevel(LOG_LEVEL) + + def tearDownModule(): + multiprocessing.set_start_method(old_start_method[0]) + # pause a bit so we don't get warning about dangling threads/processes + time.sleep(0.5) + multiprocessing.process._cleanup() + gc.collect() + tmp = set(multiprocessing.process._dangling) - set(dangling[0]) + if tmp: + print('Dangling processes:', tmp, file=sys.stderr) + del tmp + tmp = set(threading._dangling) - set(dangling[1]) + if tmp: + print('Dangling threads:', tmp, file=sys.stderr) + + remote_globs['setUpModule'] = setUpModule + remote_globs['tearDownModule'] = tearDownModule diff --git a/Lib/test/audiotests.py b/Lib/test/audiotests.py index 147cda002a..60c6550cf9 100644 --- a/Lib/test/audiotests.py +++ b/Lib/test/audiotests.py @@ -47,6 +47,12 @@ class AudioTests: params = f.getparams() self.assertEqual(params, (nchannels, sampwidth, framerate, nframes, comptype, compname)) + self.assertEqual(params.nchannels, nchannels) + self.assertEqual(params.sampwidth, sampwidth) + self.assertEqual(params.framerate, framerate) + self.assertEqual(params.nframes, nframes) + self.assertEqual(params.comptype, comptype) + self.assertEqual(params.compname, compname) dump = pickle.dumps(params) self.assertEqual(pickle.loads(dump), params) @@ -63,15 +69,12 @@ class AudioWriteTests(AudioTests): return f def check_file(self, testfile, nframes, frames): - f = self.module.open(testfile, 'rb') - try: + with self.module.open(testfile, 'rb') as f: self.assertEqual(f.getnchannels(), self.nchannels) self.assertEqual(f.getsampwidth(), self.sampwidth) self.assertEqual(f.getframerate(), self.framerate) self.assertEqual(f.getnframes(), nframes) self.assertEqual(f.readframes(nframes), frames) - finally: - f.close() def test_write_params(self): f = self.create_file(TESTFN) @@ -81,6 +84,53 @@ class AudioWriteTests(AudioTests): self.nframes, self.comptype, self.compname) f.close() + def test_write_context_manager_calls_close(self): + # Close checks for a minimum header and will raise an error + # if it is not set, so this proves that close is called. + with self.assertRaises(self.module.Error): + with self.module.open(TESTFN, 'wb'): + pass + with self.assertRaises(self.module.Error): + with open(TESTFN, 'wb') as testfile: + with self.module.open(testfile): + pass + + def test_context_manager_with_open_file(self): + with open(TESTFN, 'wb') as testfile: + with self.module.open(testfile) as f: + f.setnchannels(self.nchannels) + f.setsampwidth(self.sampwidth) + f.setframerate(self.framerate) + f.setcomptype(self.comptype, self.compname) + self.assertEqual(testfile.closed, self.close_fd) + with open(TESTFN, 'rb') as testfile: + with self.module.open(testfile) as f: + self.assertFalse(f.getfp().closed) + params = f.getparams() + self.assertEqual(params.nchannels, self.nchannels) + self.assertEqual(params.sampwidth, self.sampwidth) + self.assertEqual(params.framerate, self.framerate) + if not self.close_fd: + self.assertIsNone(f.getfp()) + self.assertEqual(testfile.closed, self.close_fd) + + def test_context_manager_with_filename(self): + # If the file doesn't get closed, this test won't fail, but it will + # produce a resource leak warning. + with self.module.open(TESTFN, 'wb') as f: + f.setnchannels(self.nchannels) + f.setsampwidth(self.sampwidth) + f.setframerate(self.framerate) + f.setcomptype(self.comptype, self.compname) + with self.module.open(TESTFN) as f: + self.assertFalse(f.getfp().closed) + params = f.getparams() + self.assertEqual(params.nchannels, self.nchannels) + self.assertEqual(params.sampwidth, self.sampwidth) + self.assertEqual(params.framerate, self.framerate) + if not self.close_fd: + self.assertIsNone(f.getfp()) + def test_write(self): f = self.create_file(TESTFN) f.setnframes(self.nframes) @@ -203,12 +253,9 @@ class AudioTestsWithSourceFile(AudioTests): with open(TESTFN, 'rb') as testfile: self.assertEqual(testfile.read(13), b'ababagalamaga') - f = self.module.open(testfile, 'rb') - try: + with self.module.open(testfile, 'rb') as f: self.assertEqual(f.getnchannels(), self.nchannels) self.assertEqual(f.getsampwidth(), self.sampwidth) self.assertEqual(f.getframerate(), self.framerate) self.assertEqual(f.getnframes(), self.sndfilenframes) self.assertEqual(f.readframes(self.nframes), self.frames) - finally: - f.close() diff --git a/Lib/test/badsyntax_future10.py b/Lib/test/badsyntax_future10.py new file mode 100644 index 0000000000..fa5ab67a98 --- /dev/null +++ b/Lib/test/badsyntax_future10.py @@ -0,0 +1,3 @@ +from __future__ import absolute_import +"spam, bar, blah" +from __future__ import print_function diff --git a/Lib/test/bytecode_helper.py b/Lib/test/bytecode_helper.py new file mode 100644 index 0000000000..c4943cda00 --- /dev/null +++ b/Lib/test/bytecode_helper.py @@ -0,0 +1,72 @@ +"""bytecode_helper - support tools for testing correct bytecode generation""" + +import unittest +import dis +import io + +_UNSPECIFIED = object() + +class BytecodeTestCase(unittest.TestCase): + """Custom assertion methods for inspecting bytecode.""" + + def get_disassembly_as_string(self, co): + s = io.StringIO() + dis.dis(co, file=s) + return s.getvalue() + + def assertInstructionMatches(self, instr, expected, *, line_offset=0): + # Deliberately test opname first, since that gives a more + # meaningful error message than testing opcode + self.assertEqual(instr.opname, expected.opname) + self.assertEqual(instr.opcode, expected.opcode) + self.assertEqual(instr.arg, expected.arg) + self.assertEqual(instr.argval, expected.argval) + self.assertEqual(instr.argrepr, expected.argrepr) + self.assertEqual(instr.offset, expected.offset) + if expected.starts_line is None: + self.assertIsNone(instr.starts_line) + else: + self.assertEqual(instr.starts_line, + expected.starts_line + line_offset) + self.assertEqual(instr.is_jump_target, expected.is_jump_target) + + + def assertBytecodeExactlyMatches(self, x, expected, *, line_offset=0): + """Throws AssertionError if any discrepancy is found in bytecode + + *x* is the object to be introspected + *expected* is a list of dis.Instruction objects + + Set *line_offset* as appropriate to adjust for the location of the + object to be disassembled within the test file. If the expected list + assumes the first line is line 1, then an appropriate offset would be + ``1 - f.__code__.co_firstlineno``. + """ + actual = dis.get_instructions(x, line_offset=line_offset) + self.assertEqual(list(actual), expected) + + def assertInBytecode(self, x, opname, argval=_UNSPECIFIED): + """Returns instr if op is found, otherwise throws AssertionError""" + for instr in dis.get_instructions(x): + if instr.opname == opname: + if argval is _UNSPECIFIED or instr.argval == argval: + return instr + disassembly = self.get_disassembly_as_string(x) + if argval is _UNSPECIFIED: + msg = '%s not found in bytecode:\n%s' % (opname, disassembly) + else: + msg = '(%s,%r) not found in bytecode:\n%s' + msg = msg % (opname, argval, disassembly) + self.fail(msg) + + def assertNotInBytecode(self, x, opname, argval=_UNSPECIFIED): + """Throws AssertionError if op is found""" + for instr in dis.get_instructions(x): + if instr.opname == opname: + disassembly = self.get_disassembly_as_string(co) + if opargval is _UNSPECIFIED: + msg = '%s occurs in bytecode:\n%s' % (opname, disassembly) + elif instr.argval == argval: + msg = '(%s,%r) occurs in bytecode:\n%s' + msg = msg % (opname, argval, disassembly) + self.fail(msg) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 3226bce7da..49140a5526 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -619,6 +619,10 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): eq(td(hours=-.2/us_per_hour), td(0)) eq(td(days=-.4/us_per_day, hours=-.2/us_per_hour), td(microseconds=-1)) + # Test for a patch in Issue 8860 + eq(td(microseconds=0.5), 0.5*td(microseconds=1.0)) + eq(td(microseconds=0.5)//td.resolution, 0.5*td.resolution//td.resolution) + def test_massive_normalization(self): td = timedelta(microseconds=-1) self.assertEqual((td.days, td.seconds, td.microseconds), diff --git a/Lib/test/final_a.py b/Lib/test/final_a.py new file mode 100644 index 0000000000..390ee8895a --- /dev/null +++ b/Lib/test/final_a.py @@ -0,0 +1,19 @@ +""" +Fodder for module finalization tests in test_module. +""" + +import shutil +import test.final_b + +x = 'a' + +class C: + def __del__(self): + # Inspect module globals and builtins + print("x =", x) + print("final_b.x =", test.final_b.x) + print("shutil.rmtree =", getattr(shutil.rmtree, '__name__', None)) + print("len =", getattr(len, '__name__', None)) + +c = C() +_underscored = C() diff --git a/Lib/test/final_b.py b/Lib/test/final_b.py new file mode 100644 index 0000000000..7228d82b88 --- /dev/null +++ b/Lib/test/final_b.py @@ -0,0 +1,19 @@ +""" +Fodder for module finalization tests in test_module. +""" + +import shutil +import test.final_a + +x = 'b' + +class C: + def __del__(self): + # Inspect module globals and builtins + print("x =", x) + print("final_a.x =", test.final_a.x) + print("shutil.rmtree =", getattr(shutil.rmtree, '__name__', None)) + print("len =", getattr(len, '__name__', None)) + +c = C() +_underscored = C() diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py index 88527df25e..19b54ec736 100644 --- a/Lib/test/fork_wait.py +++ b/Lib/test/fork_wait.py @@ -28,7 +28,7 @@ class ForkWait(unittest.TestCase): self.alive[id] = os.getpid() try: time.sleep(SHORTSLEEP) - except IOError: + except OSError: pass def wait_impl(self, cpid): diff --git a/Lib/test/keycert3.pem b/Lib/test/keycert3.pem new file mode 100644 index 0000000000..5bfa62c4ca --- /dev/null +++ b/Lib/test/keycert3.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMLgD0kAKDb5cFyP +jbwNfR5CtewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM +9z2j1OlaN+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZ +aggEdkj1TsSsv1zWIYKlPIjlvhuxAgMBAAECgYA0aH+T2Vf3WOPv8KdkcJg6gCRe +yJKXOWgWRcicx/CUzOEsTxmFIDPLxqAWA3k7v0B+3vjGw5Y9lycV/5XqXNoQI14j +y09iNsumds13u5AKkGdTJnZhQ7UKdoVHfuP44ZdOv/rJ5/VD6F4zWywpe90pcbK+ +AWDVtusgGQBSieEl1QJBAOyVrUG5l2yoUBtd2zr/kiGm/DYyXlIthQO/A3/LngDW +5/ydGxVsT7lAVOgCsoT+0L4efTh90PjzW8LPQrPBWVMCQQDS3h/FtYYd5lfz+FNL +9CEe1F1w9l8P749uNUD0g317zv1tatIqVCsQWHfVHNdVvfQ+vSFw38OORO00Xqs9 +1GJrAkBkoXXEkxCZoy4PteheO/8IWWLGGr6L7di6MzFl1lIqwT6D8L9oaV2vynFT +DnKop0pa09Unhjyw57KMNmSE2SUJAkEArloTEzpgRmCq4IK2/NpCeGdHS5uqRlbh +1VIa/xGps7EWQl5Mn8swQDel/YP3WGHTjfx7pgSegQfkyaRtGpZ9OQJAa9Vumj8m +JAAtI0Bnga8hgQx7BhTQY4CadDxyiRGOGYhwUzYVCqkb2sbVRH9HnwUaJT7cWBY3 +RnJdHOMXWem7/w== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443281 (0xb09264b1f2da21d1) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:c2:e0:0f:49:00:28:36:f9:70:5c:8f:8d:bc:0d: + 7d:1e:42:b5:ec:1d:5c:2f:a4:31:70:16:0f:c0:cb: + c6:24:d3:be:13:16:ee:a5:67:97:03:a6:df:a9:99: + 96:cc:c7:2a:fb:11:7f:4e:65:4f:8a:5e:82:21:4c: + f7:3d:a3:d4:e9:5a:37:e7:22:fd:7e:cd:53:6d:93: + 34:de:9c:ad:84:a2:37:be:c5:8d:82:4f:e3:ae:23: + f3:be:a7:75:2c:72:0f:ea:f3:ca:cd:fc:e9:3f:b5: + af:56:99:6a:08:04:76:48:f5:4e:c4:ac:bf:5c:d6: + 21:82:a5:3c:88:e5:be:1b:b1 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + 2f:42:5f:a3:09:2c:fa:51:88:c7:37:7f:ea:0e:63:f0:a2:9a: + e5:5a:e2:c8:20:f0:3f:60:bc:c8:0f:b6:c6:76:ce:db:83:93: + f5:a3:33:67:01:8e:04:cd:00:9a:73:fd:f3:35:86:fa:d7:13: + e2:46:c6:9d:c0:29:53:d4:a9:90:b8:77:4b:e6:83:76:e4:92: + d6:9c:50:cf:43:d0:c6:01:77:61:9a:de:9b:70:f7:72:cd:59: + 00:31:69:d9:b4:ca:06:9c:6d:c3:c7:80:8c:68:e6:b5:a2:f8: + ef:1d:bb:16:9f:77:77:ef:87:62:22:9b:4d:69:a4:3a:1a:f1: + 21:5e:8c:32:ac:92:fd:15:6b:18:c2:7f:15:0d:98:30:ca:75: + 8f:1a:71:df:da:1d:b2:ef:9a:e8:2d:2e:02:fd:4a:3c:aa:96: + 0b:06:5d:35:b3:3d:24:87:4b:e0:b0:58:60:2f:45:ac:2e:48: + 8a:b0:99:10:65:27:ff:cc:b1:d8:fd:bd:26:6b:b9:0c:05:2a: + f4:45:63:35:51:07:ed:83:85:fe:6f:69:cb:bb:40:a8:ae:b6: + 3b:56:4a:2d:a4:ed:6d:11:2c:4d:ed:17:24:fd:47:bc:d3:41: + a2:d3:06:fe:0c:90:d8:d8:94:26:c4:ff:cc:a1:d8:42:77:eb: + fc:a9:94:71 +-----BEGIN CERTIFICATE----- +MIICpDCCAYwCCQCwkmSx8toh0TANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBfMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDEwlsb2NhbGhv +c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMLgD0kAKDb5cFyPjbwNfR5C +tewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM9z2j1Ola +N+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZaggEdkj1 +TsSsv1zWIYKlPIjlvhuxAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAC9CX6MJLPpR +iMc3f+oOY/CimuVa4sgg8D9gvMgPtsZ2ztuDk/WjM2cBjgTNAJpz/fM1hvrXE+JG +xp3AKVPUqZC4d0vmg3bkktacUM9D0MYBd2Ga3ptw93LNWQAxadm0ygacbcPHgIxo +5rWi+O8duxafd3fvh2Iim01ppDoa8SFejDKskv0VaxjCfxUNmDDKdY8acd/aHbLv +mugtLgL9SjyqlgsGXTWzPSSHS+CwWGAvRawuSIqwmRBlJ//Msdj9vSZruQwFKvRF +YzVRB+2Dhf5vacu7QKiutjtWSi2k7W0RLE3tFyT9R7zTQaLTBv4MkNjYlCbE/8yh +2EJ36/yplHE= +-----END CERTIFICATE----- diff --git a/Lib/test/keycert4.pem b/Lib/test/keycert4.pem new file mode 100644 index 0000000000..53355c8a50 --- /dev/null +++ b/Lib/test/keycert4.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAK5UQiMI5VkNs2Qv +L7gUaiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2 +NkX0ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1 +L2OQhEx1GM6RydHdgX69G64LXcY5AgMBAAECgYAhsRMfJkb9ERLMl/oG/5sLQu9L +pWDKt6+ZwdxzlZbggQ85CMYshjLKIod2DLL/sLf2x1PRXyRG131M1E3k8zkkz6de +R1uDrIN/x91iuYzfLQZGh8bMY7Yjd2eoroa6R/7DjpElGejLxOAaDWO0ST2IFQy9 +myTGS2jSM97wcXfsSQJBANP3jelJoS5X6BRjTSneY21wcocxVuQh8pXpErALVNsT +drrFTeaBuZp7KvbtnIM5g2WRNvaxLZlAY/hXPJvi6ncCQQDSix1cebml6EmPlEZS +Mm8gwI2F9ufUunwJmBJcz826Do0ZNGByWDAM/JQZH4FX4GfAFNuj8PUb+GQfadkx +i1DPAkEA0lVsNHojvuDsIo8HGuzarNZQT2beWjJ1jdxh9t7HrTx7LIps6rb/fhOK +Zs0R6gVAJaEbcWAPZ2tFyECInAdnsQJAUjaeXXjuxFkjOFym5PvqpvhpivEx78Bu +JPTr3rAKXmfGMxxfuOa0xK1wSyshP6ZR/RBn/+lcXPKubhHQDOegwwJAJF1DBQnN ++/tLmOPULtDwfP4Zixn+/8GmGOahFoRcu6VIGHmRilJTn6MOButw7Glv2YdeC6l/ +e83Gq6ffLVfKNQ== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443282 (0xb09264b1f2da21d2) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:ae:54:42:23:08:e5:59:0d:b3:64:2f:2f:b8:14: + 6a:20:dd:15:eb:cd:51:74:63:53:80:c7:01:ed:d9: + cf:36:0b:64:d1:3a:f6:1f:60:3b:d5:42:49:2d:7a: + b4:9e:5f:4f:95:44:bb:41:19:c8:6a:f4:7b:75:76: + 36:45:f4:66:85:34:1d:cf:d4:69:8e:2a:c7:b2:c7: + 9a:7e:52:61:9a:48:c6:12:67:91:fe:d2:c8:72:4a: + d7:35:1a:1a:55:34:fc:bc:58:a8:8b:86:0a:d1:79: + 76:ac:75:2f:63:90:84:4c:75:18:ce:91:c9:d1:dd: + 81:7e:bd:1b:ae:0b:5d:c6:39 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + ad:45:8a:8e:ef:c6:ef:04:41:5c:2c:4a:84:dc:02:76:0c:d0: + 66:0f:f0:16:04:58:4d:fd:68:b7:b8:d3:a8:41:a5:5c:3c:6f: + 65:3c:d1:f8:ce:43:35:e7:41:5f:53:3d:c9:2c:c3:7d:fc:56: + 4a:fa:47:77:38:9d:bb:97:28:0a:3b:91:19:7f:bc:74:ae:15: + 6b:bd:20:36:67:45:a5:1e:79:d7:75:e6:89:5c:6d:54:84:d1: + 95:d7:a7:b4:33:3c:af:37:c4:79:8f:5e:75:dc:75:c2:18:fb: + 61:6f:2d:dc:38:65:5b:ba:67:28:d0:88:d7:8d:b9:23:5a:8e: + e8:c6:bb:db:ce:d5:b8:41:2a:ce:93:08:b6:95:ad:34:20:18: + d5:3b:37:52:74:50:0b:07:2c:b0:6d:a4:4c:7b:f4:e0:fd:d1: + af:17:aa:20:cd:62:e3:f0:9d:37:69:db:41:bd:d4:1c:fb:53: + 20:da:88:9d:76:26:67:ce:01:90:a7:80:1d:a9:5b:39:73:68: + 54:0a:d1:2a:03:1b:8f:3c:43:5d:5d:c4:51:f1:a7:e7:11:da: + 31:2c:49:06:af:04:f4:b8:3c:99:c4:20:b9:06:36:a2:00:92: + 61:1d:0c:6d:24:05:e2:82:e1:47:db:a0:5f:ba:b9:fb:ba:fa: + 49:12:1e:ce +-----BEGIN CERTIFICATE----- +MIICpzCCAY8CCQCwkmSx8toh0jANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBiMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRUwEwYDVQQDEwxmYWtlaG9z +dG5hbWUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAK5UQiMI5VkNs2QvL7gU +aiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2NkX0 +ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1L2OQ +hEx1GM6RydHdgX69G64LXcY5AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAK1Fio7v +xu8EQVwsSoTcAnYM0GYP8BYEWE39aLe406hBpVw8b2U80fjOQzXnQV9TPcksw338 +Vkr6R3c4nbuXKAo7kRl/vHSuFWu9IDZnRaUeedd15olcbVSE0ZXXp7QzPK83xHmP +XnXcdcIY+2FvLdw4ZVu6ZyjQiNeNuSNajujGu9vO1bhBKs6TCLaVrTQgGNU7N1J0 +UAsHLLBtpEx79OD90a8XqiDNYuPwnTdp20G91Bz7UyDaiJ12JmfOAZCngB2pWzlz +aFQK0SoDG488Q11dxFHxp+cR2jEsSQavBPS4PJnEILkGNqIAkmEdDG0kBeKC4Ufb +oF+6ufu6+kkSHs4= +-----END CERTIFICATE----- diff --git a/Lib/test/leakers/test_gestalt.py b/Lib/test/leakers/test_gestalt.py deleted file mode 100644 index e0081c1fc5..0000000000 --- a/Lib/test/leakers/test_gestalt.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys - -if sys.platform != 'darwin': - raise ValueError("This test only leaks on Mac OS X") - -def leak(): - # taken from platform._mac_ver_lookup() - from gestalt import gestalt - import MacOS - - try: - gestalt('sysu') - except MacOS.Error: - pass diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index bfbf44e0db..1a24bdeed3 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -80,6 +80,11 @@ class BaseLockTests(BaseTestCase): lock = self.locktype() del lock + def test_repr(self): + lock = self.locktype() + repr(lock) + del lock + def test_acquire_destroy(self): lock = self.locktype() lock.acquire() diff --git a/Lib/test/make_ssl_certs.py b/Lib/test/make_ssl_certs.py index 48d2e57f4b..f630813b2c 100644 --- a/Lib/test/make_ssl_certs.py +++ b/Lib/test/make_ssl_certs.py @@ -2,6 +2,7 @@ and friends.""" import os +import shutil import sys import tempfile from subprocess import * @@ -20,11 +21,52 @@ req_template = """ [req_x509_extensions] subjectAltName = DNS:{hostname} + + [ ca ] + default_ca = CA_default + + [ CA_default ] + dir = cadir + database = $dir/index.txt + default_md = sha1 + default_days = 3600 + certificate = pycacert.pem + private_key = pycakey.pem + serial = $dir/serial + RANDFILE = $dir/.rand + + policy = policy_match + + [ policy_match ] + countryName = match + stateOrProvinceName = optional + organizationName = match + organizationalUnitName = optional + commonName = supplied + emailAddress = optional + + [ policy_anything ] + countryName = optional + stateOrProvinceName = optional + localityName = optional + organizationName = optional + organizationalUnitName = optional + commonName = supplied + emailAddress = optional + + + [ v3_ca ] + + subjectKeyIdentifier=hash + authorityKeyIdentifier=keyid:always,issuer + basicConstraints = CA:true + """ here = os.path.abspath(os.path.dirname(__file__)) -def make_cert_key(hostname): +def make_cert_key(hostname, sign=False): + print("creating cert for " + hostname) tempnames = [] for i in range(3): with tempfile.NamedTemporaryFile(delete=False) as f: @@ -33,10 +75,25 @@ def make_cert_key(hostname): try: with open(req_file, 'w') as f: f.write(req_template.format(hostname=hostname)) - args = ['req', '-new', '-days', '3650', '-nodes', '-x509', + args = ['req', '-new', '-days', '3650', '-nodes', '-newkey', 'rsa:1024', '-keyout', key_file, - '-out', cert_file, '-config', req_file] + '-config', req_file] + if sign: + with tempfile.NamedTemporaryFile(delete=False) as f: + tempnames.append(f.name) + reqfile = f.name + args += ['-out', reqfile ] + + else: + args += ['-x509', '-out', cert_file ] check_call(['openssl'] + args) + + if sign: + args = ['ca', '-config', req_file, '-out', cert_file, '-outdir', 'cadir', + '-policy', 'policy_anything', '-batch', '-infiles', reqfile ] + check_call(['openssl'] + args) + + with open(cert_file, 'r') as f: cert = f.read() with open(key_file, 'r') as f: @@ -46,6 +103,32 @@ def make_cert_key(hostname): for name in tempnames: os.remove(name) +TMP_CADIR = 'cadir' + +def unmake_ca(): + shutil.rmtree(TMP_CADIR) + +def make_ca(): + os.mkdir(TMP_CADIR) + with open(os.path.join('cadir','index.txt'),'a+') as f: + pass # empty file + with open(os.path.join('cadir','index.txt.attr'),'w+') as f: + f.write('unique_subject = no') + + with tempfile.NamedTemporaryFile("w") as t: + t.write(req_template.format(hostname='our-ca-server')) + t.flush() + with tempfile.NamedTemporaryFile() as f: + args = ['req', '-new', '-days', '3650', '-extensions', 'v3_ca', '-nodes', + '-newkey', 'rsa:2048', '-keyout', 'pycakey.pem', + '-out', f.name, + '-subj', '/C=XY/L=Castle Anthrax/O=Python Software Foundation CA/CN=our-ca-server'] + check_call(['openssl'] + args) + args = ['ca', '-config', t.name, '-create_serial', + '-out', 'pycacert.pem', '-batch', '-outdir', TMP_CADIR, + '-keyfile', 'pycakey.pem', '-days', '3650', + '-selfsign', '-extensions', 'v3_ca', '-infiles', f.name ] + check_call(['openssl'] + args) if __name__ == '__main__': os.chdir(here) @@ -54,11 +137,34 @@ if __name__ == '__main__': f.write(cert) with open('ssl_key.pem', 'w') as f: f.write(key) + print("password protecting ssl_key.pem in ssl_key.passwd.pem") + check_call(['openssl','rsa','-in','ssl_key.pem','-out','ssl_key.passwd.pem','-des3','-passout','pass:somepass']) + check_call(['openssl','rsa','-in','ssl_key.pem','-out','keycert.passwd.pem','-des3','-passout','pass:somepass']) + with open('keycert.pem', 'w') as f: f.write(key) f.write(cert) + + with open('keycert.passwd.pem', 'a+') as f: + f.write(cert) + # For certificate matching tests + make_ca() cert, key = make_cert_key('fakehostname') with open('keycert2.pem', 'w') as f: f.write(key) f.write(cert) + + cert, key = make_cert_key('localhost', True) + with open('keycert3.pem', 'w') as f: + f.write(key) + f.write(cert) + + cert, key = make_cert_key('fakehostname', True) + with open('keycert4.pem', 'w') as f: + f.write(key) + f.write(cert) + + unmake_ca() + print("\n\nPlease change the values in test_ssl.py, test_parse_cert function related to notAfter,notBefore and serialNumber") + check_call(['openssl','x509','-in','keycert.pem','-dates','-serial','-noout']) diff --git a/Lib/test/mock_socket.py b/Lib/test/mock_socket.py index d09e78c1d5..8ef0ec8c8d 100644 --- a/Lib/test/mock_socket.py +++ b/Lib/test/mock_socket.py @@ -140,12 +140,8 @@ def gethostbyname(name): return "" -class gaierror(Exception): - pass - - -class error(Exception): - pass +gaierror = socket_module.gaierror +error = socket_module.error # Constants diff --git a/Lib/test/mp_fork_bomb.py b/Lib/test/mp_fork_bomb.py index 908afe3045..017e010ba0 100644 --- a/Lib/test/mp_fork_bomb.py +++ b/Lib/test/mp_fork_bomb.py @@ -7,6 +7,11 @@ def foo(): # correctly on Windows. However, we should get a RuntimeError rather # than the Windows equivalent of a fork bomb. +if len(sys.argv) > 1: + multiprocessing.set_start_method(sys.argv[1]) +else: + multiprocessing.set_start_method('spawn') + p = multiprocessing.Process(target=foo) p.start() p.join() diff --git a/Lib/test/multibytecodec_support.py b/Lib/test/multibytecodec_support.py index 26bac7be10..dcaae7b95c 100644 --- a/Lib/test/multibytecodec_support.py +++ b/Lib/test/multibytecodec_support.py @@ -282,7 +282,7 @@ class TestBase_Mapping(unittest.TestCase): unittest.TestCase.__init__(self, *args, **kw) try: self.open_mapping_file().close() # test it to report the error early - except (IOError, HTTPException): + except (OSError, HTTPException): self.skipTest("Could not retrieve "+self.mapfileurl) def open_mapping_file(self): diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 052290d764..197112024b 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -601,30 +601,6 @@ class AbstractPickleTests(unittest.TestCase): self.assertRaises(KeyError, self.loads, b'g0\np0') self.assertEqual(self.loads(b'((Kdtp0\nh\x00l.))'), [(100,), (100,)]) - def test_insecure_strings(self): - # XXX Some of these tests are temporarily disabled - insecure = [b"abc", b"2 + 2", # not quoted - ## b"'abc' + 'def'", # not a single quoted string - b"'abc", # quote is not closed - b"'abc\"", # open quote and close quote don't match - b"'abc' ?", # junk after close quote - b"'\\'", # trailing backslash - # Variations on issue #17710 - b"'", - b'"', - b"' ", - b"' ", - b"' ", - b"' ", - b'" ', - # some tests of the quoting rules - ## b"'abc\"\''", - ## b"'\\\\a\'\'\'\\\'\\\\\''", - ] - for b in insecure: - buf = b"S" + b + b"\012p0\012." - self.assertRaises(ValueError, self.loads, buf) - def test_unicode(self): endcases = ['', '<\\u>', '<\\\u1234>', '<\n>', '<\\>', '<\\\U00012345>', @@ -1214,6 +1190,35 @@ class AbstractPickleTests(unittest.TestCase): dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.' self.assertRaises(ValueError, self.loads, dumped) + def test_badly_escaped_string(self): + self.assertRaises(ValueError, self.loads, b"S'\\'\n.") + + def test_badly_quoted_string(self): + # Issue #17710 + badpickles = [b"S'\n.", + b'S"\n.', + b'S\' \n.', + b'S" \n.', + b'S\'"\n.', + b'S"\'\n.', + b"S' ' \n.", + b'S" " \n.', + b"S ''\n.", + b'S ""\n.', + b'S \n.', + b'S\n.', + b'S.'] + for p in badpickles: + self.assertRaises(pickle.UnpicklingError, self.loads, p) + + def test_correctly_quoted_string(self): + goodpickles = [(b"S''\n.", ''), + (b'S""\n.', ''), + (b'S"\\n"\n.', '\n'), + (b"S'\\n'\n.", '\n')] + for p, expected in goodpickles: + self.assertEqual(self.loads(p), expected) + def _check_pickling_with_opcode(self, obj, opcode, proto): pickled = self.dumps(obj, proto) self.assertTrue(opcode_in_pickle(opcode, pickled)) diff --git a/Lib/test/pycacert.pem b/Lib/test/pycacert.pem new file mode 100644 index 0000000000..09b1f3e08a --- /dev/null +++ b/Lib/test/pycacert.pem @@ -0,0 +1,78 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 12723342612721443280 (0xb09264b1f2da21d0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Jan 2 19:47:07 2023 GMT + Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e7:de:e9:e3:0c:9f:00:b6:a1:fd:2b:5b:96:d2: + 6f:cc:e0:be:86:b9:20:5e:ec:03:7a:55:ab:ea:a4: + e9:f9:49:85:d2:66:d5:ed:c7:7a:ea:56:8e:2d:8f: + e7:42:e2:62:28:a9:9f:d6:1b:8e:eb:b5:b4:9c:9f: + 14:ab:df:e6:94:8b:76:1d:3e:6d:24:61:ed:0c:bf: + 00:8a:61:0c:df:5c:c8:36:73:16:00:cd:47:ba:6d: + a4:a4:74:88:83:23:0a:19:fc:09:a7:3c:4a:4b:d3: + e7:1d:2d:e4:ea:4c:54:21:f3:26:db:89:37:18:d4: + 02:bb:40:32:5f:a4:ff:2d:1c:f7:d4:bb:ec:8e:cf: + 5c:82:ac:e6:7c:08:6c:48:85:61:07:7f:25:e0:5c: + e0:bc:34:5f:e0:b9:04:47:75:c8:47:0b:8d:bc:d6: + c8:68:5f:33:83:62:d2:20:44:35:b1:ad:81:1a:8a: + cd:bc:35:b0:5c:8b:47:d6:18:e9:9c:18:97:cc:01: + 3c:29:cc:e8:1e:e4:e4:c1:b8:de:e7:c2:11:18:87: + 5a:93:34:d8:a6:25:f7:14:71:eb:e4:21:a2:d2:0f: + 2e:2e:d4:62:00:35:d3:d6:ef:5c:60:4b:4c:a9:14: + e2:dd:15:58:46:37:33:26:b7:e7:2e:5d:ed:42:e4: + c5:4d + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + X509v3 Authority Key Identifier: + keyid:BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + 7d:0a:f5:cb:8d:d3:5d:bd:99:8e:f8:2b:0f:ba:eb:c2:d9:a6: + 27:4f:2e:7b:2f:0e:64:d8:1c:35:50:4e:ee:fc:90:b9:8d:6d: + a8:c5:c6:06:b0:af:f3:2d:bf:3b:b8:42:07:dd:18:7d:6d:95: + 54:57:85:18:60:47:2f:eb:78:1b:f9:e8:17:fd:5a:0d:87:17: + 28:ac:4c:6a:e6:bc:29:f4:f4:55:70:29:42:de:85:ea:ab:6c: + 23:06:64:30:75:02:8e:53:bc:5e:01:33:37:cc:1e:cd:b8:a4: + fd:ca:e4:5f:65:3b:83:1c:86:f1:55:02:a0:3a:8f:db:91:b7: + 40:14:b4:e7:8d:d2:ee:73:ba:e3:e5:34:2d:bc:94:6f:4e:24: + 06:f7:5f:8b:0e:a7:8e:6b:de:5e:75:f4:32:9a:50:b1:44:33: + 9a:d0:05:e2:78:82:ff:db:da:8a:63:eb:a9:dd:d1:bf:a0:61: + ad:e3:9e:8a:24:5d:62:0e:e7:4c:91:7f:ef:df:34:36:3b:2f: + 5d:f5:84:b2:2f:c4:6d:93:96:1a:6f:30:28:f1:da:12:9a:64: + b4:40:33:1d:bd:de:2b:53:a8:ea:be:d6:bc:4e:96:f5:44:fb: + 32:18:ae:d5:1f:f6:69:af:b6:4e:7b:1d:58:ec:3b:a9:53:a3: + 5e:58:c8:9e +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/Lib/test/pycakey.pem b/Lib/test/pycakey.pem new file mode 100644 index 0000000000..fc6effefb2 --- /dev/null +++ b/Lib/test/pycakey.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDn3unjDJ8AtqH9 +K1uW0m/M4L6GuSBe7AN6VavqpOn5SYXSZtXtx3rqVo4tj+dC4mIoqZ/WG47rtbSc +nxSr3+aUi3YdPm0kYe0MvwCKYQzfXMg2cxYAzUe6baSkdIiDIwoZ/AmnPEpL0+cd +LeTqTFQh8ybbiTcY1AK7QDJfpP8tHPfUu+yOz1yCrOZ8CGxIhWEHfyXgXOC8NF/g +uQRHdchHC4281shoXzODYtIgRDWxrYEais28NbBci0fWGOmcGJfMATwpzOge5OTB +uN7nwhEYh1qTNNimJfcUcevkIaLSDy4u1GIANdPW71xgS0ypFOLdFVhGNzMmt+cu +Xe1C5MVNAgMBAAECggEBAJPM7QuUrPn4cLN/Ysd15lwTWn9oHDFFgkYFvCs66gXE +ju/6Kx2BjWE4wTJby09AHM/MqB0DvguT7Mf1Q2j3tPQ1HZowg8OwRDleuwp6KIls +jBbhL0Jdl/5HC67ktWvZ9wNvO/wFG1rQfT6FVajf9LUbWEaSZbOG2SLhHfsHorzu +xjTJaI3bQ/0+79B1exwk5ruwhzFRd/XpY8hls7D/RfPIuHDlBghkW3N59KFWrf5h +6bNEh2THm0+IyGcGqs0FD+QCOXyvsjwSUswqrr2ctLREOeDcd5ReUjSxYgjcJRrm +J7ceIY/+uwDJxw/OlnmBvF6pQMkKwYW2gFztu+g2t4UCgYEA/9yo01Exz4crxXsy +tAlnDJM++nZcm07rtFjTKHUfKY/cCgNTa8udM0svnfwlid/dpgLsI38gx04HHC1i +EZ4acz+ToIWedLxM0nq73//xeRWEazOvCz1mMTZaMldahTWAyzN8qVK2B/625Yy4 +wNYWyweBBwEB8MzaCs73spksXOsCgYEA5/7wvhiofYGFAfMuANeJIwDL2OtBnoOv +mVNfCmi3GC38fzwyi5ZpskWDiS2woJ+LQfs9Qu4EcZbUFLd7gbeOvb5gmFUtYope +LitUUKunIR18MkQ+mQDBpQPQPhk4QJP5reCbWkrfTu7b5o/iS41s6fBTFmuzhLcT +C71vFdCyeKcCgYAiCCqYeOtELDmBOeLDmaCQRqGQ1N96dOPbCBmF/xYXBCCDYG/f +HaUaJnz96YTgstsbcrYP/p/Qgqtlbw/lQf9IpwMuzbcG1ejt8g89OyDWNyt2ytgU +iaUnFJCos3/Byh0Iah/BsdOueo2/OJl2ZMOBW80orlSgv86cs2y037TL4wKBgQDm +OOyW+MlbowhnIvfoBfwlLEkefnej4nKD6WRLZBcue5Qyf355X06Mhsc9foXlH+6G +D9h/bswiHNdhp6N82rdgPGiHQx/CxiUoE/+b/nvgNO5mw6qLE2EXbG1e8pAMJcyE +bHw+YkawggDfELI036fRj5gki8SeUz8nS1nNgElbyQKBgCRDX9Jh+MwSLu4QBWdt +/fi+lv3K6kun/fI7EOV1vCV/j871tICu7pu5BrOLxAHqoVfU9AUX299/2KjCb5pv +kjogiUK6qWCWBlfuqDNWGCoUGt1rhznUva0nNjSMy5rinBhhjpROZC2pw48lOluP +UuvXsaPph7GTqPuy4Kab12YC +-----END PRIVATE KEY----- diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index ae62c6e7a0..a5d707edae 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1,11 +1,18 @@ #! /usr/bin/env python3 """ -Usage: +Script to run Python regression tests. +Run this script with -h or --help for documentation. +""" + +USAGE = """\ python -m test [options] [test_name1 [test_name2 ...]] python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] +""" +DESCRIPTION = """\ +Run Python regression tests. If no arguments or options are provided, finds all files matching the pattern "test_*" in the Lib/test subdirectory and runs @@ -15,63 +22,10 @@ For more rigorous testing, it is useful to use the following command line: python -E -Wd -m test [options] [test_name1 ...] +""" - -Options: - --h/--help -- print this text and exit ---timeout TIMEOUT - -- dump the traceback and exit if a test takes more - than TIMEOUT seconds; disabled if TIMEOUT is negative - or equals to zero ---wait -- wait for user input, e.g., allow a debugger to be attached - -Verbosity - --v/--verbose -- run tests in verbose mode with output to stdout --w/--verbose2 -- re-run failed tests in verbose mode --W/--verbose3 -- display test output on failure --d/--debug -- print traceback for failed tests --q/--quiet -- no output unless one or more tests fail --o/--slow -- print the slowest 10 tests - --header -- print header with interpreter info - -Selecting tests - --r/--randomize -- randomize test execution order (see below) - --randseed -- pass a random seed to reproduce a previous random run --f/--fromfile -- read names of tests to run from a file (see below) --x/--exclude -- arguments are tests to *exclude* --s/--single -- single step through a set of tests (see below) --m/--match PAT -- match test cases and methods with glob pattern PAT --G/--failfast -- fail as soon as a test fails (only with -v or -W) --u/--use RES1,RES2,... - -- specify which special resource intensive tests to run --M/--memlimit LIMIT - -- run very large memory-consuming tests - --testdir DIR - -- execute test files in the specified directory (instead - of the Python stdlib test suite) - -Special runs - --l/--findleaks -- if GC is available detect tests that leak memory --L/--runleaks -- run the leaks(1) command just before exit --R/--huntrleaks RUNCOUNTS - -- search for reference leaks (needs debug build, v. slow) --j/--multiprocess PROCESSES - -- run PROCESSES processes at once --T/--coverage -- turn on code coverage tracing using the trace module --D/--coverdir DIRECTORY - -- Directory where coverage files are put --N/--nocoverdir -- Put coverage files alongside modules --t/--threshold THRESHOLD - -- call gc.set_threshold(THRESHOLD) --n/--nowindows -- suppress error message boxes on Windows --F/--forever -- run the specified tests in a loop, until an error happens - - -Additional Option Details: +EPILOG = """\ +Additional option details: -r randomizes test execution order. You can use --randseed=int to provide a int seed value for the randomizer; this is useful for reproducing troublesome @@ -168,11 +122,12 @@ option '-uall,-gui'. # We import importlib *ASAP* in order to test #15386 import importlib +import argparse import builtins import faulthandler -import getopt import io import json +import locale import logging import os import platform @@ -194,7 +149,7 @@ try: except ImportError: threading = None try: - import multiprocessing.process + import _multiprocessing, multiprocessing.process except ImportError: multiprocessing = None @@ -246,20 +201,262 @@ from test import support RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') -TEMPDIR = os.path.abspath(tempfile.gettempdir()) - -def usage(msg): - print(msg, file=sys.stderr) - print("Use --help for usage", file=sys.stderr) - sys.exit(2) - +# When tests are run from the Python build directory, it is best practice +# to keep the test files in a subfolder. This eases the cleanup of leftover +# files using the "make distclean" command. +if sysconfig.is_python_build(): + TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') +else: + TEMPDIR = tempfile.gettempdir() +TEMPDIR = os.path.abspath(TEMPDIR) + +class _ArgParser(argparse.ArgumentParser): + + def error(self, message): + super().error(message + "\nPass -h or --help for complete help.") + +def _create_parser(): + # Set prog to prevent the uninformative "__main__.py" from displaying in + # error messages when using "python -m test ...". + parser = _ArgParser(prog='regrtest.py', + usage=USAGE, + description=DESCRIPTION, + epilog=EPILOG, + add_help=False, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # Arguments with this clause added to its help are described further in + # the epilog's "Additional option details" section. + more_details = ' See the section at bottom for more details.' + + group = parser.add_argument_group('General options') + # We add help explicitly to control what argument group it renders under. + group.add_argument('-h', '--help', action='help', + help='show this help message and exit') + group.add_argument('--timeout', metavar='TIMEOUT', type=float, + help='dump the traceback and exit if a test takes ' + 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'is negative or equals to zero') + group.add_argument('--wait', action='store_true', + help='wait for user input, e.g., allow a debugger ' + 'to be attached') + group.add_argument('--slaveargs', metavar='ARGS') + group.add_argument('-S', '--start', metavar='START', + help='the name of the test at which to start.' + + more_details) + + group = parser.add_argument_group('Verbosity') + group.add_argument('-v', '--verbose', action='count', + help='run tests in verbose mode with output to stdout') + group.add_argument('-w', '--verbose2', action='store_true', + help='re-run failed tests in verbose mode') + group.add_argument('-W', '--verbose3', action='store_true', + help='display test output on failure') + group.add_argument('-q', '--quiet', action='store_true', + help='no output unless one or more tests fail') + group.add_argument('-o', '--slow', action='store_true', dest='print_slow', + help='print the slowest 10 tests') + group.add_argument('--header', action='store_true', + help='print header with interpreter info') + + group = parser.add_argument_group('Selecting tests') + group.add_argument('-r', '--randomize', action='store_true', + help='randomize test execution order.' + more_details) + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a random seed to reproduce a previous ' + 'random run') + group.add_argument('-f', '--fromfile', metavar='FILE', + help='read names of tests to run from a file.' + + more_details) + group.add_argument('-x', '--exclude', action='store_true', + help='arguments are tests to *exclude*') + group.add_argument('-s', '--single', action='store_true', + help='single step through a set of tests.' + + more_details) + group.add_argument('-m', '--match', metavar='PAT', + dest='match_tests', + help='match test cases and methods with glob pattern PAT') + group.add_argument('-G', '--failfast', action='store_true', + help='fail as soon as a test fails (only with -v or -W)') + group.add_argument('-u', '--use', metavar='RES1,RES2,...', + action='append', type=resources_list, + help='specify which special resource intensive tests ' + 'to run.' + more_details) + group.add_argument('-M', '--memlimit', metavar='LIMIT', + help='run very large memory-consuming tests.' + + more_details) + group.add_argument('--testdir', metavar='DIR', + type=relative_filename, + help='execute test files in the specified directory ' + '(instead of the Python stdlib test suite)') + + group = parser.add_argument_group('Special runs') + group.add_argument('-l', '--findleaks', action='store_true', + help='if GC is available detect tests that leak memory') + group.add_argument('-L', '--runleaks', action='store_true', + help='run the leaks(1) command just before exit.' + + more_details) + group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', + type=huntrleaks, + help='search for reference leaks (needs debug build, ' + 'very slow).' + more_details) + group.add_argument('-j', '--multiprocess', metavar='PROCESSES', + dest='use_mp', type=int, + help='run PROCESSES processes at once') + group.add_argument('-T', '--coverage', action='store_true', + dest='trace', + help='turn on code coverage tracing using the trace ' + 'module') + group.add_argument('-D', '--coverdir', metavar='DIR', + type=relative_filename, + help='directory where coverage files are put') + group.add_argument('-N', '--nocoverdir', + action='store_const', const=None, dest='coverdir', + help='put coverage files alongside modules') + group.add_argument('-t', '--threshold', metavar='THRESHOLD', + type=int, + help='call gc.set_threshold(THRESHOLD)') + group.add_argument('-n', '--nowindows', action='store_true', + help='suppress error message boxes on Windows') + group.add_argument('-F', '--forever', action='store_true', + help='run the specified tests in a loop, until an ' + 'error happens') + + parser.add_argument('args', nargs=argparse.REMAINDER, + help=argparse.SUPPRESS) + + return parser + +def relative_filename(string): + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + return os.path.join(support.SAVEDCWD, string) + +def huntrleaks(string): + args = string.split(':') + if len(args) not in (2, 3): + raise argparse.ArgumentTypeError( + 'needs 2 or 3 colon-separated arguments') + nwarmup = int(args[0]) if args[0] else 5 + ntracked = int(args[1]) if args[1] else 4 + fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' + return nwarmup, ntracked, fname + +def resources_list(string): + u = [x.lower() for x in string.split(',')] + for r in u: + if r == 'all' or r == 'none': + continue + if r[0] == '-': + r = r[1:] + if r not in RESOURCE_NAMES: + raise argparse.ArgumentTypeError('invalid resource: ' + r) + return u -def main(tests=None, testdir=None, verbose=0, quiet=False, +def _parse_args(args, **kwargs): + # Defaults + ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, exclude=False, single=False, randomize=False, fromfile=None, findleaks=False, use_resources=None, trace=False, coverdir='coverage', runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None): + header=False, failfast=False, match_tests=None) + for k, v in kwargs.items(): + if not hasattr(ns, k): + raise TypeError('%r is an invalid keyword argument ' + 'for this function' % k) + setattr(ns, k, v) + if ns.use_resources is None: + ns.use_resources = [] + + parser = _create_parser() + parser.parse_args(args=args, namespace=ns) + + if ns.single and ns.fromfile: + parser.error("-s and -f don't go together!") + if ns.use_mp and ns.trace: + parser.error("-T and -j don't go together!") + if ns.use_mp and ns.findleaks: + parser.error("-l and -j don't go together!") + if ns.use_mp and ns.memlimit: + parser.error("-M and -j don't go together!") + if ns.failfast and not (ns.verbose or ns.verbose3): + parser.error("-G/--failfast needs either -v or -W") + + if ns.quiet: + ns.verbose = 0 + if ns.timeout is not None: + if hasattr(faulthandler, 'dump_traceback_later'): + if ns.timeout <= 0: + ns.timeout = None + else: + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later") + ns.timeout = None + if ns.use_mp is not None: + if ns.use_mp <= 0: + # Use all cores + extras for tests that like to sleep + ns.use_mp = 2 + (os.cpu_count() or 1) + if ns.use_mp == 1: + ns.use_mp = None + if ns.use: + for a in ns.use: + for r in a: + if r == 'all': + ns.use_resources[:] = RESOURCE_NAMES + continue + if r == 'none': + del ns.use_resources[:] + continue + remove = False + if r[0] == '-': + remove = True + r = r[1:] + if remove: + if r in ns.use_resources: + ns.use_resources.remove(r) + elif r not in ns.use_resources: + ns.use_resources.append(r) + if ns.random_seed is not None: + ns.randomize = True + + return ns + + +def run_test_in_subprocess(testname, ns): + """Run the given test in a subprocess with --slaveargs. + + ns is the option Namespace parsed from command-line arguments. regrtest + is invoked in a subprocess with the --slaveargs argument; when the + subprocess exits, its return code, stdout and stderr are returned as a + 3-tuple. + """ + from subprocess import Popen, PIPE + base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + + ['-X', 'faulthandler', '-m', 'test.regrtest']) + + slaveargs = ( + (testname, ns.verbose, ns.quiet), + dict(huntrleaks=ns.huntrleaks, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + timeout=ns.timeout, failfast=ns.failfast, + match_tests=ns.match_tests)) + # Running the child from the same working directory as regrtest's original + # invocation ensures that TEMPDIR for the child is the same when + # sysconfig.is_python_build() is true. See issue 15300. + popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], + stdout=PIPE, stderr=PIPE, + universal_newlines=True, + close_fds=(os.name != 'nt'), + cwd=support.SAVEDCWD) + stdout, stderr = popen.communicate() + retcode = popen.wait() + return retcode, stdout, stderr + + +def main(tests=None, **kwargs): """Execute a test suite. This also parses command-line options and modifies its behavior @@ -282,7 +479,6 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, directly to set the values that would normally be set by flags on the command line. """ - # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) @@ -298,187 +494,51 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, replace_stdout() support.record_original_stdout(sys.stdout) - try: - opts, args = getopt.getopt(sys.argv[1:], 'hvqxsoS:rf:lu:t:TD:NLR:FdwWM:nj:Gm:', - ['help', 'verbose', 'verbose2', 'verbose3', 'quiet', - 'exclude', 'single', 'slow', 'randomize', 'fromfile=', 'findleaks', - 'use=', 'threshold=', 'coverdir=', 'nocoverdir', - 'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=', - 'multiprocess=', 'coverage', 'slaveargs=', 'forever', 'debug', - 'start=', 'nowindows', 'header', 'testdir=', 'timeout=', 'wait', - 'failfast', 'match=']) - except getopt.error as msg: - usage(msg) - # Defaults - if random_seed is None: - random_seed = random.randrange(10000000) - if use_resources is None: - use_resources = [] - debug = False - start = None - timeout = None - for o, a in opts: - if o in ('-h', '--help'): - print(__doc__) - return - elif o in ('-v', '--verbose'): - verbose += 1 - elif o in ('-w', '--verbose2'): - verbose2 = True - elif o in ('-d', '--debug'): - debug = True - elif o in ('-W', '--verbose3'): - verbose3 = True - elif o in ('-G', '--failfast'): - failfast = True - elif o in ('-q', '--quiet'): - quiet = True; - verbose = 0 - elif o in ('-x', '--exclude'): - exclude = True - elif o in ('-S', '--start'): - start = a - elif o in ('-s', '--single'): - single = True - elif o in ('-o', '--slow'): - print_slow = True - elif o in ('-r', '--randomize'): - randomize = True - elif o == '--randseed': - randomize = True - random_seed = int(a) - elif o in ('-f', '--fromfile'): - fromfile = a - elif o in ('-m', '--match'): - match_tests = a - elif o in ('-l', '--findleaks'): - findleaks = True - elif o in ('-L', '--runleaks'): - runleaks = True - elif o in ('-t', '--threshold'): - import gc - gc.set_threshold(int(a)) - elif o in ('-T', '--coverage'): - trace = True - elif o in ('-D', '--coverdir'): - # CWD is replaced with a temporary dir before calling main(), so we - # need join it with the saved CWD so it goes where the user expects. - coverdir = os.path.join(support.SAVEDCWD, a) - elif o in ('-N', '--nocoverdir'): - coverdir = None - elif o in ('-R', '--huntrleaks'): - huntrleaks = a.split(':') - if len(huntrleaks) not in (2, 3): - print(a, huntrleaks) - usage('-R takes 2 or 3 colon-separated arguments') - if not huntrleaks[0]: - huntrleaks[0] = 5 - else: - huntrleaks[0] = int(huntrleaks[0]) - if not huntrleaks[1]: - huntrleaks[1] = 4 - else: - huntrleaks[1] = int(huntrleaks[1]) - if len(huntrleaks) == 2 or not huntrleaks[2]: - huntrleaks[2:] = ["reflog.txt"] - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - elif o in ('-M', '--memlimit'): - support.set_memlimit(a) - elif o in ('-u', '--use'): - u = [x.lower() for x in a.split(',')] - for r in u: - if r == 'all': - use_resources[:] = RESOURCE_NAMES - continue - if r == 'none': - del use_resources[:] - continue - remove = False - if r[0] == '-': - remove = True - r = r[1:] - if r not in RESOURCE_NAMES: - usage('Invalid -u/--use option: ' + a) - if remove: - if r in use_resources: - use_resources.remove(r) - elif r not in use_resources: - use_resources.append(r) - elif o in ('-n', '--nowindows'): - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - elif o in ('-F', '--forever'): - forever = True - elif o in ('-j', '--multiprocess'): - use_mp = int(a) - if use_mp <= 0: - try: - import multiprocessing - # Use all cores + extras for tests that like to sleep - use_mp = 2 + multiprocessing.cpu_count() - except (ImportError, NotImplementedError): - use_mp = 3 - if use_mp == 1: - use_mp = None - elif o == '--header': - header = True - elif o == '--slaveargs': - args, kwargs = json.loads(a) - try: - result = runtest(*args, **kwargs) - except KeyboardInterrupt: - result = INTERRUPTED, '' - except BaseException as e: - traceback.print_exc() - result = CHILD_ERROR, str(e) - sys.stdout.flush() - print() # Force a newline (just in case) - print(json.dumps(result)) - sys.exit(0) - elif o == '--testdir': - # CWD is replaced with a temporary dir before calling main(), so we - # join it with the saved CWD so it ends up where the user expects. - testdir = os.path.join(support.SAVEDCWD, a) - elif o == '--timeout': - if hasattr(faulthandler, 'dump_traceback_later'): - timeout = float(a) - if timeout <= 0: - timeout = None - else: - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later") - timeout = None - elif o == '--wait': - input("Press any key to continue...") + ns = _parse_args(sys.argv[1:], **kwargs) + + if ns.huntrleaks: + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() + if ns.memlimit is not None: + support.set_memlimit(ns.memlimit) + if ns.threshold is not None: + import gc + gc.set_threshold(ns.threshold) + if ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass else: - print(("No handler for option {}. Please report this as a bug " - "at http://bugs.python.org.").format(o), file=sys.stderr) - sys.exit(1) - if single and fromfile: - usage("-s and -f don't go together!") - if use_mp and trace: - usage("-T and -j don't go together!") - if use_mp and findleaks: - usage("-l and -j don't go together!") - if use_mp and support.max_memuse: - usage("-M and -j don't go together!") - if failfast and not (verbose or verbose3): - usage("-G/--failfast needs either -v or -W") + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + if ns.wait: + input("Press any key to continue...") + + if ns.slaveargs is not None: + args, kwargs = json.loads(ns.slaveargs) + if kwargs.get('huntrleaks'): + unittest.BaseTestSuite._cleanup = False + try: + result = runtest(*args, **kwargs) + except KeyboardInterrupt: + result = INTERRUPTED, '' + except BaseException as e: + traceback.print_exc() + result = CHILD_ERROR, str(e) + sys.stdout.flush() + print() # Force a newline (just in case) + print(json.dumps(result)) + sys.exit(0) good = [] bad = [] @@ -487,12 +547,12 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, environment_changed = [] interrupted = False - if findleaks: + if ns.findleaks: try: import gc except ImportError: print('No GC available, disabling findleaks.') - findleaks = False + ns.findleaks = False else: # Uncomment the line below to report garbage that is not # freeable by reference counting alone. By default only @@ -500,42 +560,43 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, #gc.set_debug(gc.DEBUG_SAVEALL) found_garbage = [] - if single: + if ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + if ns.single: filename = os.path.join(TEMPDIR, 'pynexttest') try: - fp = open(filename, 'r') - next_test = fp.read().strip() - tests = [next_test] - fp.close() - except IOError: + with open(filename, 'r') as fp: + next_test = fp.read().strip() + tests = [next_test] + except OSError: pass - if fromfile: + if ns.fromfile: tests = [] - fp = open(os.path.join(support.SAVEDCWD, fromfile)) - count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') - for line in fp: - line = count_pat.sub('', line) - guts = line.split() # assuming no test has whitespace in its name - if guts and not guts[0].startswith('#'): - tests.extend(guts) - fp.close() + with open(os.path.join(support.SAVEDCWD, ns.fromfile)) as fp: + count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') + for line in fp: + line = count_pat.sub('', line) + guts = line.split() # assuming no test has whitespace in its name + if guts and not guts[0].startswith('#'): + tests.extend(guts) # Strip .py extensions. - removepy(args) + removepy(ns.args) removepy(tests) stdtests = STDTESTS[:] nottests = NOTTESTS.copy() - if exclude: - for arg in args: + if ns.exclude: + for arg in ns.args: if arg in stdtests: stdtests.remove(arg) nottests.add(arg) - args = [] + ns.args = [] # For a partial run, we do not need to clutter the output. - if verbose or header or not (quiet or single or tests or args): + if ns.verbose or ns.header or not (ns.quiet or ns.single or tests or ns.args): # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) print("== ", platform.platform(aliased=True), @@ -545,37 +606,39 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, # if testdir is set, then we are not running the python tests suite, so # don't add default tests to be executed or skipped (pass empty values) - if testdir: - alltests = findtests(testdir, list(), set()) + if ns.testdir: + alltests = findtests(ns.testdir, list(), set()) else: - alltests = findtests(testdir, stdtests, nottests) + alltests = findtests(ns.testdir, stdtests, nottests) - selected = tests or args or alltests - if single: + selected = tests or ns.args or alltests + if ns.single: selected = selected[:1] try: next_single_test = alltests[alltests.index(selected[0])+1] except IndexError: next_single_test = None # Remove all the selected tests that precede start if it's set. - if start: + if ns.start: try: - del selected[:selected.index(start)] + del selected[:selected.index(ns.start)] except ValueError: - print("Couldn't find starting test (%s), using all tests" % start) - if randomize: - random.seed(random_seed) - print("Using random seed", random_seed) + print("Couldn't find starting test (%s), using all tests" % ns.start) + if ns.randomize: + if ns.random_seed is None: + ns.random_seed = random.randrange(10000000) + random.seed(ns.random_seed) + print("Using random seed", ns.random_seed) random.shuffle(selected) - if trace: + if ns.trace: import trace, tempfile tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, tempfile.gettempdir()], trace=False, count=True) test_times = [] - support.verbose = verbose # Tell tests to be moderately quiet - support.use_resources = use_resources + support.verbose = ns.verbose # Tell tests to be moderately quiet + support.use_resources = ns.use_resources save_modules = sys.modules.keys() def accumulate_result(test, result): @@ -593,7 +656,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, skipped.append(test) resource_denieds.append(test) - if forever: + if ns.forever: def test_forever(tests=list(selected)): while True: for test in tests: @@ -608,19 +671,16 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, test_count = '/{}'.format(len(selected)) test_count_width = len(test_count) - 1 - if use_mp: + if ns.use_mp: try: from threading import Thread except ImportError: print("Multiprocess option requires thread support") sys.exit(2) from queue import Queue - from subprocess import Popen, PIPE - debug_output_pat = re.compile(r"\[\d+ refs\]$") + debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") output = Queue() pending = MultiprocessTests(tests) - opt_args = support.args_from_interpreter_flags() - base_cmd = [sys.executable] + opt_args + ['-m', 'test.regrtest'] def work(): # A worker thread. try: @@ -630,24 +690,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, except StopIteration: output.put((None, None, None, None)) return - args_tuple = ( - (test, verbose, quiet), - dict(huntrleaks=huntrleaks, use_resources=use_resources, - debug=debug, output_on_failure=verbose3, - timeout=timeout, failfast=failfast, - match_tests=match_tests) - ) - # -E is needed by some tests, e.g. test_import - # Running the child from the same working directory ensures - # that TEMPDIR for the child is the same when - # sysconfig.is_python_build() is true. See issue 15300. - popen = Popen(base_cmd + ['--slaveargs', json.dumps(args_tuple)], - stdout=PIPE, stderr=PIPE, - universal_newlines=True, - close_fds=(os.name != 'nt'), - cwd=support.SAVEDCWD) - stdout, stderr = popen.communicate() - retcode = popen.wait() + retcode, stdout, stderr = run_test_in_subprocess(test, ns) # Strip last refcount output line if it exists, since it # comes from the shutdown of the interpreter in the subcommand. stderr = debug_output_pat.sub("", stderr) @@ -664,19 +707,19 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, except BaseException: output.put((None, None, None, None)) raise - workers = [Thread(target=work) for i in range(use_mp)] + workers = [Thread(target=work) for i in range(ns.use_mp)] for worker in workers: worker.start() finished = 0 test_index = 1 try: - while finished < use_mp: + while finished < ns.use_mp: test, stdout, stderr, result = output.get() if test is None: finished += 1 continue accumulate_result(test, result) - if not quiet: + if not ns.quiet: fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" print(fmt.format( test_count_width, test_index, test_count, @@ -699,29 +742,30 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, worker.join() else: for test_index, test in enumerate(tests, 1): - if not quiet: + if not ns.quiet: fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" print(fmt.format( test_count_width, test_index, test_count, len(bad), test)) sys.stdout.flush() - if trace: + if ns.trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - tracer.runctx('runtest(test, verbose, quiet, timeout=timeout)', + tracer.runctx('runtest(test, ns.verbose, ns.quiet, timeout=ns.timeout)', globals=globals(), locals=vars()) else: try: - result = runtest(test, verbose, quiet, huntrleaks, debug, - output_on_failure=verbose3, - timeout=timeout, failfast=failfast, - match_tests=match_tests) + result = runtest(test, ns.verbose, ns.quiet, + ns.huntrleaks, + output_on_failure=ns.verbose3, + timeout=ns.timeout, failfast=ns.failfast, + match_tests=ns.match_tests) accumulate_result(test, result) except KeyboardInterrupt: interrupted = True break except: raise - if findleaks: + if ns.findleaks: gc.collect() if gc.garbage: print("Warning: test created", len(gc.garbage), end=' ') @@ -742,11 +786,11 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, omitted = set(selected) - set(good) - set(bad) - set(skipped) print(count(len(omitted), "test"), "omitted:") printlist(omitted) - if good and not quiet: + if good and not ns.quiet: if not bad and not skipped and not interrupted and len(good) > 1: print("All", end=' ') print(count(len(good), "test"), "OK.") - if print_slow: + if ns.print_slow: test_times.sort(reverse=True) print("10 slowest tests:") for time, test in test_times[:10]: @@ -760,32 +804,19 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, print("{} altered the execution environment:".format( count(len(environment_changed), "test"))) printlist(environment_changed) - if skipped and not quiet: + if skipped and not ns.quiet: print(count(len(skipped), "test"), "skipped:") printlist(skipped) - e = _ExpectedSkips() - plat = sys.platform - if e.isvalid(): - surprise = set(skipped) - e.getexpected() - set(resource_denieds) - if surprise: - print(count(len(surprise), "skip"), \ - "unexpected on", plat + ":") - printlist(surprise) - else: - print("Those skips are all expected on", plat + ".") - else: - print("Ask someone to teach regrtest.py about which tests are") - print("expected to get skipped on", plat + ".") - - if verbose2 and bad: + if ns.verbose2 and bad: print("Re-running failed tests in verbose mode") for test in bad: print("Re-running test %r in verbose mode" % test) sys.stdout.flush() try: - verbose = True - ok = runtest(test, True, quiet, huntrleaks, debug, timeout=timeout) + ns.verbose = True + ok = runtest(test, True, ns.quiet, ns.huntrleaks, + timeout=ns.timeout) except KeyboardInterrupt: # print a newline separate from the ^C print() @@ -793,18 +824,18 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, except: raise - if single: + if ns.single: if next_single_test: with open(filename, 'w') as fp: fp.write(next_single_test + '\n') else: os.unlink(filename) - if trace: + if ns.trace: r = tracer.results() - r.write_results(show_missing=True, summary=True, coverdir=coverdir) + r.write_results(show_missing=True, summary=True, coverdir=ns.coverdir) - if runleaks: + if ns.runleaks: os.system("leaks %d" % os.getpid()) sys.exit(len(bad) > 0 or interrupted) @@ -877,7 +908,7 @@ def replace_stdout(): atexit.register(restore_stdout) def runtest(test, verbose, quiet, - huntrleaks=False, debug=False, use_resources=None, + huntrleaks=False, use_resources=None, output_on_failure=False, failfast=False, match_tests=None, timeout=None): """Run a single test. @@ -885,14 +916,15 @@ def runtest(test, verbose, quiet, test -- the name of the test verbose -- if true, print more messages quiet -- if true, don't print 'skipped' messages (probably redundant) - test_times -- a list of (time, test_name) pairs huntrleaks -- run multiple times to test for leaks; requires a debug build; a triple corresponding to -R's three arguments + use_resources -- list of extra resources to use output_on_failure -- if true, display test output on failure timeout -- dump the traceback and exit if a test takes more than timeout seconds + failfast, match_tests -- See regrtest command-line flags for these. - Returns one of the test result constants: + Returns the tuple result, test_time, where result is one of the constants: INTERRUPTED KeyboardInterrupt when run under -j RESOURCE_DENIED test skipped because resource denied SKIPPED test skipped for some other reason @@ -930,7 +962,7 @@ def runtest(test, verbose, quiet, sys.stdout = stream sys.stderr = stream result = runtest_inner(test, verbose, quiet, huntrleaks, - debug, display_failure=False) + display_failure=False) if result[0] == FAILED: output = stream.getvalue() orig_stderr.write(output) @@ -940,7 +972,7 @@ def runtest(test, verbose, quiet, sys.stderr = orig_stderr else: support.verbose = verbose # Tell tests to be moderately quiet - result = runtest_inner(test, verbose, quiet, huntrleaks, debug, + result = runtest_inner(test, verbose, quiet, huntrleaks, display_failure=not verbose) return result finally: @@ -992,10 +1024,12 @@ class saved_test_environment: 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', 'warnings.filters', 'asyncore.socket_map', 'logging._handlers', 'logging._handlerList', 'sys.gettrace', - 'sys.warnoptions', 'threading._dangling', - 'multiprocessing.process._dangling', + 'sys.warnoptions', + # multiprocessing.process._cleanup() may release ref + # to a thread, so check processes first. + 'multiprocessing.process._dangling', 'threading._dangling', 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', - 'support.TESTFN', + 'support.TESTFN', 'locale', 'warnings.showwarning', ) def get_sys_argv(self): @@ -1123,6 +1157,8 @@ class saved_test_environment: def get_multiprocessing_process__dangling(self): if not multiprocessing: return None + # Unjoined process objects can survive after process exits + multiprocessing.process._cleanup() # This copies the weakrefs without making any strong reference return multiprocessing.process._dangling.copy() def restore_multiprocessing_process__dangling(self, saved): @@ -1164,6 +1200,25 @@ class saved_test_environment: elif os.path.isdir(support.TESTFN): shutil.rmtree(support.TESTFN) + _lc = [getattr(locale, lc) for lc in dir(locale) + if lc.startswith('LC_')] + def get_locale(self): + pairings = [] + for lc in self._lc: + try: + pairings.append((lc, locale.setlocale(lc, None))) + except (TypeError, ValueError): + continue + return pairings + def restore_locale(self, saved): + for lc, setting in saved: + locale.setlocale(lc, setting) + + def get_warnings_showwarning(self): + return warnings.showwarning + def restore_warnings_showwarning(self, fxn): + warnings.showwarning = fxn + def resource_info(self): for name in self.resources: method_suffix = name.replace('.', '_') @@ -1198,7 +1253,7 @@ class saved_test_environment: def runtest_inner(test, verbose, quiet, - huntrleaks=False, debug=False, display_failure=True): + huntrleaks=False, display_failure=True): support.unload(test) test_time = 0.0 @@ -1211,8 +1266,7 @@ def runtest_inner(test, verbose, quiet, abstest = 'test.' + test with saved_test_environment(test, verbose, quiet) as environment: start_time = time.time() - the_package = __import__(abstest, globals(), locals(), []) - the_module = getattr(the_package, test) + the_module = importlib.import_module(abstest) # If the test has a test_main, that will run the appropriate # tests. If not, use normal unittest test loading. test_runner = getattr(the_module, "test_main", None) @@ -1221,8 +1275,7 @@ def runtest_inner(test, verbose, quiet, test_runner = lambda: support.run_unittest(tests) test_runner() if huntrleaks: - refleak = dash_R(the_module, test, test_runner, - huntrleaks) + refleak = dash_R(the_module, test, test_runner, huntrleaks) test_time = time.time() - start_time except support.ResourceDenied as msg: if not quiet: @@ -1328,41 +1381,50 @@ def dash_R(the_module, test, indirect_test, huntrleaks): for obj in abc.__subclasses__() + [abc]: abcs[obj] = obj._abc_registry.copy() - if indirect_test: - def run_the_test(): - indirect_test() - else: - def run_the_test(): - del sys.modules[the_module.__name__] - exec('import ' + the_module.__name__) - - deltas = [] nwarmup, ntracked, fname = huntrleaks fname = os.path.join(support.SAVEDCWD, fname) repcount = nwarmup + ntracked + rc_deltas = [0] * repcount + alloc_deltas = [0] * repcount + print("beginning", repcount, "repetitions", file=sys.stderr) print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr) sys.stderr.flush() - dash_R_cleanup(fs, ps, pic, zdc, abcs) for i in range(repcount): - rc_before = sys.gettotalrefcount() - run_the_test() + indirect_test() + alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) sys.stderr.write('.') sys.stderr.flush() - dash_R_cleanup(fs, ps, pic, zdc, abcs) - rc_after = sys.gettotalrefcount() if i >= nwarmup: - deltas.append(rc_after - rc_before) + rc_deltas[i] = rc_after - rc_before + alloc_deltas[i] = alloc_after - alloc_before + alloc_before, rc_before = alloc_after, rc_after print(file=sys.stderr) - if any(deltas): - msg = '%s leaked %s references, sum=%s' % (test, deltas, sum(deltas)) - print(msg, file=sys.stderr) - sys.stderr.flush() - with open(fname, "a") as refrep: - print(msg, file=refrep) - refrep.flush() - return True - return False + # These checkers return False on success, True on failure + def check_rc_deltas(deltas): + return any(deltas) + def check_alloc_deltas(deltas): + # At least 1/3rd of 0s + if 3 * deltas.count(0) < len(deltas): + return True + # Nothing else than 1s, 0s and -1s + if not set(deltas) <= {1,0,-1}: + return True + return False + failed = False + for deltas, item_name, checker in [ + (rc_deltas, 'references', check_rc_deltas), + (alloc_deltas, 'memory blocks', check_alloc_deltas)]: + if checker(deltas): + msg = '%s leaked %s %s, sum=%s' % ( + test, deltas[nwarmup:], item_name, sum(deltas)) + print(msg, file=sys.stderr) + sys.stderr.flush() + with open(fname, "a") as refrep: + print(msg, file=refrep) + refrep.flush() + failed = True + return failed def dash_R_cleanup(fs, ps, pic, zdc, abcs): import gc, copyreg @@ -1428,8 +1490,11 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): else: ctypes._reset_cache() - # Collect cyclic trash. + # Collect cyclic trash and read memory statistics immediately after. + func1 = sys.getallocatedblocks + func2 = sys.gettotalrefcount gc.collect() + return func1(), func2() def warm_caches(): # char cache @@ -1472,307 +1537,10 @@ def printlist(x, width=70, indent=4): print(fill(' '.join(str(elt) for elt in sorted(x)), width, initial_indent=blanks, subsequent_indent=blanks)) -# Map sys.platform to a string containing the basenames of tests -# expected to be skipped on that platform. -# -# Special cases: -# test_pep277 -# The _ExpectedSkips constructor adds this to the set of expected -# skips if not os.path.supports_unicode_filenames. -# test_timeout -# Controlled by test_timeout.skip_expected. Requires the network -# resource and a socket module. -# -# Tests that are expected to be skipped everywhere except on one platform -# are also handled separately. - -_expectations = ( - ('win32', - """ - test__locale - test_crypt - test_curses - test_dbm - test_devpoll - test_fcntl - test_fork1 - test_epoll - test_dbm_gnu - test_dbm_ndbm - test_grp - test_ioctl - test_largefile - test_kqueue - test_openpty - test_ossaudiodev - test_pipes - test_poll - test_posix - test_pty - test_pwd - test_resource - test_signal - test_syslog - test_threadsignals - test_wait3 - test_wait4 - """), - ('linux', - """ - test_curses - test_devpoll - test_largefile - test_kqueue - test_ossaudiodev - """), - ('unixware', - """ - test_epoll - test_largefile - test_kqueue - test_minidom - test_openpty - test_pyexpat - test_sax - test_sundry - """), - ('openunix', - """ - test_epoll - test_largefile - test_kqueue - test_minidom - test_openpty - test_pyexpat - test_sax - test_sundry - """), - ('sco_sv', - """ - test_asynchat - test_fork1 - test_epoll - test_gettext - test_largefile - test_locale - test_kqueue - test_minidom - test_openpty - test_pyexpat - test_queue - test_sax - test_sundry - test_thread - test_threaded_import - test_threadedtempfile - test_threading - """), - ('darwin', - """ - test__locale - test_curses - test_devpoll - test_epoll - test_dbm_gnu - test_gdb - test_largefile - test_locale - test_minidom - test_ossaudiodev - test_poll - """), - ('sunos', - """ - test_curses - test_dbm - test_epoll - test_kqueue - test_dbm_gnu - test_gzip - test_openpty - test_zipfile - test_zlib - """), - ('hp-ux', - """ - test_curses - test_epoll - test_dbm_gnu - test_gzip - test_largefile - test_locale - test_kqueue - test_minidom - test_openpty - test_pyexpat - test_sax - test_zipfile - test_zlib - """), - ('cygwin', - """ - test_curses - test_dbm - test_devpoll - test_epoll - test_ioctl - test_kqueue - test_largefile - test_locale - test_ossaudiodev - test_socketserver - """), - ('os2emx', - """ - test_audioop - test_curses - test_epoll - test_kqueue - test_largefile - test_mmap - test_openpty - test_ossaudiodev - test_pty - test_resource - test_signal - """), - ('freebsd', - """ - test_devpoll - test_epoll - test_dbm_gnu - test_locale - test_ossaudiodev - test_pep277 - test_pty - test_socketserver - test_tcl - test_tk - test_ttk_guionly - test_ttk_textonly - test_timeout - test_urllibnet - test_multiprocessing - """), - ('aix', - """ - test_bz2 - test_epoll - test_dbm_gnu - test_gzip - test_kqueue - test_ossaudiodev - test_tcl - test_tk - test_ttk_guionly - test_ttk_textonly - test_zipimport - test_zlib - """), - ('openbsd', - """ - test_ctypes - test_devpoll - test_epoll - test_dbm_gnu - test_locale - test_normalization - test_ossaudiodev - test_pep277 - test_tcl - test_tk - test_ttk_guionly - test_ttk_textonly - test_multiprocessing - """), - ('netbsd', - """ - test_ctypes - test_curses - test_devpoll - test_epoll - test_dbm_gnu - test_locale - test_ossaudiodev - test_pep277 - test_tcl - test_tk - test_ttk_guionly - test_ttk_textonly - test_multiprocessing - """), -) - -class _ExpectedSkips: - def __init__(self): - import os.path - from test import test_timeout - - self.valid = False - expected = None - for item in _expectations: - if sys.platform.startswith(item[0]): - expected = item[1] - break - if expected is not None: - self.expected = set(expected.split()) - - # These are broken tests, for now skipped on every platform. - # XXX Fix these! - self.expected.add('test_nis') - - # expected to be skipped on every platform, even Linux - if not os.path.supports_unicode_filenames: - self.expected.add('test_pep277') - - # doctest, profile and cProfile tests fail when the codec for the - # fs encoding isn't built in because PyUnicode_Decode() adds two - # calls into Python. - encs = ("utf-8", "latin-1", "ascii", "mbcs", "utf-16", "utf-32") - if sys.getfilesystemencoding().lower() not in encs: - self.expected.add('test_profile') - self.expected.add('test_cProfile') - self.expected.add('test_doctest') - - if test_timeout.skip_expected: - self.expected.add('test_timeout') - - if sys.platform != "win32": - # test_sqlite is only reliable on Windows where the library - # is distributed with Python - WIN_ONLY = {"test_unicode_file", "test_winreg", - "test_winsound", "test_startfile", - "test_sqlite", "test_msilib"} - self.expected |= WIN_ONLY - - if sys.platform != 'sunos5': - self.expected.add('test_nis') - - if support.python_is_optimized(): - self.expected.add("test_gdb") - - self.valid = True - - def isvalid(self): - "Return true iff _ExpectedSkips knows about the current platform." - return self.valid - - def getexpected(self): - """Return set of test names we expect to skip on current platform. - - self.isvalid() must be true. - """ - - assert self.isvalid() - return self.expected - -def _make_temp_dir_for_build(TEMPDIR): - # When tests are run from the Python build directory, it is best practice - # to keep the test files in a subfolder. It eases the cleanup of leftover - # files using command "make distclean". + +def main_in_temp_cwd(): + """Run main() in a temporary working directory.""" if sysconfig.is_python_build(): - TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') - TEMPDIR = os.path.abspath(TEMPDIR) try: os.mkdir(TEMPDIR) except FileExistsError: @@ -1781,10 +1549,16 @@ def _make_temp_dir_for_build(TEMPDIR): # Define a writable temp dir that will be used as cwd while running # the tests. The name of the dir includes the pid to allow parallel # testing (see the -j option). - TESTCWD = 'test_python_{}'.format(os.getpid()) + test_cwd = 'test_python_{}'.format(os.getpid()) + test_cwd = os.path.join(TEMPDIR, test_cwd) + + # Run the tests in a context manager that temporarily changes the CWD to a + # temporary and writable directory. If it's not possible to create or + # change the CWD, the original CWD will be used. The original CWD is + # available from support.SAVEDCWD. + with support.temp_cwd(test_cwd, quiet=True): + main() - TESTCWD = os.path.join(TEMPDIR, TESTCWD) - return TEMPDIR, TESTCWD if __name__ == '__main__': # Remove regrtest.py's own directory from the module search path. Despite @@ -1808,11 +1582,4 @@ if __name__ == '__main__': # sanity check assert __file__ == os.path.abspath(sys.argv[0]) - TEMPDIR, TESTCWD = _make_temp_dir_for_build(TEMPDIR) - - # Run the tests in a context manager that temporary changes the CWD to a - # temporary and writable directory. If it's not possible to create or - # change the CWD, the original CWD will be used. The original CWD is - # available from support.SAVEDCWD. - with support.temp_cwd(TESTCWD, quiet=True): - main() + main_in_temp_cwd() diff --git a/Lib/test/script_helper.py b/Lib/test/script_helper.py index ab201649e5..4d5c1f120a 100644 --- a/Lib/test/script_helper.py +++ b/Lib/test/script_helper.py @@ -12,13 +12,22 @@ import contextlib import shutil import zipfile -from imp import source_from_cache +from importlib.util import source_from_cache from test.support import make_legacy_pyc, strip_python_stderr, temp_dir # Executing the interpreter in a subprocess def _assert_python(expected_success, *args, **env_vars): - cmd_line = [sys.executable] - if not env_vars: + if '__isolated' in env_vars: + isolated = env_vars.pop('__isolated') + else: + isolated = not env_vars + cmd_line = [sys.executable, '-X', 'faulthandler'] + if isolated: + # isolated mode: ignore Python environment variables, ignore user + # site-packages, and don't add the current directory to sys.path + cmd_line.append('-I') + elif not env_vars: + # ignore Python environment variables cmd_line.append('-E') # Need to preserve the original environment, for in-place testing of # shared library builds. @@ -39,7 +48,7 @@ def _assert_python(expected_success, *args, **env_vars): p.stdout.close() p.stderr.close() rc = p.returncode - err = strip_python_stderr(err) + err = strip_python_stderr(err) if (rc and expected_success) or (not rc and not expected_success): raise AssertionError( "Process return code is %d, " @@ -49,18 +58,32 @@ def _assert_python(expected_success, *args, **env_vars): def assert_python_ok(*args, **env_vars): """ Assert that running the interpreter with `args` and optional environment - variables `env_vars` is ok and return a (return code, stdout, stderr) tuple. + variables `env_vars` succeeds (rc == 0) and return a (return code, stdout, + stderr) tuple. + + If the __cleanenv keyword is set, env_vars is used a fresh environment. + + Python is started in isolated mode (command line option -I), + except if the __isolated keyword is set to False. """ return _assert_python(True, *args, **env_vars) def assert_python_failure(*args, **env_vars): """ Assert that running the interpreter with `args` and optional environment - variables `env_vars` fails and return a (return code, stdout, stderr) tuple. + variables `env_vars` fails (rc != 0) and return a (return code, stdout, + stderr) tuple. + + See assert_python_ok() for more options. """ return _assert_python(False, *args, **env_vars) def spawn_python(*args, **kw): + """Run a Python subprocess with the given arguments. + + kw is extra keyword args to pass to subprocess.Popen. Returns a Popen + object. + """ cmd_line = [sys.executable, '-E'] cmd_line.extend(args) return subprocess.Popen(cmd_line, stdin=subprocess.PIPE, @@ -68,6 +91,7 @@ def spawn_python(*args, **kw): **kw) def kill_python(p): + """Run the given Popen process until completion and return stdout.""" p.stdin.close() data = p.stdout.read() p.stdout.close() diff --git a/Lib/test/sortperf.py b/Lib/test/sortperf.py index af7c0b4bd1..90722f7abc 100644 --- a/Lib/test/sortperf.py +++ b/Lib/test/sortperf.py @@ -22,7 +22,7 @@ def randfloats(n): fn = os.path.join(td, "rr%06d" % n) try: fp = open(fn, "rb") - except IOError: + except OSError: r = random.random result = [r() for i in range(n)] try: @@ -35,9 +35,9 @@ def randfloats(n): if fp: try: os.unlink(fn) - except os.error: + except OSError: pass - except IOError as msg: + except OSError as msg: print("can't write", fn, ":", msg) else: result = marshal.load(fp) diff --git a/Lib/test/ssl_servers.py b/Lib/test/ssl_servers.py index 8686153a17..759b3f487e 100644 --- a/Lib/test/ssl_servers.py +++ b/Lib/test/ssl_servers.py @@ -35,7 +35,7 @@ class HTTPSServer(_HTTPServer): try: sock, addr = self.socket.accept() sslconn = self.context.wrap_socket(sock, server_side=True) - except socket.error as e: + except OSError as e: # socket errors are silenced by the caller, print them here if support.verbose: sys.stderr.write("Got an error:\n%s\n" % e) @@ -147,9 +147,11 @@ class HTTPSServerThread(threading.Thread): self.server.shutdown() -def make_https_server(case, certfile=CERTFILE, host=HOST, handler_class=None): - # we assume the certfile contains both private key and certificate - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) +def make_https_server(case, *, context=None, certfile=CERTFILE, + host=HOST, handler_class=None): + if context is None: + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + # We assume the certfile contains both private key and certificate context.load_cert_chain(certfile) server = HTTPSServerThread(context, host, handler_class) flag = threading.Event() diff --git a/Lib/test/subprocessdata/fd_status.py b/Lib/test/subprocessdata/fd_status.py index 1f61e13a34..d12bd95abe 100644 --- a/Lib/test/subprocessdata/fd_status.py +++ b/Lib/test/subprocessdata/fd_status.py @@ -1,17 +1,27 @@ """When called as a script, print a comma-separated list of the open -file descriptors on stdout.""" +file descriptors on stdout. + +Usage: +fd_stats.py: check all file descriptors +fd_status.py fd1 fd2 ...: check only specified file descriptors +""" import errno import os - -try: - _MAXFD = os.sysconf("SC_OPEN_MAX") -except: - _MAXFD = 256 +import stat +import sys if __name__ == "__main__": fds = [] - for fd in range(0, _MAXFD): + if len(sys.argv) == 1: + try: + _MAXFD = os.sysconf("SC_OPEN_MAX") + except: + _MAXFD = 256 + test_fds = range(0, _MAXFD) + else: + test_fds = map(int, sys.argv[1:]) + for fd in test_fds: try: st = os.fstat(fd) except OSError as e: @@ -19,6 +29,6 @@ if __name__ == "__main__": continue raise # Ignore Solaris door files - if st.st_mode & 0xF000 != 0xd000: + if not stat.S_ISDOOR(st.st_mode): fds.append(fd) print(','.join(map(str, fds))) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index dbd7846737..d4f756681a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -15,10 +15,10 @@ import shutil import warnings import unittest import importlib +import importlib.util import collections.abc import re import subprocess -import imp import time import sysconfig import fnmatch @@ -57,6 +57,11 @@ try: except ImportError: lzma = None +try: + import resource +except ImportError: + resource = None + __all__ = [ "Error", "TestFailed", "ResourceDenied", "import_module", "verbose", "use_resources", "max_memuse", "record_original_stdout", @@ -76,7 +81,7 @@ __all__ = [ "TestHandler", "Matcher", "can_symlink", "skip_unless_symlink", "skip_unless_xattr", "import_fresh_module", "requires_zlib", "PIPE_MAX_SIZE", "failfast", "anticipate_failure", "run_with_tz", - "requires_gzip", "requires_bz2", "requires_lzma", "suppress_crash_popup", + "requires_gzip", "requires_bz2", "requires_lzma", "SuppressCrashReport" ] class Error(Exception): @@ -98,7 +103,8 @@ def _ignore_deprecated_imports(ignore=True): """Context manager to suppress package and module deprecation warnings when importing them. - If ignore is False, this context manager has no effect.""" + If ignore is False, this context manager has no effect. + """ if ignore: with warnings.catch_warnings(): warnings.filterwarnings("ignore", ".+ (module|package)", @@ -108,16 +114,21 @@ def _ignore_deprecated_imports(ignore=True): yield -def import_module(name, deprecated=False): +def import_module(name, deprecated=False, *, required_on=()): """Import and return the module to be tested, raising SkipTest if it is not available. If deprecated is True, any module or package deprecation messages - will be suppressed.""" + will be suppressed. If a module is required on a platform but optional for + others, set required_on to an iterable of platform prefixes which will be + compared against sys.platform. + """ with _ignore_deprecated_imports(deprecated): try: return importlib.import_module(name) except ImportError as msg: + if sys.platform.startswith(tuple(required_on)): + raise raise unittest.SkipTest(str(msg)) @@ -303,25 +314,20 @@ else: def unlink(filename): try: _unlink(filename) - except OSError as error: - # The filename need not exist. - if error.errno not in (errno.ENOENT, errno.ENOTDIR): - raise + except (FileNotFoundError, NotADirectoryError): + pass def rmdir(dirname): try: _rmdir(dirname) - except OSError as error: - # The directory need not exist. - if error.errno != errno.ENOENT: - raise + except FileNotFoundError: + pass def rmtree(path): try: _rmtree(path) - except OSError as error: - if error.errno != errno.ENOENT: - raise + except FileNotFoundError: + pass def make_legacy_pyc(source): """Move a PEP 3147 pyc/pyo file to its legacy pyc/pyo location. @@ -333,7 +339,7 @@ def make_legacy_pyc(source): does not need to exist, however the PEP 3147 pyc file must exist. :return: The file system path to the legacy pyc file. """ - pyc_file = imp.cache_from_source(source) + pyc_file = importlib.util.cache_from_source(source) up_one = os.path.dirname(os.path.abspath(source)) legacy_pyc = os.path.join(up_one, source + ('c' if __debug__ else 'o')) os.rename(pyc_file, legacy_pyc) @@ -352,8 +358,8 @@ def forget(modname): # combinations of PEP 3147 and legacy pyc and pyo files. unlink(source + 'c') unlink(source + 'o') - unlink(imp.cache_from_source(source, debug_override=True)) - unlink(imp.cache_from_source(source, debug_override=False)) + unlink(importlib.util.cache_from_source(source, debug_override=True)) + unlink(importlib.util.cache_from_source(source, debug_override=False)) # On some platforms, should not run gui test even if it is allowed # in `use_resources'. @@ -514,7 +520,7 @@ def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): the SO_REUSEADDR socket option having different semantics on Windows versus Unix/Linux. On Unix, you can't have two AF_INET SOCK_STREAM sockets bind, listen and then accept connections on identical host/ports. An EADDRINUSE - socket.error will be raised at some point (depending on the platform and + OSError will be raised at some point (depending on the platform and the order bind and listen were called on each socket). However, on Windows, if SO_REUSEADDR is set on the sockets, no EADDRINUSE @@ -586,9 +592,9 @@ def _is_ipv6_enabled(): sock = None try: sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - sock.bind(('::1', 0)) + sock.bind((HOSTv6, 0)) return True - except (socket.error, socket.gaierror): + except OSError: pass finally: if sock: @@ -1175,9 +1181,9 @@ class TransientResource(object): # Context managers that raise ResourceDenied when various issues # with the Internet connection manifest themselves as exceptions. # XXX deprecate these and use transient_internet() instead -time_out = TransientResource(IOError, errno=errno.ETIMEDOUT) -socket_peer_reset = TransientResource(socket.error, errno=errno.ECONNRESET) -ioerror_peer_reset = TransientResource(IOError, errno=errno.ECONNRESET) +time_out = TransientResource(OSError, errno=errno.ETIMEDOUT) +socket_peer_reset = TransientResource(OSError, errno=errno.ECONNRESET) +ioerror_peer_reset = TransientResource(OSError, errno=errno.ECONNRESET) @contextlib.contextmanager @@ -1223,17 +1229,17 @@ def transient_internet(resource_name, *, timeout=30.0, errnos=()): if timeout is not None: socket.setdefaulttimeout(timeout) yield - except IOError as err: + except OSError as err: # urllib can wrap original socket errors multiple times (!), we must # unwrap to get at the original error. while True: a = err.args - if len(a) >= 1 and isinstance(a[0], IOError): + if len(a) >= 1 and isinstance(a[0], OSError): err = a[0] # The error can also be wrapped as args[1]: # except socket.error as msg: - # raise IOError('socket error', msg).with_traceback(sys.exc_info()[2]) - elif len(a) >= 2 and isinstance(a[1], IOError): + # raise OSError('socket error', msg).with_traceback(sys.exc_info()[2]) + elif len(a) >= 2 and isinstance(a[1], OSError): err = a[1] else: break @@ -1768,12 +1774,12 @@ def threading_setup(): def threading_cleanup(*original_values): if not _thread: return - _MAX_COUNT = 10 + _MAX_COUNT = 100 for count in range(_MAX_COUNT): values = _thread._count(), threading._dangling if values == original_values: break - time.sleep(0.1) + time.sleep(0.01) gc_collect() # XXX print a warning in case of failure? @@ -1875,7 +1881,7 @@ def strip_python_stderr(stderr): This will typically be run on the result of the communicate() method of a subprocess.Popen object. """ - stderr = re.sub(br"\[\d+ refs\]\r?\n?", b"", stderr).strip() + stderr = re.sub(br"\[\d+ refs, \d+ blocks\]\r?\n?", b"", stderr).strip() return stderr def args_from_interpreter_flags(): @@ -2006,27 +2012,67 @@ def skip_unless_xattr(test): return test if ok else unittest.skip(msg)(test) -if sys.platform.startswith('win'): - @contextlib.contextmanager - def suppress_crash_popup(): - """Disable Windows Error Reporting dialogs using SetErrorMode.""" - # see http://msdn.microsoft.com/en-us/library/windows/desktop/ms680621%28v=vs.85%29.aspx - # GetErrorMode is not available on Windows XP and Windows Server 2003, - # but SetErrorMode returns the previous value, so we can use that - import ctypes - k32 = ctypes.windll.kernel32 - SEM_NOGPFAULTERRORBOX = 0x02 - old_error_mode = k32.SetErrorMode(SEM_NOGPFAULTERRORBOX) - k32.SetErrorMode(old_error_mode | SEM_NOGPFAULTERRORBOX) - try: - yield - finally: - k32.SetErrorMode(old_error_mode) -else: - # this is a no-op for other platforms - @contextlib.contextmanager - def suppress_crash_popup(): - yield +class SuppressCrashReport: + """Try to prevent a crash report from popping up. + + On Windows, don't display the Windows Error Reporting dialog. On UNIX, + disable the creation of coredump file. + """ + old_value = None + + def __enter__(self): + """On Windows, disable Windows Error Reporting dialogs using + SetErrorMode. + + On UNIX, try to save the previous core file size limit, then set + soft limit to 0. + """ + if sys.platform.startswith('win'): + # see http://msdn.microsoft.com/en-us/library/windows/desktop/ms680621.aspx + # GetErrorMode is not available on Windows XP and Windows Server 2003, + # but SetErrorMode returns the previous value, so we can use that + import ctypes + self._k32 = ctypes.windll.kernel32 + SEM_NOGPFAULTERRORBOX = 0x02 + self.old_value = self._k32.SetErrorMode(SEM_NOGPFAULTERRORBOX) + self._k32.SetErrorMode(self.old_value | SEM_NOGPFAULTERRORBOX) + else: + if resource is not None: + try: + self.old_value = resource.getrlimit(resource.RLIMIT_CORE) + resource.setrlimit(resource.RLIMIT_CORE, + (0, self.old_value[1])) + except (ValueError, OSError): + pass + if sys.platform == 'darwin': + # Check if the 'Crash Reporter' on OSX was configured + # in 'Developer' mode and warn that it will get triggered + # when it is. + # + # This assumes that this context manager is used in tests + # that might trigger the next manager. + value = subprocess.Popen(['/usr/bin/defaults', 'read', + 'com.apple.CrashReporter', 'DialogType'], + stdout=subprocess.PIPE).communicate()[0] + if value.strip() == b'developer': + print("this test triggers the Crash Reporter, " + "that is intentional", end='', flush=True) + + return self + + def __exit__(self, *ignore_exc): + """Restore Windows ErrorMode or core file behavior to initial value.""" + if self.old_value is None: + return + + if sys.platform.startswith('win'): + self._k32.SetErrorMode(self.old_value) + else: + if resource is not None: + try: + resource.setrlimit(resource.RLIMIT_CORE, self.old_value) + except (ValueError, OSError): + pass def patch(test_instance, object_to_patch, attr_name, new_value): diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index 608ec01f14..8cc285f70a 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -29,17 +29,20 @@ class AllTest(unittest.TestCase): if not hasattr(sys.modules[modname], "__all__"): raise NoAll(modname) names = {} - try: - exec("from %s import *" % modname, names) - except Exception as e: - # Include the module name in the exception string - self.fail("__all__ failure in {}: {}: {}".format( - modname, e.__class__.__name__, e)) - if "__builtins__" in names: - del names["__builtins__"] - keys = set(names) - all = set(sys.modules[modname].__all__) - self.assertEqual(keys, all) + with self.subTest(module=modname): + try: + exec("from %s import *" % modname, names) + except Exception as e: + # Include the module name in the exception string + self.fail("__all__ failure in {}: {}: {}".format( + modname, e.__class__.__name__, e)) + if "__builtins__" in names: + del names["__builtins__"] + keys = set(names) + all_list = sys.modules[modname].__all__ + all_set = set(all_list) + self.assertCountEqual(all_set, all_list, "in module {}".format(modname)) + self.assertEqual(keys, all_set, "in module {}".format(modname)) def walk_modules(self, basedir, modpath): for fn in sorted(os.listdir(basedir)): @@ -110,8 +113,5 @@ class AllTest(unittest.TestCase): print('Following modules failed to be imported:', failed_imports) -def test_main(): - support.run_unittest(AllTest) - if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index d4d7556518..93f9dae215 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -68,6 +68,19 @@ class TestLegacyAPI(unittest.TestCase): class TestABC(unittest.TestCase): + def test_ABC_helper(self): + # create an ABC using the helper class and perform basic checks + class C(abc.ABC): + @classmethod + @abc.abstractmethod + def foo(cls): return cls.__name__ + self.assertEqual(type(C), abc.ABCMeta) + self.assertRaises(TypeError, C) + class D(C): + @classmethod + def foo(cls): return super().foo() + self.assertEqual(D.foo(), 'D') + def test_abstractmethod_basics(self): @abc.abstractmethod def foo(self): pass @@ -288,7 +301,10 @@ class TestABC(unittest.TestCase): b = B() self.assertFalse(isinstance(b, A)) self.assertFalse(isinstance(b, (A,))) + token_old = abc.get_cache_token() A.register(B) + token_new = abc.get_cache_token() + self.assertNotEqual(token_old, token_new) self.assertTrue(isinstance(b, A)) self.assertTrue(isinstance(b, (A,))) diff --git a/Lib/test/test_aifc.py b/Lib/test/test_aifc.py index b77354bb07..3e9a1d13c2 100644 --- a/Lib/test/test_aifc.py +++ b/Lib/test/test_aifc.py @@ -166,6 +166,21 @@ class AifcMiscTest(audiotests.AudioTests, unittest.TestCase): #This file contains chunk types aifc doesn't recognize. self.f = aifc.open(findfile('Sine-1000Hz-300ms.aif')) + def test_params_added(self): + f = self.f = aifc.open(TESTFN, 'wb') + f.aiff() + f.setparams((1, 1, 1, 1, b'NONE', b'')) + f.close() + + f = self.f = aifc.open(TESTFN, 'rb') + params = f.getparams() + self.assertEqual(params.nchannels, f.getnchannels()) + self.assertEqual(params.sampwidth, f.getsampwidth()) + self.assertEqual(params.framerate, f.getframerate()) + self.assertEqual(params.nframes, f.getnframes()) + self.assertEqual(params.comptype, f.getcomptype()) + self.assertEqual(params.compname, f.getcompname()) + def test_write_header_comptype_sampwidth(self): for comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'): fout = aifc.open(io.BytesIO(), 'wb') @@ -369,12 +384,14 @@ class AIFCLowLevelTest(unittest.TestCase): def test_write_aiff_by_extension(self): sampwidth = 2 - fout = self.fout = aifc.open(TESTFN + '.aiff', 'wb') + filename = TESTFN + '.aiff' + fout = self.fout = aifc.open(filename, 'wb') + self.addCleanup(unlink, filename) fout.setparams((1, sampwidth, 1, 1, b'ULAW', b'')) frames = b'\x00' * fout.getnchannels() * sampwidth fout.writeframes(frames) fout.close() - f = self.f = aifc.open(TESTFN + '.aiff', 'rb') + f = self.f = aifc.open(filename, 'rb') self.assertEqual(f.getcomptype(), b'NONE') f.close() diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index c06c940bf2..c10c5909bf 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -14,6 +14,7 @@ import argparse from io import StringIO from test import support +from unittest import mock class StdIOBuffer(StringIO): pass @@ -1421,6 +1422,19 @@ class TestFileTypeRepr(TestCase): type = argparse.FileType('wb', 1) self.assertEqual("FileType('wb', 1)", repr(type)) + def test_r_latin(self): + type = argparse.FileType('r', encoding='latin_1') + self.assertEqual("FileType('r', encoding='latin_1')", repr(type)) + + def test_w_big5_ignore(self): + type = argparse.FileType('w', encoding='big5', errors='ignore') + self.assertEqual("FileType('w', encoding='big5', errors='ignore')", + repr(type)) + + def test_r_1_replace(self): + type = argparse.FileType('r', 1, errors='replace') + self.assertEqual("FileType('r', 1, errors='replace')", repr(type)) + class RFile(object): seen = {} @@ -1557,6 +1571,24 @@ class TestFileTypeWB(TempDirMixin, ParserTestCase): ] +class TestFileTypeOpenArgs(TestCase): + """Test that open (the builtin) is correctly called""" + + def test_open_args(self): + FT = argparse.FileType + cases = [ + (FT('rb'), ('rb', -1, None, None)), + (FT('w', 1), ('w', 1, None, None)), + (FT('w', errors='replace'), ('w', -1, None, 'replace')), + (FT('wb', encoding='big5'), ('wb', -1, 'big5', None)), + (FT('w', 0, 'l1', 'strict'), ('w', 0, 'l1', 'strict')), + ] + with mock.patch('builtins.open') as m: + for type, args in cases: + type('foo') + m.assert_called_with('foo', *args) + + class TestTypeCallable(ParserTestCase): """Test some callables as option/argument types""" @@ -4327,7 +4359,7 @@ class TestOptionalsHelpVersionActions(TestCase): def test_version_format(self): parser = ErrorRaisingArgumentParser(prog='PPP') parser.add_argument('-v', '--version', action='version', version='%(prog)s 3.5') - msg = self._get_error(parser.parse_args, ['-v']).stderr + msg = self._get_error(parser.parse_args, ['-v']).stdout self.assertEqual('PPP 3.5\n', msg) def test_version_no_help(self): @@ -4340,7 +4372,7 @@ class TestOptionalsHelpVersionActions(TestCase): def test_version_action(self): parser = ErrorRaisingArgumentParser(prog='XXX') parser.add_argument('-V', action='version', version='%(prog)s 3.7') - msg = self._get_error(parser.parse_args, ['-V']).stderr + msg = self._get_error(parser.parse_args, ['-V']).stdout self.assertEqual('XXX 3.7\n', msg) def test_no_help(self): diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index f21b69f367..a44ee07ce3 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -353,12 +353,12 @@ class BaseTest: support.unlink(support.TESTFN) def test_fromfile_ioerror(self): - # Issue #5395: Check if fromfile raises a proper IOError + # Issue #5395: Check if fromfile raises a proper OSError # instead of EOFError. a = array.array(self.typecode) f = open(support.TESTFN, 'wb') try: - self.assertRaises(IOError, a.fromfile, f, len(self.example)) + self.assertRaises(OSError, a.fromfile, f, len(self.example)) finally: f.close() support.unlink(support.TESTFN) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 63528885e9..e69422a292 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -180,20 +180,36 @@ eval_tests = [ class AST_Tests(unittest.TestCase): - def _assertTrueorder(self, ast_node, parent_pos): + def _assertTrueorder(self, ast_node, parent_pos, reverse_check = False): + def should_reverse_check(parent, child): + # In some situations, the children of nodes occur before + # their parents, for example in a.b.c, a occurs before b + # but a is a child of b. + if isinstance(parent, ast.Call): + if parent.func == child: + return True + if isinstance(parent, (ast.Attribute, ast.Subscript)): + return True + return False + if not isinstance(ast_node, ast.AST) or ast_node._fields is None: return if isinstance(ast_node, (ast.expr, ast.stmt, ast.excepthandler)): node_pos = (ast_node.lineno, ast_node.col_offset) - self.assertTrue(node_pos >= parent_pos) + if reverse_check: + self.assertTrue(node_pos <= parent_pos) + else: + self.assertTrue(node_pos >= parent_pos) parent_pos = (ast_node.lineno, ast_node.col_offset) for name in ast_node._fields: value = getattr(ast_node, name) if isinstance(value, list): for child in value: - self._assertTrueorder(child, parent_pos) + self._assertTrueorder(child, parent_pos, + should_reverse_check(ast_node, child)) elif value is not None: - self._assertTrueorder(value, parent_pos) + self._assertTrueorder(value, parent_pos, + should_reverse_check(ast_node, value)) def test_AST_objects(self): x = ast.AST() @@ -262,14 +278,14 @@ class AST_Tests(unittest.TestCase): def test_arguments(self): x = ast.arguments() - self.assertEqual(x._fields, ('args', 'vararg', 'varargannotation', - 'kwonlyargs', 'kwarg', 'kwargannotation', - 'defaults', 'kw_defaults')) + self.assertEqual(x._fields, ('args', 'vararg', + 'kwonlyargs', 'kw_defaults', + 'kwarg', 'defaults')) with self.assertRaises(AttributeError): x.vararg - x = ast.arguments(*range(1, 9)) + x = ast.arguments(*range(1, 7)) self.assertEqual(x.vararg, 2) def test_field_attr_writable(self): @@ -439,7 +455,7 @@ class ASTHelpers_Test(unittest.TestCase): "lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), " "lineno=1, col_offset=5), Str(s='and cheese', lineno=1, " "col_offset=11)], keywords=[], starargs=None, kwargs=None, " - "lineno=1, col_offset=0), lineno=1, col_offset=0)])" + "lineno=1, col_offset=4), lineno=1, col_offset=0)])" ) def test_copy_location(self): @@ -460,7 +476,7 @@ class ASTHelpers_Test(unittest.TestCase): "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " "lineno=1, col_offset=0), args=[Str(s='spam', lineno=1, " "col_offset=6)], keywords=[], starargs=None, kwargs=None, " - "lineno=1, col_offset=0), lineno=1, col_offset=0), " + "lineno=1, col_offset=5), lineno=1, col_offset=0), " "Expr(value=Call(func=Name(id='spam', ctx=Load(), lineno=1, " "col_offset=0), args=[Str(s='eggs', lineno=1, col_offset=0)], " "keywords=[], starargs=None, kwargs=None, lineno=1, " @@ -560,8 +576,8 @@ class ASTValidatorTests(unittest.TestCase): self.mod(m, "must have Load context", "eval") def _check_arguments(self, fac, check): - def arguments(args=None, vararg=None, varargannotation=None, - kwonlyargs=None, kwarg=None, kwargannotation=None, + def arguments(args=None, vararg=None, + kwonlyargs=None, kwarg=None, defaults=None, kw_defaults=None): if args is None: args = [] @@ -571,20 +587,12 @@ class ASTValidatorTests(unittest.TestCase): defaults = [] if kw_defaults is None: kw_defaults = [] - args = ast.arguments(args, vararg, varargannotation, kwonlyargs, - kwarg, kwargannotation, defaults, kw_defaults) + args = ast.arguments(args, vararg, kwonlyargs, kw_defaults, + kwarg, defaults) return fac(args) args = [ast.arg("x", ast.Name("x", ast.Store()))] check(arguments(args=args), "must have Load context") - check(arguments(varargannotation=ast.Num(3)), - "varargannotation but no vararg") - check(arguments(varargannotation=ast.Name("x", ast.Store()), vararg="x"), - "must have Load context") check(arguments(kwonlyargs=args), "must have Load context") - check(arguments(kwargannotation=ast.Num(42)), - "kwargannotation but no kwarg") - check(arguments(kwargannotation=ast.Name("x", ast.Store()), - kwarg="x"), "must have Load context") check(arguments(defaults=[ast.Num(3)]), "more positional defaults than args") check(arguments(kw_defaults=[ast.Num(4)]), @@ -599,7 +607,7 @@ class ASTValidatorTests(unittest.TestCase): "must have Load context") def test_funcdef(self): - a = ast.arguments([], None, None, [], None, None, [], []) + a = ast.arguments([], None, [], [], None, []) f = ast.FunctionDef("x", a, [], [], None) self.stmt(f, "empty body on FunctionDef") f = ast.FunctionDef("x", a, [ast.Pass()], [ast.Name("x", ast.Store())], @@ -770,7 +778,7 @@ class ASTValidatorTests(unittest.TestCase): self.expr(u, "must have Load context") def test_lambda(self): - a = ast.arguments([], None, None, [], None, None, [], []) + a = ast.arguments([], None, [], [], None, []) self.expr(ast.Lambda(a, ast.Name("x", ast.Store())), "must have Load context") def fac(args): @@ -928,6 +936,9 @@ class ASTValidatorTests(unittest.TestCase): def test_tuple(self): self._sequence(ast.Tuple) + def test_nameconstant(self): + self.expr(ast.NameConstant(4), "singleton must be True, False, or None") + def test_stdlib_validates(self): stdlib = os.path.dirname(ast.__file__) tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")] @@ -936,13 +947,10 @@ class ASTValidatorTests(unittest.TestCase): fn = os.path.join(stdlib, module) with open(fn, "r", encoding="utf-8") as fp: source = fp.read() - mod = ast.parse(source) + mod = ast.parse(source, fn) compile(mod, fn, "exec") -def test_main(): - support.run_unittest(AST_Tests, ASTHelpers_Test, ASTValidatorTests) - def main(): if __name__ != '__main__': return @@ -955,20 +963,20 @@ def main(): print("]") print("main()") raise SystemExit - test_main() + unittest.main() #### EVERYTHING BELOW IS GENERATED ##### exec_results = [ -('Module', [('Expr', (1, 0), ('Name', (1, 0), 'None', ('Load',)))]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Pass', (1, 9))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None)], None, None, [], None, None, [], []), [('Pass', (1, 10))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None)], None, None, [], None, None, [('Num', (1, 8), 0)], []), [('Pass', (1, 12))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], 'args', None, [], None, None, [], []), [('Pass', (1, 14))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], 'kwargs', None, [], []), [('Pass', (1, 17))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None), ('arg', 'b', None), ('arg', 'c', None), ('arg', 'd', None), ('arg', 'e', None)], 'args', None, [], 'kwargs', None, [('Num', (1, 11), 1), ('Name', (1, 16), 'None', ('Load',)), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])], []), [('Pass', (1, 52))], [], None)]), +('Module', [('Expr', (1, 0), ('NameConstant', (1, 0), None))]), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None)]), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, []), [('Pass', (1, 10))], [], None)]), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, [('Num', (1, 8), 0)]), [('Pass', (1, 12))], [], None)]), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None), [], [], None, []), [('Pass', (1, 14))], [], None)]), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None), []), [('Pass', (1, 17))], [], None)]), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None), ('arg', (1, 9), 'b', None), ('arg', (1, 14), 'c', None), ('arg', (1, 22), 'd', None), ('arg', (1, 28), 'e', None)], ('arg', (1, 35), 'args', None), [], [], ('arg', (1, 43), 'kwargs', None), [('Num', (1, 11), 1), ('NameConstant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Pass', (1, 52))], [], None)]), ('Module', [('ClassDef', (1, 0), 'C', [], [], None, None, [('Pass', (1, 8))], [])]), ('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], None, None, [('Pass', (1, 17))], [])]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]), ('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]), ('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]), ('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Num', (1, 5), 1))]), @@ -977,7 +985,7 @@ exec_results = [ ('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]), ('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]), ('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]), -('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], [], None, None), None)]), +('Module', [('Raise', (1, 0), ('Call', (1, 15), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], [], None, None), None)]), ('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])]), ('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])]), ('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]), @@ -1002,29 +1010,29 @@ single_results = [ ('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]), ] eval_results = [ -('Expression', ('Name', (1, 0), 'None', ('Load',))), +('Expression', ('NameConstant', (1, 0), None)), ('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])), ('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))), ('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))), -('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, [], None, None, [], []), ('Name', (1, 7), 'None', ('Load',)))), +('Expression', ('Lambda', (1, 0), ('arguments', [], None, [], [], None, []), ('NameConstant', (1, 7), None))), ('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])), ('Expression', ('Dict', (1, 0), [], [])), -('Expression', ('Set', (1, 0), [('Name', (1, 1), 'None', ('Load',))])), +('Expression', ('Set', (1, 0), [('NameConstant', (1, 1), None)])), ('Expression', ('Dict', (1, 0), [('Num', (2, 6), 1)], [('Num', (4, 10), 2)])), ('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])), ('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])), ('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])), -('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))), +('Expression', ('Call', (1, 1), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))), ('Expression', ('Num', (1, 0), 10)), ('Expression', ('Str', (1, 0), 'string')), -('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))), -('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))), +('Expression', ('Attribute', (1, 2), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))), +('Expression', ('Subscript', (1, 2), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))), ('Expression', ('Name', (1, 0), 'v', ('Load',))), ('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))), ('Expression', ('List', (1, 0), [], ('Load',))), ('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))), ('Expression', ('Tuple', (1, 1), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))), ('Expression', ('Tuple', (1, 0), [], ('Load',))), -('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)), +('Expression', ('Call', (1, 7), ('Attribute', (1, 6), ('Attribute', (1, 4), ('Attribute', (1, 2), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 12), ('Attribute', (1, 10), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)), ] main() diff --git a/Lib/test/test_asynchat.py b/Lib/test/test_asynchat.py index c79fe6f613..f93a52d8c9 100644 --- a/Lib/test/test_asynchat.py +++ b/Lib/test/test_asynchat.py @@ -15,6 +15,7 @@ except ImportError: HOST = support.HOST SERVER_QUIT = b'QUIT\n' +TIMEOUT = 3.0 if threading: class echo_server(threading.Thread): @@ -123,7 +124,9 @@ class TestAsynchat(unittest.TestCase): c.push(b"I'm not dead yet!" + term) c.push(SERVER_QUIT) asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - s.join() + s.join(timeout=TIMEOUT) + if s.is_alive(): + self.fail("join() timed out") self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) @@ -154,7 +157,9 @@ class TestAsynchat(unittest.TestCase): c.push(data) c.push(SERVER_QUIT) asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - s.join() + s.join(timeout=TIMEOUT) + if s.is_alive(): + self.fail("join() timed out") self.assertEqual(c.contents, [data[:termlen]]) @@ -174,7 +179,9 @@ class TestAsynchat(unittest.TestCase): c.push(data) c.push(SERVER_QUIT) asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - s.join() + s.join(timeout=TIMEOUT) + if s.is_alive(): + self.fail("join() timed out") self.assertEqual(c.contents, []) self.assertEqual(c.buffer, data) @@ -186,7 +193,9 @@ class TestAsynchat(unittest.TestCase): p = asynchat.simple_producer(data+SERVER_QUIT, buffer_size=8) c.push_with_producer(p) asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - s.join() + s.join(timeout=TIMEOUT) + if s.is_alive(): + self.fail("join() timed out") self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) @@ -196,7 +205,9 @@ class TestAsynchat(unittest.TestCase): data = b"hello world\nI'm not dead yet!\n" c.push_with_producer(data+SERVER_QUIT) asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - s.join() + s.join(timeout=TIMEOUT) + if s.is_alive(): + self.fail("join() timed out") self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) @@ -207,7 +218,9 @@ class TestAsynchat(unittest.TestCase): c.push(b"hello world\n\nI'm not dead yet!\n") c.push(SERVER_QUIT) asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - s.join() + s.join(timeout=TIMEOUT) + if s.is_alive(): + self.fail("join() timed out") self.assertEqual(c.contents, [b"hello world", b"", b"I'm not dead yet!"]) @@ -226,7 +239,9 @@ class TestAsynchat(unittest.TestCase): # where the server echoes all of its data before we can check that it # got any down below. s.start_resend_event.set() - s.join() + s.join(timeout=TIMEOUT) + if s.is_alive(): + self.fail("join() timed out") self.assertEqual(c.contents, []) # the server might have been able to send a byte or two back, but this @@ -268,9 +283,5 @@ class TestFifo(unittest.TestCase): self.assertEqual(f.pop(), (0, None)) -def test_main(verbose=None): - support.run_unittest(TestAsynchat, TestAsynchat_WithPoll, - TestHelperFunctions, TestFifo) - if __name__ == "__main__": - test_main(verbose=True) + unittest.main() diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index 5d0632ef26..084d247295 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -19,6 +19,7 @@ try: except ImportError: threading = None +TIMEOUT = 3 HAS_UNIX_SOCKETS = hasattr(socket, 'AF_UNIX') class dummysocket: @@ -395,7 +396,10 @@ class DispatcherWithSendTests(unittest.TestCase): self.assertEqual(cap.getvalue(), data*2) finally: - t.join() + t.join(timeout=TIMEOUT) + if t.is_alive(): + self.fail("join() timed out") + class DispatcherWithSendTests_UsePoll(DispatcherWithSendTests): @@ -740,7 +744,12 @@ class BaseTestAPI: s.create_socket(self.family) self.assertEqual(s.socket.family, self.family) SOCK_NONBLOCK = getattr(socket, 'SOCK_NONBLOCK', 0) - self.assertEqual(s.socket.type, socket.SOCK_STREAM | SOCK_NONBLOCK) + sock_type = socket.SOCK_STREAM | SOCK_NONBLOCK + if hasattr(socket, 'SOCK_CLOEXEC'): + self.assertIn(s.socket.type, + (sock_type | socket.SOCK_CLOEXEC, sock_type)) + else: + self.assertEqual(s.socket.type, sock_type) def test_bind(self): if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: @@ -754,7 +763,7 @@ class BaseTestAPI: s2 = asyncore.dispatcher() s2.create_socket(self.family) # EADDRINUSE indicates the socket was correctly bound - self.assertRaises(socket.error, s2.bind, (self.addr[0], port)) + self.assertRaises(OSError, s2.bind, (self.addr[0], port)) def test_set_reuse_addr(self): if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: @@ -762,7 +771,7 @@ class BaseTestAPI: sock = socket.socket(self.family) try: sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - except socket.error: + except OSError: unittest.skip("SO_REUSEADDR not supported on this platform") else: # if SO_REUSEADDR succeeded for sock we expect asyncore @@ -787,7 +796,11 @@ class BaseTestAPI: t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, count=500)) t.start() - self.addCleanup(t.join) + def cleanup(): + t.join(timeout=TIMEOUT) + if t.is_alive(): + self.fail("join() timed out") + self.addCleanup(cleanup) s = socket.socket(self.family, socket.SOCK_STREAM) s.settimeout(.2) @@ -795,7 +808,7 @@ class BaseTestAPI: struct.pack('ii', 1, 0)) try: s.connect(server.address) - except socket.error: + except OSError: pass finally: s.close() diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index 30c3b4a07b..b641015b70 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -2,6 +2,7 @@ import sys import unittest import io import atexit +import _testcapi from test import support ### helpers @@ -23,7 +24,9 @@ def raise1(): def raise2(): raise SystemError -class TestCase(unittest.TestCase): + +class GeneralTest(unittest.TestCase): + def setUp(self): self.save_stdout = sys.stdout self.save_stderr = sys.stderr @@ -141,8 +144,43 @@ class TestCase(unittest.TestCase): self.assertEqual(l, [5]) +class SubinterpreterTest(unittest.TestCase): + + def test_callbacks_leak(self): + # This test shows a leak in refleak mode if atexit doesn't + # take care to free callbacks in its per-subinterpreter module + # state. + n = atexit._ncallbacks() + code = r"""if 1: + import atexit + def f(): + pass + atexit.register(f) + del atexit + """ + ret = _testcapi.run_in_subinterp(code) + self.assertEqual(ret, 0) + self.assertEqual(atexit._ncallbacks(), n) + + def test_callbacks_leak_refcycle(self): + # Similar to the above, but with a refcycle through the atexit + # module. + n = atexit._ncallbacks() + code = r"""if 1: + import atexit + def f(): + pass + atexit.register(f) + atexit.__atexit = atexit + """ + ret = _testcapi.run_in_subinterp(code) + self.assertEqual(ret, 0) + self.assertEqual(atexit._ncallbacks(), n) + + def test_main(): - support.run_unittest(TestCase) + support.run_unittest(__name__) + if __name__ == "__main__": test_main() diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 13695de67e..54f392d4d6 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -5,10 +5,21 @@ import binascii import os import sys import subprocess - +import struct +from array import array class LegacyBase64TestCase(unittest.TestCase): + + # Legacy API is not as permissive as the modern API + def check_type_errors(self, f): + self.assertRaises(TypeError, f, "") + self.assertRaises(TypeError, f, []) + multidimensional = memoryview(b"1234").cast('B', (2, 2)) + self.assertRaises(TypeError, f, multidimensional) + int_data = memoryview(b"1234").cast('I') + self.assertRaises(TypeError, f, int_data) + def test_encodebytes(self): eq = self.assertEqual eq(base64.encodebytes(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=\n") @@ -24,7 +35,9 @@ class LegacyBase64TestCase(unittest.TestCase): b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==\n") # Non-bytes eq(base64.encodebytes(bytearray(b'abc')), b'YWJj\n') - self.assertRaises(TypeError, base64.encodebytes, "") + eq(base64.encodebytes(memoryview(b'abc')), b'YWJj\n') + eq(base64.encodebytes(array('B', b'abc')), b'YWJj\n') + self.check_type_errors(base64.encodebytes) def test_decodebytes(self): eq = self.assertEqual @@ -41,7 +54,9 @@ class LegacyBase64TestCase(unittest.TestCase): eq(base64.decodebytes(b''), b'') # Non-bytes eq(base64.decodebytes(bytearray(b'YWJj\n')), b'abc') - self.assertRaises(TypeError, base64.decodebytes, "") + eq(base64.decodebytes(memoryview(b'YWJj\n')), b'abc') + eq(base64.decodebytes(array('B', b'YWJj\n')), b'abc') + self.check_type_errors(base64.decodebytes) def test_encode(self): eq = self.assertEqual @@ -73,6 +88,38 @@ class LegacyBase64TestCase(unittest.TestCase): class BaseXYTestCase(unittest.TestCase): + + # Modern API completely ignores exported dimension and format data and + # treats any buffer as a stream of bytes + def check_encode_type_errors(self, f): + self.assertRaises(TypeError, f, "") + self.assertRaises(TypeError, f, []) + + def check_decode_type_errors(self, f): + self.assertRaises(TypeError, f, []) + + def check_other_types(self, f, bytes_data, expected): + eq = self.assertEqual + eq(f(bytearray(bytes_data)), expected) + eq(f(memoryview(bytes_data)), expected) + eq(f(array('B', bytes_data)), expected) + self.check_nonbyte_element_format(base64.b64encode, bytes_data) + self.check_multidimensional(base64.b64encode, bytes_data) + + def check_multidimensional(self, f, data): + padding = b"\x00" if len(data) % 2 else b"" + bytes_data = data + padding # Make sure cast works + shape = (len(bytes_data) // 2, 2) + multidimensional = memoryview(bytes_data).cast('B', shape) + self.assertEqual(f(multidimensional), f(bytes_data)) + + def check_nonbyte_element_format(self, f, data): + padding = b"\x00" * ((4 - len(data)) % 4) + bytes_data = data + padding # Make sure cast works + int_data = memoryview(bytes_data).cast('I') + self.assertEqual(f(int_data), f(bytes_data)) + + def test_b64encode(self): eq = self.assertEqual # Test default alphabet @@ -90,13 +137,16 @@ class BaseXYTestCase(unittest.TestCase): b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==") # Test with arbitrary alternative characters eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=b'*$'), b'01a*b$cd') - # Non-bytes - eq(base64.b64encode(bytearray(b'abcd')), b'YWJjZA==') eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=bytearray(b'*$')), b'01a*b$cd') - # Check if passing a str object raises an error - self.assertRaises(TypeError, base64.b64encode, "") - self.assertRaises(TypeError, base64.b64encode, b"", altchars="") + eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=memoryview(b'*$')), + b'01a*b$cd') + eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=array('B', b'*$')), + b'01a*b$cd') + # Non-bytes + self.check_other_types(base64.b64encode, b'abcd', b'YWJjZA==') + self.check_encode_type_errors(base64.b64encode) + self.assertRaises(TypeError, base64.b64encode, b"", altchars="*$") # Test standard alphabet eq(base64.standard_b64encode(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=") eq(base64.standard_b64encode(b"a"), b"YQ==") @@ -110,15 +160,15 @@ class BaseXYTestCase(unittest.TestCase): b"RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NT" b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==") # Non-bytes - eq(base64.standard_b64encode(bytearray(b'abcd')), b'YWJjZA==') - # Check if passing a str object raises an error - self.assertRaises(TypeError, base64.standard_b64encode, "") + self.check_other_types(base64.standard_b64encode, + b'abcd', b'YWJjZA==') + self.check_encode_type_errors(base64.standard_b64encode) # Test with 'URL safe' alternative characters eq(base64.urlsafe_b64encode(b'\xd3V\xbeo\xf7\x1d'), b'01a-b_cd') # Non-bytes - eq(base64.urlsafe_b64encode(bytearray(b'\xd3V\xbeo\xf7\x1d')), b'01a-b_cd') - # Check if passing a str object raises an error - self.assertRaises(TypeError, base64.urlsafe_b64encode, "") + self.check_other_types(base64.urlsafe_b64encode, + b'\xd3V\xbeo\xf7\x1d', b'01a-b_cd') + self.check_encode_type_errors(base64.urlsafe_b64encode) def test_b64decode(self): eq = self.assertEqual @@ -141,7 +191,8 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.b64decode(data), res) eq(base64.b64decode(data.decode('ascii')), res) # Non-bytes - eq(base64.b64decode(bytearray(b"YWJj")), b"abc") + self.check_other_types(base64.b64decode, b"YWJj", b"abc") + self.check_decode_type_errors(base64.b64decode) # Test with arbitrary alternative characters tests_altchars = {(b'01a*b$cd', b'*$'): b'\xd3V\xbeo\xf7\x1d', @@ -160,7 +211,8 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.standard_b64decode(data), res) eq(base64.standard_b64decode(data.decode('ascii')), res) # Non-bytes - eq(base64.standard_b64decode(bytearray(b"YWJj")), b"abc") + self.check_other_types(base64.standard_b64decode, b"YWJj", b"abc") + self.check_decode_type_errors(base64.standard_b64decode) # Test with 'URL safe' alternative characters tests_urlsafe = {b'01a-b_cd': b'\xd3V\xbeo\xf7\x1d', @@ -170,7 +222,9 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.urlsafe_b64decode(data), res) eq(base64.urlsafe_b64decode(data.decode('ascii')), res) # Non-bytes - eq(base64.urlsafe_b64decode(bytearray(b'01a-b_cd')), b'\xd3V\xbeo\xf7\x1d') + self.check_other_types(base64.urlsafe_b64decode, b'01a-b_cd', + b'\xd3V\xbeo\xf7\x1d') + self.check_decode_type_errors(base64.urlsafe_b64decode) def test_b64decode_padding_error(self): self.assertRaises(binascii.Error, base64.b64decode, b'abc') @@ -205,8 +259,8 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.b32encode(b'abcd'), b'MFRGGZA=') eq(base64.b32encode(b'abcde'), b'MFRGGZDF') # Non-bytes - eq(base64.b32encode(bytearray(b'abcd')), b'MFRGGZA=') - self.assertRaises(TypeError, base64.b32encode, "") + self.check_other_types(base64.b32encode, b'abcd', b'MFRGGZA=') + self.check_encode_type_errors(base64.b32encode) def test_b32decode(self): eq = self.assertEqual @@ -222,7 +276,8 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.b32decode(data), res) eq(base64.b32decode(data.decode('ascii')), res) # Non-bytes - eq(base64.b32decode(bytearray(b'MFRGG===')), b'abc') + self.check_other_types(base64.b32decode, b'MFRGG===', b"abc") + self.check_decode_type_errors(base64.b32decode) def test_b32decode_casefold(self): eq = self.assertEqual @@ -277,8 +332,9 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.b16encode(b'\x01\x02\xab\xcd\xef'), b'0102ABCDEF') eq(base64.b16encode(b'\x00'), b'00') # Non-bytes - eq(base64.b16encode(bytearray(b'\x01\x02\xab\xcd\xef')), b'0102ABCDEF') - self.assertRaises(TypeError, base64.b16encode, "") + self.check_other_types(base64.b16encode, b'\x01\x02\xab\xcd\xef', + b'0102ABCDEF') + self.check_encode_type_errors(base64.b16encode) def test_b16decode(self): eq = self.assertEqual @@ -293,7 +349,15 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.b16decode(b'0102abcdef', True), b'\x01\x02\xab\xcd\xef') eq(base64.b16decode('0102abcdef', True), b'\x01\x02\xab\xcd\xef') # Non-bytes - eq(base64.b16decode(bytearray(b"0102ABCDEF")), b'\x01\x02\xab\xcd\xef') + self.check_other_types(base64.b16decode, b"0102ABCDEF", + b'\x01\x02\xab\xcd\xef') + self.check_decode_type_errors(base64.b16decode) + eq(base64.b16decode(bytearray(b"0102abcdef"), True), + b'\x01\x02\xab\xcd\xef') + eq(base64.b16decode(memoryview(b"0102abcdef"), True), + b'\x01\x02\xab\xcd\xef') + eq(base64.b16decode(array('B', b"0102abcdef"), True), + b'\x01\x02\xab\xcd\xef') def test_decode_nonascii_str(self): decode_funcs = (base64.b64decode, diff --git a/Lib/test/test_bisect.py b/Lib/test/test_bisect.py index 7b9bd19fcb..580a963f62 100644 --- a/Lib/test/test_bisect.py +++ b/Lib/test/test_bisect.py @@ -7,7 +7,7 @@ py_bisect = support.import_fresh_module('bisect', blocked=['_bisect']) c_bisect = support.import_fresh_module('bisect', fresh=['_bisect']) class Range(object): - """A trivial range()-like object without any integer width limitations.""" + """A trivial range()-like object that has an insert() method.""" def __init__(self, start, stop): self.start = start self.stop = stop @@ -120,10 +120,10 @@ class TestBisect: def test_negative_lo(self): # Issue 3301 mod = self.module - self.assertRaises(ValueError, mod.bisect_left, [1, 2, 3], 5, -1, 3), - self.assertRaises(ValueError, mod.bisect_right, [1, 2, 3], 5, -1, 3), - self.assertRaises(ValueError, mod.insort_left, [1, 2, 3], 5, -1, 3), - self.assertRaises(ValueError, mod.insort_right, [1, 2, 3], 5, -1, 3), + self.assertRaises(ValueError, mod.bisect_left, [1, 2, 3], 5, -1, 3) + self.assertRaises(ValueError, mod.bisect_right, [1, 2, 3], 5, -1, 3) + self.assertRaises(ValueError, mod.insort_left, [1, 2, 3], 5, -1, 3) + self.assertRaises(ValueError, mod.insort_right, [1, 2, 3], 5, -1, 3) def test_large_range(self): # Issue 13496 diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 04e3f61e89..1667847a9d 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4290,9 +4290,5 @@ class TestBufferProtocol(unittest.TestCase): self.assertRaises(BufferError, memoryview, x) -def test_main(): - support.run_unittest(TestBufferProtocol) - - if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index c342a4329f..2411c9b311 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -464,6 +464,11 @@ class BuiltinTest(unittest.TestCase): self.assertRaises(TypeError, eval, ()) self.assertRaises(SyntaxError, eval, bom[:2] + b'a') + class X: + def __getitem__(self, key): + raise ValueError + self.assertRaises(ValueError, eval, "foo", {}, X()) + def test_general_eval(self): # Tests that general mappings can be used for the locals argument @@ -579,7 +584,10 @@ class BuiltinTest(unittest.TestCase): raise frozendict_error("frozendict is readonly") # read-only builtins - frozen_builtins = frozendict(__builtins__) + if isinstance(__builtins__, types.ModuleType): + frozen_builtins = frozendict(__builtins__.__dict__) + else: + frozen_builtins = frozendict(__builtins__) code = compile("__builtins__['superglobal']=2; print(superglobal)", "test", "exec") self.assertRaises(frozendict_error, exec, code, {'__builtins__': frozen_builtins}) @@ -839,8 +847,19 @@ class BuiltinTest(unittest.TestCase): self.assertEqual(max(1, 2.0, 3), 3) self.assertEqual(max(1.0, 2, 3), 3) + self.assertRaises(TypeError, max) + self.assertRaises(TypeError, max, 42) + self.assertRaises(ValueError, max, ()) + class BadSeq: + def __getitem__(self, index): + raise ValueError + self.assertRaises(ValueError, max, BadSeq()) + for stmt in ( "max(key=int)", # no args + "max(default=None)", + "max(1, 2, default=None)", # require container for default + "max(default=None, key=int)", "max(1, key=int)", # single arg not iterable "max(1, 2, keystone=int)", # wrong keyword "max(1, 2, key=int, abc=int)", # two many keywords @@ -857,6 +876,13 @@ class BuiltinTest(unittest.TestCase): self.assertEqual(max((1,2), key=neg), 1) # two elem iterable self.assertEqual(max(1, 2, key=neg), 1) # two elems + self.assertEqual(max((), default=None), None) # zero elem iterable + self.assertEqual(max((1,), default=None), 1) # one elem iterable + self.assertEqual(max((1,2), default=None), 2) # two elem iterable + + self.assertEqual(max((), default=1, key=neg), 1) + self.assertEqual(max((1, 2), default=3, key=neg), 1) + data = [random.randrange(200) for i in range(100)] keys = dict((elem, random.randrange(50)) for elem in data) f = keys.__getitem__ @@ -883,6 +909,9 @@ class BuiltinTest(unittest.TestCase): for stmt in ( "min(key=int)", # no args + "min(default=None)", + "min(1, 2, default=None)", # require container for default + "min(default=None, key=int)", "min(1, key=int)", # single arg not iterable "min(1, 2, keystone=int)", # wrong keyword "min(1, 2, key=int, abc=int)", # two many keywords @@ -899,6 +928,13 @@ class BuiltinTest(unittest.TestCase): self.assertEqual(min((1,2), key=neg), 2) # two elem iterable self.assertEqual(min(1, 2, key=neg), 2) # two elems + self.assertEqual(min((), default=None), None) # zero elem iterable + self.assertEqual(min((1,), default=None), 1) # one elem iterable + self.assertEqual(min((1,2), default=None), 1) # two elem iterable + + self.assertEqual(min((), default=1, key=neg), 1) + self.assertEqual(min((1, 2), default=1, key=neg), 2) + data = [random.randrange(200) for i in range(100)] keys = dict((elem, random.randrange(50)) for elem in data) f = keys.__getitem__ @@ -940,29 +976,25 @@ class BuiltinTest(unittest.TestCase): def write_testfile(self): # NB the first 4 lines are also used to test input, below fp = open(TESTFN, 'w') - try: + self.addCleanup(unlink, TESTFN) + with fp: fp.write('1+1\n') fp.write('The quick brown fox jumps over the lazy dog') fp.write('.\n') fp.write('Dear John\n') fp.write('XXX'*100) fp.write('YYY'*100) - finally: - fp.close() def test_open(self): self.write_testfile() fp = open(TESTFN, 'r') - try: + with fp: self.assertEqual(fp.readline(4), '1+1\n') self.assertEqual(fp.readline(), 'The quick brown fox jumps over the lazy dog.\n') self.assertEqual(fp.readline(4), 'Dear') self.assertEqual(fp.readline(100), ' John\n') self.assertEqual(fp.read(300), 'XXX'*100) self.assertEqual(fp.read(1000), 'YYY'*100) - finally: - fp.close() - unlink(TESTFN) def test_open_default_encoding(self): old_environ = dict(os.environ) @@ -977,15 +1009,17 @@ class BuiltinTest(unittest.TestCase): self.write_testfile() current_locale_encoding = locale.getpreferredencoding(False) fp = open(TESTFN, 'w') - try: + with fp: self.assertEqual(fp.encoding, current_locale_encoding) - finally: - fp.close() - unlink(TESTFN) finally: os.environ.clear() os.environ.update(old_environ) + def test_open_non_inheritable(self): + fileobj = open(__file__) + with fileobj: + self.assertFalse(os.get_inheritable(fileobj.fileno())) + def test_ord(self): self.assertEqual(ord(' '), 32) self.assertEqual(ord('A'), 65) @@ -1096,7 +1130,6 @@ class BuiltinTest(unittest.TestCase): sys.stdin = savestdin sys.stdout = savestdout fp.close() - unlink(TESTFN) @unittest.skipUnless(pty, "the pty and signal modules must be available") def check_input_tty(self, prompt, terminal_input, stdio_encoding=None): @@ -1476,17 +1509,11 @@ class BuiltinTest(unittest.TestCase): # -------------------------------------------------------------------- # Issue #7994: object.__format__ with a non-empty format string is # deprecated - def test_deprecated_format_string(obj, fmt_str, should_raise_warning): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always", DeprecationWarning) - format(obj, fmt_str) - if should_raise_warning: - self.assertEqual(len(w), 1) - self.assertIsInstance(w[0].message, DeprecationWarning) - self.assertIn('object.__format__ with a non-empty format ' - 'string', str(w[0].message)) + def test_deprecated_format_string(obj, fmt_str, should_raise): + if should_raise: + self.assertRaises(TypeError, format, obj, fmt_str) else: - self.assertEqual(len(w), 0) + format(obj, fmt_str) fmt_strs = ['', 's'] diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 3520e837a1..847c7a613f 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -288,8 +288,22 @@ class BaseBytesTest: self.assertEqual(self.type2test(b"").join(lst), b"abc") self.assertEqual(self.type2test(b"").join(tuple(lst)), b"abc") self.assertEqual(self.type2test(b"").join(iter(lst)), b"abc") - self.assertEqual(self.type2test(b".").join([b"ab", b"cd"]), b"ab.cd") - # XXX more... + dot_join = self.type2test(b".:").join + self.assertEqual(dot_join([b"ab", b"cd"]), b"ab.:cd") + self.assertEqual(dot_join([memoryview(b"ab"), b"cd"]), b"ab.:cd") + self.assertEqual(dot_join([b"ab", memoryview(b"cd")]), b"ab.:cd") + self.assertEqual(dot_join([bytearray(b"ab"), b"cd"]), b"ab.:cd") + self.assertEqual(dot_join([b"ab", bytearray(b"cd")]), b"ab.:cd") + # Stress it with many items + seq = [b"abc"] * 1000 + expected = b"abc" + b".:abc" * 999 + self.assertEqual(dot_join(seq), expected) + # Error handling and cleanup when some item in the middle of the + # sequence has the wrong type. + with self.assertRaises(TypeError): + dot_join([bytearray(b"ab"), "cd", b"ef"]) + with self.assertRaises(TypeError): + dot_join([memoryview(b"ab"), "cd", b"ef"]) def test_count(self): b = self.type2test(b'mississippi') @@ -759,7 +773,7 @@ class ByteArrayTest(BaseBytesTest, unittest.TestCase): finally: try: os.remove(tfn) - except os.error: + except OSError: pass def test_reverse(self): @@ -895,6 +909,15 @@ class ByteArrayTest(BaseBytesTest, unittest.TestCase): with self.assertRaises(ValueError): b[3:4] = elem + def test_setslice_extend(self): + # Exercise the resizing logic (see issue #19087) + b = bytearray(range(100)) + self.assertEqual(list(b), list(range(100))) + del b[:10] + self.assertEqual(list(b), list(range(10, 100))) + b.extend(range(100, 110)) + self.assertEqual(list(b), list(range(10, 110))) + def test_extended_set_del_slice(self): indices = (0, None, 1, 3, 19, 300, 1<<333, -1, -2, -31, -300) for start in indices: @@ -1274,6 +1297,11 @@ class BytearrayPEP3137Test(unittest.TestCase, self.assertEqual(val, newval) self.assertTrue(val is not newval, expr+' returned val on a mutable object') + sep = self.marshal(b'') + newval = sep.join([val]) + self.assertEqual(val, newval) + self.assertIsNot(val, newval) + class FixedStringTest(test.string_tests.BaseTest): diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 912fac1c33..7090cd6935 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 from test import support -from test.support import TESTFN, bigmemtest, _4G +from test.support import bigmemtest, _4G import unittest from io import BytesIO @@ -18,10 +18,10 @@ except ImportError: bz2 = support.import_module('bz2') from bz2 import BZ2File, BZ2Compressor, BZ2Decompressor -has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx") class BaseTest(unittest.TestCase): "Base for other testcases." + TEXT_LINES = [ b'root:x:0:0:root:/root:/bin/bash\n', b'bin:x:1:1:bin:/bin:\n', @@ -50,13 +50,17 @@ class BaseTest(unittest.TestCase): EMPTY_DATA = b'BZh9\x17rE8P\x90\x00\x00\x00\x00' def setUp(self): - self.filename = TESTFN + self.filename = support.TESTFN def tearDown(self): if os.path.isfile(self.filename): os.unlink(self.filename) - if has_cmdline_bunzip2: + if sys.platform == "win32": + # bunzip2 isn't available to run on Windows. + def decompress(self, data): + return bz2.decompress(data) + else: def decompress(self, data): pop = subprocess.Popen("bunzip2", shell=True, stdin=subprocess.PIPE, @@ -70,31 +74,21 @@ class BaseTest(unittest.TestCase): ret = bz2.decompress(data) return ret - else: - # bunzip2 isn't available to run on Windows. - def decompress(self, data): - return bz2.decompress(data) class BZ2FileTest(BaseTest): - "Test BZ2File type miscellaneous methods." + "Test the BZ2File class." def createTempFile(self, streams=1): with open(self.filename, "wb") as f: f.write(self.DATA * streams) def testBadArgs(self): - with self.assertRaises(TypeError): - BZ2File(123.456) - with self.assertRaises(ValueError): - BZ2File("/dev/null", "z") - with self.assertRaises(ValueError): - BZ2File("/dev/null", "rx") - with self.assertRaises(ValueError): - BZ2File("/dev/null", "rbt") - with self.assertRaises(ValueError): - BZ2File("/dev/null", compresslevel=0) - with self.assertRaises(ValueError): - BZ2File("/dev/null", compresslevel=10) + self.assertRaises(TypeError, BZ2File, 123.456) + self.assertRaises(ValueError, BZ2File, "/dev/null", "z") + self.assertRaises(ValueError, BZ2File, "/dev/null", "rx") + self.assertRaises(ValueError, BZ2File, "/dev/null", "rbt") + self.assertRaises(ValueError, BZ2File, "/dev/null", compresslevel=0) + self.assertRaises(ValueError, BZ2File, "/dev/null", compresslevel=10) def testRead(self): self.createTempFile() @@ -215,9 +209,8 @@ class BZ2FileTest(BaseTest): self.createTempFile() bz2f = BZ2File(self.filename) bz2f.close() - self.assertRaises(ValueError, bz2f.__next__) - # This call will deadlock if the above .__next__ call failed to - # release the lock. + self.assertRaises(ValueError, next, bz2f) + # This call will deadlock if the above call failed to release the lock. self.assertRaises(ValueError, bz2f.readlines) def testWrite(self): @@ -261,8 +254,8 @@ class BZ2FileTest(BaseTest): bz2f.write(b"abc") with BZ2File(self.filename, "r") as bz2f: - self.assertRaises(IOError, bz2f.write, b"a") - self.assertRaises(IOError, bz2f.writelines, [b"a"]) + self.assertRaises(OSError, bz2f.write, b"a") + self.assertRaises(OSError, bz2f.writelines, [b"a"]) def testAppend(self): with BZ2File(self.filename, "w") as bz2f: @@ -380,7 +373,7 @@ class BZ2FileTest(BaseTest): bz2f.close() self.assertRaises(ValueError, bz2f.seekable) - bz2f = BZ2File(BytesIO(), mode="w") + bz2f = BZ2File(BytesIO(), "w") try: self.assertFalse(bz2f.seekable()) finally: @@ -406,7 +399,7 @@ class BZ2FileTest(BaseTest): bz2f.close() self.assertRaises(ValueError, bz2f.readable) - bz2f = BZ2File(BytesIO(), mode="w") + bz2f = BZ2File(BytesIO(), "w") try: self.assertFalse(bz2f.readable()) finally: @@ -423,7 +416,7 @@ class BZ2FileTest(BaseTest): bz2f.close() self.assertRaises(ValueError, bz2f.writable) - bz2f = BZ2File(BytesIO(), mode="w") + bz2f = BZ2File(BytesIO(), "w") try: self.assertTrue(bz2f.writable()) finally: @@ -437,7 +430,7 @@ class BZ2FileTest(BaseTest): del o def testOpenNonexistent(self): - self.assertRaises(IOError, BZ2File, "/non/existent") + self.assertRaises(OSError, BZ2File, "/non/existent") def testReadlinesNoNewline(self): # Issue #1191043: readlines() fails on a file containing no newline. @@ -477,7 +470,7 @@ class BZ2FileTest(BaseTest): # Issue #7205: Using a BZ2File from several threads shouldn't deadlock. data = b"1" * 2**20 nthreads = 10 - with bz2.BZ2File(self.filename, 'wb') as f: + with BZ2File(self.filename, 'wb') as f: def comp(): for i in range(5): f.write(data) @@ -488,28 +481,27 @@ class BZ2FileTest(BaseTest): t.join() def testWithoutThreading(self): - bz2 = support.import_fresh_module("bz2", blocked=("threading",)) - with bz2.BZ2File(self.filename, "wb") as f: + module = support.import_fresh_module("bz2", blocked=("threading",)) + with module.BZ2File(self.filename, "wb") as f: f.write(b"abc") - with bz2.BZ2File(self.filename, "rb") as f: + with module.BZ2File(self.filename, "rb") as f: self.assertEqual(f.read(), b"abc") def testMixedIterationAndReads(self): self.createTempFile() linelen = len(self.TEXT_LINES[0]) halflen = linelen // 2 - with bz2.BZ2File(self.filename) as bz2f: + with BZ2File(self.filename) as bz2f: bz2f.read(halflen) self.assertEqual(next(bz2f), self.TEXT_LINES[0][halflen:]) self.assertEqual(bz2f.read(), self.TEXT[linelen:]) - with bz2.BZ2File(self.filename) as bz2f: + with BZ2File(self.filename) as bz2f: bz2f.readline() self.assertEqual(next(bz2f), self.TEXT_LINES[1]) self.assertEqual(bz2f.readline(), self.TEXT_LINES[2]) - with bz2.BZ2File(self.filename) as bz2f: + with BZ2File(self.filename) as bz2f: bz2f.readlines() - with self.assertRaises(StopIteration): - next(bz2f) + self.assertRaises(StopIteration, next, bz2f) self.assertEqual(bz2f.readlines(), []) def testMultiStreamOrdering(self): @@ -577,6 +569,20 @@ class BZ2FileTest(BaseTest): bz2f.seek(-150, 1) self.assertEqual(bz2f.read(), self.TEXT[500-150:]) + def test_read_truncated(self): + # Drop the eos_magic field (6 bytes) and CRC (4 bytes). + truncated = self.DATA[:-10] + with BZ2File(BytesIO(truncated)) as f: + self.assertRaises(EOFError, f.read) + with BZ2File(BytesIO(truncated)) as f: + self.assertEqual(f.read(len(self.TEXT)), self.TEXT) + self.assertRaises(EOFError, f.read, 1) + # Incomplete 4-byte file header, and block header of at least 146 bits. + for i in range(22): + with BZ2File(BytesIO(truncated[:i])) as f: + self.assertRaises(EOFError, f.read, 1) + + class BZ2CompressorTest(BaseTest): def testCompress(self): bz2c = BZ2Compressor() @@ -703,97 +709,102 @@ class CompressDecompressTest(BaseTest): class OpenTest(BaseTest): + "Test the open function." + + def open(self, *args, **kwargs): + return bz2.open(*args, **kwargs) + def test_binary_modes(self): - with bz2.open(self.filename, "wb") as f: + with self.open(self.filename, "wb") as f: f.write(self.TEXT) with open(self.filename, "rb") as f: - file_data = bz2.decompress(f.read()) + file_data = self.decompress(f.read()) self.assertEqual(file_data, self.TEXT) - with bz2.open(self.filename, "rb") as f: + with self.open(self.filename, "rb") as f: self.assertEqual(f.read(), self.TEXT) - with bz2.open(self.filename, "ab") as f: + with self.open(self.filename, "ab") as f: f.write(self.TEXT) with open(self.filename, "rb") as f: - file_data = bz2.decompress(f.read()) + file_data = self.decompress(f.read()) self.assertEqual(file_data, self.TEXT * 2) def test_implicit_binary_modes(self): # Test implicit binary modes (no "b" or "t" in mode string). - with bz2.open(self.filename, "w") as f: + with self.open(self.filename, "w") as f: f.write(self.TEXT) with open(self.filename, "rb") as f: - file_data = bz2.decompress(f.read()) + file_data = self.decompress(f.read()) self.assertEqual(file_data, self.TEXT) - with bz2.open(self.filename, "r") as f: + with self.open(self.filename, "r") as f: self.assertEqual(f.read(), self.TEXT) - with bz2.open(self.filename, "a") as f: + with self.open(self.filename, "a") as f: f.write(self.TEXT) with open(self.filename, "rb") as f: - file_data = bz2.decompress(f.read()) + file_data = self.decompress(f.read()) self.assertEqual(file_data, self.TEXT * 2) def test_text_modes(self): text = self.TEXT.decode("ascii") text_native_eol = text.replace("\n", os.linesep) - with bz2.open(self.filename, "wt") as f: + with self.open(self.filename, "wt") as f: f.write(text) with open(self.filename, "rb") as f: - file_data = bz2.decompress(f.read()).decode("ascii") + file_data = self.decompress(f.read()).decode("ascii") self.assertEqual(file_data, text_native_eol) - with bz2.open(self.filename, "rt") as f: + with self.open(self.filename, "rt") as f: self.assertEqual(f.read(), text) - with bz2.open(self.filename, "at") as f: + with self.open(self.filename, "at") as f: f.write(text) with open(self.filename, "rb") as f: - file_data = bz2.decompress(f.read()).decode("ascii") + file_data = self.decompress(f.read()).decode("ascii") self.assertEqual(file_data, text_native_eol * 2) def test_fileobj(self): - with bz2.open(BytesIO(self.DATA), "r") as f: + with self.open(BytesIO(self.DATA), "r") as f: self.assertEqual(f.read(), self.TEXT) - with bz2.open(BytesIO(self.DATA), "rb") as f: + with self.open(BytesIO(self.DATA), "rb") as f: self.assertEqual(f.read(), self.TEXT) text = self.TEXT.decode("ascii") - with bz2.open(BytesIO(self.DATA), "rt") as f: + with self.open(BytesIO(self.DATA), "rt") as f: self.assertEqual(f.read(), text) def test_bad_params(self): # Test invalid parameter combinations. - with self.assertRaises(ValueError): - bz2.open(self.filename, "wbt") - with self.assertRaises(ValueError): - bz2.open(self.filename, "rb", encoding="utf-8") - with self.assertRaises(ValueError): - bz2.open(self.filename, "rb", errors="ignore") - with self.assertRaises(ValueError): - bz2.open(self.filename, "rb", newline="\n") + self.assertRaises(ValueError, + self.open, self.filename, "wbt") + self.assertRaises(ValueError, + self.open, self.filename, "rb", encoding="utf-8") + self.assertRaises(ValueError, + self.open, self.filename, "rb", errors="ignore") + self.assertRaises(ValueError, + self.open, self.filename, "rb", newline="\n") def test_encoding(self): # Test non-default encoding. text = self.TEXT.decode("ascii") text_native_eol = text.replace("\n", os.linesep) - with bz2.open(self.filename, "wt", encoding="utf-16-le") as f: + with self.open(self.filename, "wt", encoding="utf-16-le") as f: f.write(text) with open(self.filename, "rb") as f: - file_data = bz2.decompress(f.read()).decode("utf-16-le") + file_data = self.decompress(f.read()).decode("utf-16-le") self.assertEqual(file_data, text_native_eol) - with bz2.open(self.filename, "rt", encoding="utf-16-le") as f: + with self.open(self.filename, "rt", encoding="utf-16-le") as f: self.assertEqual(f.read(), text) def test_encoding_error_handler(self): # Test with non-default encoding error handler. - with bz2.open(self.filename, "wb") as f: + with self.open(self.filename, "wb") as f: f.write(b"foo\xffbar") - with bz2.open(self.filename, "rt", encoding="ascii", errors="ignore") \ + with self.open(self.filename, "rt", encoding="ascii", errors="ignore") \ as f: self.assertEqual(f.read(), "foobar") def test_newline(self): # Test with explicit newline (universal newline mode disabled). text = self.TEXT.decode("ascii") - with bz2.open(self.filename, "wt", newline="\n") as f: + with self.open(self.filename, "wt", newline="\n") as f: f.write(text) - with bz2.open(self.filename, "rt", newline="\r") as f: + with self.open(self.filename, "rt", newline="\r") as f: self.assertEqual(f.readlines(), [text]) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index f1ea5a9fde..bbbacc2665 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -44,7 +44,7 @@ class CAPITest(unittest.TestCase): @unittest.skipUnless(threading, 'Threading required for this test.') def test_no_FatalError_infinite_loop(self): - with support.suppress_crash_popup(): + with support.SuppressCrashReport(): p = subprocess.Popen([sys.executable, "-c", 'import _testcapi;' '_testcapi.crash_no_current_thread()'], @@ -193,6 +193,9 @@ class TestPendingCalls(unittest.TestCase): self.pendingcalls_submit(l, n) self.pendingcalls_wait(l, n) + +class SubinterpreterTest(unittest.TestCase): + def test_subinterps(self): import builtins r, w = os.pipe() @@ -208,6 +211,7 @@ class TestPendingCalls(unittest.TestCase): self.assertNotEqual(pickle.load(f), id(sys.modules)) self.assertNotEqual(pickle.load(f), id(builtins)) + # Bug #6012 class Test6012(unittest.TestCase): def test(self): @@ -354,7 +358,8 @@ class TestThreadState(unittest.TestCase): def test_main(): support.run_unittest(CAPITest, TestPendingCalls, Test6012, - EmbeddingTest, SkipitemTest, TestThreadState) + EmbeddingTest, SkipitemTest, TestThreadState, + SubinterpreterTest) for name in dir(_testcapi): if name.startswith('test_'): diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index a89d7e4f10..327c1455fc 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -4,6 +4,7 @@ import test.support, unittest import os +import shutil import sys import subprocess import tempfile @@ -41,8 +42,10 @@ class CmdLineTest(unittest.TestCase): def test_version(self): version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii") - rc, out, err = assert_python_ok('-V') - self.assertTrue(err.startswith(version)) + for switch in '-V', '--version': + rc, out, err = assert_python_ok(switch) + self.assertFalse(err.startswith(version)) + self.assertTrue(out.startswith(version)) def test_verbose(self): # -v causes imports to write to stderr. If the write to @@ -54,14 +57,49 @@ class CmdLineTest(unittest.TestCase): self.assertNotIn(b'stack overflow', err) def test_xoptions(self): - rc, out, err = assert_python_ok('-c', 'import sys; print(sys._xoptions)') - opts = eval(out.splitlines()[0]) + def get_xoptions(*args): + # use subprocess module directly because test.script_helper adds + # "-X faulthandler" to the command line + args = (sys.executable, '-E') + args + args += ('-c', 'import sys; print(sys._xoptions)') + out = subprocess.check_output(args) + opts = eval(out.splitlines()[0]) + return opts + + opts = get_xoptions() self.assertEqual(opts, {}) - rc, out, err = assert_python_ok( - '-Xa', '-Xb=c,d=e', '-c', 'import sys; print(sys._xoptions)') - opts = eval(out.splitlines()[0]) + + opts = get_xoptions('-Xa', '-Xb=c,d=e') self.assertEqual(opts, {'a': True, 'b': 'c,d=e'}) + def test_showrefcount(self): + def run_python(*args): + # this is similar to assert_python_ok but doesn't strip + # the refcount from stderr. It can be replaced once + # assert_python_ok stops doing that. + cmd = [sys.executable] + cmd.extend(args) + PIPE = subprocess.PIPE + p = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE) + out, err = p.communicate() + p.stdout.close() + p.stderr.close() + rc = p.returncode + self.assertEqual(rc, 0) + return rc, out, err + code = 'import sys; print(sys._xoptions)' + # normally the refcount is hidden + rc, out, err = run_python('-c', code) + self.assertEqual(out.rstrip(), b'{}') + self.assertEqual(err, b'') + # "-X showrefcount" shows the refcount, but only in debug builds + rc, out, err = run_python('-X', 'showrefcount', '-c', code) + self.assertEqual(out.rstrip(), b"{'showrefcount': True}") + if hasattr(sys, 'gettotalrefcount'): # debug build + self.assertRegex(err, br'^\[\d+ refs, \d+ blocks\]') + else: + self.assertEqual(err, b'') + def test_run_module(self): # Test expected operation of the '-m' switch # Switch needs an argument @@ -70,9 +108,9 @@ class CmdLineTest(unittest.TestCase): assert_python_failure('-m', 'fnord43520xyz') # Check the runpy module also gives an error for # a nonexistent module - assert_python_failure('-m', 'runpy', 'fnord43520xyz'), + assert_python_failure('-m', 'runpy', 'fnord43520xyz') # All good if module is located and run successfully - assert_python_ok('-m', 'timeit', '-n', '1'), + assert_python_ok('-m', 'timeit', '-n', '1') def test_run_module_bug1764407(self): # -m and -i need to play well together @@ -213,6 +251,23 @@ class CmdLineTest(unittest.TestCase): self.assertIn(path1.encode('ascii'), out) self.assertIn(path2.encode('ascii'), out) + def test_empty_PYTHONPATH_issue16309(self): + # On Posix, it is documented that setting PATH to the + # empty string is equivalent to not setting PATH at all, + # which is an exception to the rule that in a string like + # "/bin::/usr/bin" the empty string in the middle gets + # interpreted as '.' + code = """if 1: + import sys + path = ":".join(sys.path) + path = path.encode("ascii", "backslashreplace") + sys.stdout.buffer.write(path)""" + rc1, out1, err1 = assert_python_ok('-c', code, PYTHONPATH="") + rc2, out2, err2 = assert_python_ok('-c', code, __isolated=False) + # regarding to Posix specification, outputs should be equal + # for empty and unset PYTHONPATH + self.assertEqual(out1, out2) + def test_displayhook_unencodable(self): for encoding in ('ascii', 'latin-1', 'utf-8'): env = os.environ.copy() @@ -290,7 +345,7 @@ class CmdLineTest(unittest.TestCase): rc, out, err = assert_python_ok('-c', code) self.assertEqual(b'', out) self.assertRegex(err.decode('ascii', 'ignore'), - 'Exception OSError: .* ignored') + 'Exception ignored in.*\nOSError: .*') def test_closed_stdout(self): # Issue #13444: if stdout has been explicitly closed, we should @@ -385,6 +440,31 @@ class CmdLineTest(unittest.TestCase): self.assertEqual(b'', out) + def test_isolatedmode(self): + self.verify_valid_flag('-I') + self.verify_valid_flag('-IEs') + rc, out, err = assert_python_ok('-I', '-c', + 'from sys import flags as f; ' + 'print(f.no_user_site, f.ignore_environment, f.isolated)', + # dummyvar to prevent extranous -E + dummyvar="") + self.assertEqual(out.strip(), b'1 1 1') + with test.support.temp_cwd() as tmpdir: + fake = os.path.join(tmpdir, "uuid.py") + main = os.path.join(tmpdir, "main.py") + with open(fake, "w") as f: + f.write("raise RuntimeError('isolated mode test')\n") + with open(main, "w") as f: + f.write("import uuid\n") + f.write("print('ok')\n") + self.assertRaises(subprocess.CalledProcessError, + subprocess.check_output, + [sys.executable, main], cwd=tmpdir, + stderr=subprocess.DEVNULL) + out = subprocess.check_output([sys.executable, "-I", main], + cwd=tmpdir) + self.assertEqual(out.strip(), b"ok") + def test_main(): test.support.run_unittest(CmdLineTest) test.support.reap_children() diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index b4269b994e..f804d8645f 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -123,7 +123,7 @@ class CmdLineTest(unittest.TestCase): if not __debug__: cmd_line_switches += ('-' + 'O' * sys.flags.optimize,) run_args = cmd_line_switches + (script_name,) + tuple(example_args) - rc, out, err = assert_python_ok(*run_args) + rc, out, err = assert_python_ok(*run_args, __isolated=False) self._check_output(script_name, rc, out + err, expected_file, expected_argv0, expected_path0, expected_package, expected_loader) @@ -294,7 +294,7 @@ class CmdLineTest(unittest.TestCase): pkg_dir = os.path.join(script_dir, 'test_pkg') make_pkg(pkg_dir, "import sys; print('init_argv0==%r' % sys.argv[0])") script_name = _make_test_script(pkg_dir, 'script') - rc, out, err = assert_python_ok('-m', 'test_pkg.script', *example_args) + rc, out, err = assert_python_ok('-m', 'test_pkg.script', *example_args, __isolated=False) if verbose > 1: print(out) expected = "init_argv0==%r" % '-m' @@ -311,7 +311,8 @@ class CmdLineTest(unittest.TestCase): with open("-c", "w") as f: f.write("data") rc, out, err = assert_python_ok('-c', - 'import sys; print("sys.path[0]==%r" % sys.path[0])') + 'import sys; print("sys.path[0]==%r" % sys.path[0])', + __isolated=False) if verbose > 1: print(out) expected = "sys.path[0]==%r" % '' @@ -325,7 +326,8 @@ class CmdLineTest(unittest.TestCase): with support.change_cwd(path=script_dir): with open("-m", "w") as f: f.write("data") - rc, out, err = assert_python_ok('-m', 'other', *example_args) + rc, out, err = assert_python_ok('-m', 'other', *example_args, + __isolated=False) self._check_output(script_name, rc, out, script_name, script_name, '', '', importlib.machinery.SourceFileLoader) diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index adef1701d4..5fd21dc32c 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -64,6 +64,20 @@ class TestInteractiveConsole(unittest.TestCase): self.console.interact() self.assertTrue(hook.called) + def test_banner(self): + # with banner + self.infunc.side_effect = EOFError('Finished') + self.console.interact(banner='Foo') + self.assertEqual(len(self.stderr.method_calls), 2) + banner_call = self.stderr.method_calls[0] + self.assertEqual(banner_call, ['write', ('Foo\n',), {}]) + + # no banner + self.stderr.reset_mock() + self.infunc.side_effect = EOFError('Finished') + self.console.interact(banner='') + self.assertEqual(len(self.stderr.method_calls), 1) + def test_main(): support.run_unittest(TestInteractiveConsole) diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index fd88505081..84804bb0da 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -875,8 +875,6 @@ class CodecCallbackTest(unittest.TestCase): with self.assertRaises(TypeError): data.decode(encoding, "test.replacing") -def test_main(): - test.support.run_unittest(CodecCallbackTest) if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index c68088ed75..99d928dcf1 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -2285,6 +2285,24 @@ class TransformCodecTest(unittest.TestCase): sout = reader.readline() self.assertEqual(sout, b"\x80") + def test_buffer_api_usage(self): + # We check all the transform codecs accept memoryview input + # for encoding and decoding + # and also that they roundtrip correctly + original = b"12345\x80" + for encoding in bytes_transform_encodings: + data = original + view = memoryview(data) + data = codecs.encode(data, encoding) + view_encoded = codecs.encode(view, encoding) + self.assertEqual(view_encoded, data) + view = memoryview(data) + data = codecs.decode(data, encoding) + self.assertEqual(data, original) + view_decoded = codecs.decode(view, encoding) + self.assertEqual(view_decoded, data) + + @unittest.skipUnless(sys.platform == 'win32', 'code pages are specific to Windows') @@ -2295,8 +2313,8 @@ class CodePageTest(unittest.TestCase): def test_invalid_code_page(self): self.assertRaises(ValueError, codecs.code_page_encode, -1, 'a') self.assertRaises(ValueError, codecs.code_page_decode, -1, b'a') - self.assertRaises(WindowsError, codecs.code_page_encode, 123, 'a') - self.assertRaises(WindowsError, codecs.code_page_decode, 123, b'a') + self.assertRaises(OSError, codecs.code_page_encode, 123, 'a') + self.assertRaises(OSError, codecs.code_page_decode, 123, b'a') def test_code_page_name(self): self.assertRaisesRegex(UnicodeEncodeError, 'cp932', diff --git a/Lib/test/test_coding.py b/Lib/test/test_coding.py deleted file mode 100644 index 989c7a85d2..0000000000 --- a/Lib/test/test_coding.py +++ /dev/null @@ -1,63 +0,0 @@ -import unittest -from test.support import TESTFN, unlink, unload -import importlib, os, sys - -class CodingTest(unittest.TestCase): - def test_bad_coding(self): - module_name = 'bad_coding' - self.verify_bad_module(module_name) - - def test_bad_coding2(self): - module_name = 'bad_coding2' - self.verify_bad_module(module_name) - - def verify_bad_module(self, module_name): - self.assertRaises(SyntaxError, __import__, 'test.' + module_name) - - path = os.path.dirname(__file__) - filename = os.path.join(path, module_name + '.py') - with open(filename, "rb") as fp: - bytes = fp.read() - self.assertRaises(SyntaxError, compile, bytes, filename, 'exec') - - def test_exec_valid_coding(self): - d = {} - exec(b'# coding: cp949\na = "\xaa\xa7"\n', d) - self.assertEqual(d['a'], '\u3047') - - def test_file_parse(self): - # issue1134: all encodings outside latin-1 and utf-8 fail on - # multiline strings and long lines (>512 columns) - unload(TESTFN) - filename = TESTFN + ".py" - f = open(filename, "w", encoding="cp1252") - sys.path.insert(0, os.curdir) - try: - with f: - f.write("# -*- coding: cp1252 -*-\n") - f.write("'''A short string\n") - f.write("'''\n") - f.write("'A very long string %s'\n" % ("X" * 1000)) - - importlib.invalidate_caches() - __import__(TESTFN) - finally: - del sys.path[0] - unlink(filename) - unlink(filename + "c") - unlink(filename + "o") - unload(TESTFN) - - def test_error_from_string(self): - # See http://bugs.python.org/issue6289 - input = "# coding: ascii\n\N{SNOWMAN}".encode('utf-8') - with self.assertRaises(SyntaxError) as c: - compile(input, "<string>", "exec") - expected = "'ascii' codec can't decode byte 0xe2 in position 16: " \ - "ordinal not in range(128)" - self.assertTrue(c.exception.args[0].startswith(expected), - msg=c.exception.args[0]) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index ff52755354..56e8120705 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -112,6 +112,38 @@ class TestChainMap(unittest.TestCase): self.assertEqual(dict(d), dict(a=1, b=2, c=30)) self.assertEqual(dict(d.items()), dict(a=1, b=2, c=30)) + def test_new_child(self): + 'Tests for changes for issue #16613.' + c = ChainMap() + c['a'] = 1 + c['b'] = 2 + m = {'b':20, 'c': 30} + d = c.new_child(m) + self.assertEqual(d.maps, [{'b':20, 'c':30}, {'a':1, 'b':2}]) # check internal state + self.assertIs(m, d.maps[0]) + + # Use a different map than a dict + class lowerdict(dict): + def __getitem__(self, key): + if isinstance(key, str): + key = key.lower() + return dict.__getitem__(self, key) + def __contains__(self, key): + if isinstance(key, str): + key = key.lower() + return dict.__contains__(self, key) + + c = ChainMap() + c['a'] = 1 + c['b'] = 2 + m = lowerdict(b=20, c=30) + d = c.new_child(m) + self.assertIs(m, d.maps[0]) + for key in 'abc': # check contains + self.assertIn(key, d) + for k, v in dict(a=1, B=20, C=30, z=100).items(): # check get + self.assertEqual(d.get(k, 100), v) + ################################################################################ ### Named Tuples @@ -750,6 +782,8 @@ class TestCollectionABCs(ABCTestCase): self.assertTrue(issubclass(sample, Sequence)) self.assertIsInstance(range(10), Sequence) self.assertTrue(issubclass(range, Sequence)) + self.assertIsInstance(memoryview(b""), Sequence) + self.assertTrue(issubclass(memoryview, Sequence)) self.assertTrue(issubclass(str, Sequence)) self.validate_abstract_methods(Sequence, '__contains__', '__iter__', '__len__', '__getitem__') @@ -904,23 +938,26 @@ class TestCounter(unittest.TestCase): words = Counter('which witch had which witches wrist watch'.split()) update_test = Counter() update_test.update(words) - for i, dup in enumerate([ - words.copy(), - copy.copy(words), - copy.deepcopy(words), - pickle.loads(pickle.dumps(words, 0)), - pickle.loads(pickle.dumps(words, 1)), - pickle.loads(pickle.dumps(words, 2)), - pickle.loads(pickle.dumps(words, -1)), - eval(repr(words)), - update_test, - Counter(words), - ]): - msg = (i, dup, words) - self.assertTrue(dup is not words) - self.assertEqual(dup, words) - self.assertEqual(len(dup), len(words)) - self.assertEqual(type(dup), type(words)) + for label, dup in [ + ('words.copy()', words.copy()), + ('copy.copy(words)', copy.copy(words)), + ('copy.deepcopy(words)', copy.deepcopy(words)), + ('pickle.loads(pickle.dumps(words, 0))', + pickle.loads(pickle.dumps(words, 0))), + ('pickle.loads(pickle.dumps(words, 1))', + pickle.loads(pickle.dumps(words, 1))), + ('pickle.loads(pickle.dumps(words, 2))', + pickle.loads(pickle.dumps(words, 2))), + ('pickle.loads(pickle.dumps(words, -1))', + pickle.loads(pickle.dumps(words, -1))), + ('eval(repr(words))', eval(repr(words))), + ('update_test', update_test), + ('Counter(words)', Counter(words)), + ]: + with self.subTest(label=label): + msg = "\ncopy: %s\nwords: %s" % (dup, words) + self.assertIsNot(dup, words, msg) + self.assertEqual(dup, words) def test_copy_subclass(self): class MyCounter(Counter): @@ -1043,8 +1080,10 @@ class TestCounter(unittest.TestCase): # test fidelity to the pure python version c = CounterSubclassWithSetItem('abracadabra') self.assertTrue(c.called) + self.assertEqual(dict(c), {'a': 5, 'b': 2, 'c': 1, 'd': 1, 'r':2 }) c = CounterSubclassWithGet('abracadabra') self.assertTrue(c.called) + self.assertEqual(dict(c), {'a': 5, 'b': 2, 'c': 1, 'd': 1, 'r':2 }) ################################################################################ @@ -1205,24 +1244,28 @@ class TestOrderedDict(unittest.TestCase): od = OrderedDict(pairs) update_test = OrderedDict() update_test.update(od) - for i, dup in enumerate([ - od.copy(), - copy.copy(od), - copy.deepcopy(od), - pickle.loads(pickle.dumps(od, 0)), - pickle.loads(pickle.dumps(od, 1)), - pickle.loads(pickle.dumps(od, 2)), - pickle.loads(pickle.dumps(od, 3)), - pickle.loads(pickle.dumps(od, -1)), - eval(repr(od)), - update_test, - OrderedDict(od), - ]): - self.assertTrue(dup is not od) - self.assertEqual(dup, od) - self.assertEqual(list(dup.items()), list(od.items())) - self.assertEqual(len(dup), len(od)) - self.assertEqual(type(dup), type(od)) + for label, dup in [ + ('od.copy()', od.copy()), + ('copy.copy(od)', copy.copy(od)), + ('copy.deepcopy(od)', copy.deepcopy(od)), + ('pickle.loads(pickle.dumps(od, 0))', + pickle.loads(pickle.dumps(od, 0))), + ('pickle.loads(pickle.dumps(od, 1))', + pickle.loads(pickle.dumps(od, 1))), + ('pickle.loads(pickle.dumps(od, 2))', + pickle.loads(pickle.dumps(od, 2))), + ('pickle.loads(pickle.dumps(od, 3))', + pickle.loads(pickle.dumps(od, 3))), + ('pickle.loads(pickle.dumps(od, -1))', + pickle.loads(pickle.dumps(od, -1))), + ('eval(repr(od))', eval(repr(od))), + ('update_test', update_test), + ('OrderedDict(od)', OrderedDict(od)), + ]: + with self.subTest(label=label): + msg = "\ncopy: %s\nod: %s" % (dup, od) + self.assertIsNot(dup, od, msg) + self.assertEqual(dup, od) def test_yaml_linkage(self): # Verify that __reduce__ is setup in a way that supports PyYAML's dump() feature. @@ -1237,9 +1280,18 @@ class TestOrderedDict(unittest.TestCase): # do not save instance dictionary if not needed pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] od = OrderedDict(pairs) - self.assertEqual(len(od.__reduce__()), 2) + self.assertIsNone(od.__reduce__()[2]) od.x = 10 - self.assertEqual(len(od.__reduce__()), 3) + self.assertIsNotNone(od.__reduce__()[2]) + + def test_pickle_recursive(self): + od = OrderedDict() + od[1] = od + for proto in range(-1, pickle.HIGHEST_PROTOCOL + 1): + dup = pickle.loads(pickle.dumps(od, proto)) + self.assertIsNot(dup, od) + self.assertEqual(list(dup.keys()), [1]) + self.assertIs(dup[1], dup) def test_repr(self): od = OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]) diff --git a/Lib/test/test_colorsys.py b/Lib/test/test_colorsys.py index e405b8a4d0..a24e3adcb4 100644 --- a/Lib/test/test_colorsys.py +++ b/Lib/test/test_colorsys.py @@ -1,4 +1,4 @@ -import unittest, test.support +import unittest import colorsys def frange(start, stop, step): @@ -69,8 +69,32 @@ class ColorsysTest(unittest.TestCase): self.assertTripleEqual(hls, colorsys.rgb_to_hls(*rgb)) self.assertTripleEqual(rgb, colorsys.hls_to_rgb(*hls)) -def test_main(): - test.support.run_unittest(ColorsysTest) + def test_yiq_roundtrip(self): + for r in frange(0.0, 1.0, 0.2): + for g in frange(0.0, 1.0, 0.2): + for b in frange(0.0, 1.0, 0.2): + rgb = (r, g, b) + self.assertTripleEqual( + rgb, + colorsys.yiq_to_rgb(*colorsys.rgb_to_yiq(*rgb)) + ) + + def test_yiq_values(self): + values = [ + # rgb, yiq + ((0.0, 0.0, 0.0), (0.0, 0.0, 0.0)), # black + ((0.0, 0.0, 1.0), (0.11, -0.3217, 0.3121)), # blue + ((0.0, 1.0, 0.0), (0.59, -0.2773, -0.5251)), # green + ((0.0, 1.0, 1.0), (0.7, -0.599, -0.213)), # cyan + ((1.0, 0.0, 0.0), (0.3, 0.599, 0.213)), # red + ((1.0, 0.0, 1.0), (0.41, 0.2773, 0.5251)), # purple + ((1.0, 1.0, 0.0), (0.89, 0.3217, -0.3121)), # yellow + ((1.0, 1.0, 1.0), (1.0, 0.0, 0.0)), # white + ((0.5, 0.5, 0.5), (0.5, 0.0, 0.0)), # grey + ] + for (rgb, yiq) in values: + self.assertTripleEqual(yiq, colorsys.rgb_to_yiq(*rgb)) + self.assertTripleEqual(rgb, colorsys.yiq_to_rgb(*yiq)) if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index ba9fe465f8..2122ade0ae 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -1,11 +1,12 @@ import sys import compileall -import imp +import importlib.util import os import py_compile import shutil import struct import subprocess +import sys import tempfile import time import unittest @@ -18,11 +19,11 @@ class CompileallTests(unittest.TestCase): def setUp(self): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') - self.bc_path = imp.cache_from_source(self.source_path) + self.bc_path = importlib.util.cache_from_source(self.source_path) with open(self.source_path, 'w') as file: file.write('x = 123\n') self.source_path2 = os.path.join(self.directory, '_test2.py') - self.bc_path2 = imp.cache_from_source(self.source_path2) + self.bc_path2 = importlib.util.cache_from_source(self.source_path2) shutil.copyfile(self.source_path, self.source_path2) self.subdirectory = os.path.join(self.directory, '_subdir') os.mkdir(self.subdirectory) @@ -36,7 +37,7 @@ class CompileallTests(unittest.TestCase): with open(self.bc_path, 'rb') as file: data = file.read(8) mtime = int(os.stat(self.source_path).st_mtime) - compare = struct.pack('<4sl', imp.get_magic(), mtime) + compare = struct.pack('<4sl', importlib.util.MAGIC_NUMBER, mtime) return data, compare def recreation_check(self, metadata): @@ -57,7 +58,8 @@ class CompileallTests(unittest.TestCase): def test_mtime(self): # Test a change in mtime leads to a new .pyc. - self.recreation_check(struct.pack('<4sl', imp.get_magic(), 1)) + self.recreation_check(struct.pack('<4sl', importlib.util.MAGIC_NUMBER, + 1)) def test_magic_number(self): # Test a change in mtime leads to a new .pyc. @@ -97,14 +99,14 @@ class CompileallTests(unittest.TestCase): # interpreter's creates the correct file names optimize = 1 if __debug__ else 0 compileall.compile_dir(self.directory, quiet=True, optimize=optimize) - cached = imp.cache_from_source(self.source_path, - debug_override=not optimize) + cached = importlib.util.cache_from_source(self.source_path, + debug_override=not optimize) self.assertTrue(os.path.isfile(cached)) - cached2 = imp.cache_from_source(self.source_path2, - debug_override=not optimize) + cached2 = importlib.util.cache_from_source(self.source_path2, + debug_override=not optimize) self.assertTrue(os.path.isfile(cached2)) - cached3 = imp.cache_from_source(self.source_path3, - debug_override=not optimize) + cached3 = importlib.util.cache_from_source(self.source_path3, + debug_override=not optimize) self.assertTrue(os.path.isfile(cached3)) @@ -152,10 +154,12 @@ class CommandLineTests(unittest.TestCase): return rc, out, err def assertCompiled(self, fn): - self.assertTrue(os.path.exists(imp.cache_from_source(fn))) + path = importlib.util.cache_from_source(fn) + self.assertTrue(os.path.exists(path)) def assertNotCompiled(self, fn): - self.assertFalse(os.path.exists(imp.cache_from_source(fn))) + path = importlib.util.cache_from_source(fn) + self.assertFalse(os.path.exists(path)) def setUp(self): self.addCleanup(self._cleanup) @@ -190,8 +194,8 @@ class CommandLineTests(unittest.TestCase): ['-m', 'compileall', '-q', self.pkgdir])) # Verify the __pycache__ directory contents. self.assertTrue(os.path.exists(self.pkgdir_cachedir)) - expected = sorted(base.format(imp.get_tag(), ext) for base in - ('__init__.{}.{}', 'bar.{}.{}')) + expected = sorted(base.format(sys.implementation.cache_tag, ext) + for base in ('__init__.{}.{}', 'bar.{}.{}')) self.assertEqual(sorted(os.listdir(self.pkgdir_cachedir)), expected) # Make sure there are no .pyc files in the source directory. self.assertFalse([fn for fn in os.listdir(self.pkgdir) @@ -224,7 +228,7 @@ class CommandLineTests(unittest.TestCase): def test_force(self): self.assertRunOK('-q', self.pkgdir) - pycpath = imp.cache_from_source(self.barfn) + pycpath = importlib.util.cache_from_source(self.barfn) # set atime/mtime backward to avoid file timestamp resolution issues os.utime(pycpath, (time.time()-60,)*2) mtime = os.stat(pycpath).st_mtime @@ -288,10 +292,10 @@ class CommandLineTests(unittest.TestCase): bazfn = script_helper.make_script(self.pkgdir, 'baz', 'raise Exception') self.assertRunOK('-q', '-d', 'dinsdale', self.pkgdir) fn = script_helper.make_script(self.pkgdir, 'bing', 'import baz') - pyc = imp.cache_from_source(bazfn) + pyc = importlib.util.cache_from_source(bazfn) os.rename(pyc, os.path.join(self.pkgdir, 'baz.pyc')) os.remove(bazfn) - rc, out, err = script_helper.assert_python_failure(fn) + rc, out, err = script_helper.assert_python_failure(fn, __isolated=False) self.assertRegex(err, b'File "dinsdale') def test_include_bad_file(self): @@ -299,7 +303,7 @@ class CommandLineTests(unittest.TestCase): '-i', os.path.join(self.directory, 'nosuchfile'), self.pkgdir) self.assertRegex(out, b'rror.*nosuchfile') self.assertNotRegex(err, b'Traceback') - self.assertFalse(os.path.exists(imp.cache_from_source( + self.assertFalse(os.path.exists(importlib.util.cache_from_source( self.pkgdir_cachedir))) def test_include_file_with_arg(self): @@ -356,13 +360,5 @@ class CommandLineTests(unittest.TestCase): self.assertRegex(out, b"Can't list 'badfilename'") -def test_main(): - support.run_unittest( - CommandLineTests, - CompileallTests, - EncodingTest, - ) - - if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index 1bf409798b..f80d7ac522 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -220,6 +220,8 @@ class ComplexTest(unittest.TestCase): self.assertRaises(TypeError, complex, OS(None)) self.assertRaises(TypeError, complex, NS(None)) self.assertRaises(TypeError, complex, {}) + self.assertRaises(TypeError, complex, NS(1.5)) + self.assertRaises(TypeError, complex, NS(1)) self.assertAlmostEqual(complex("1+10j"), 1+10j) self.assertAlmostEqual(complex(10), 10+0j) diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 6ae450df06..4ad33091b0 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -15,6 +15,7 @@ import sys import threading import time import unittest +import weakref from concurrent import futures from concurrent.futures._base import ( @@ -52,6 +53,11 @@ def sleep_and_print(t, msg): sys.stdout.flush() +class MyObject(object): + def my_method(self): + pass + + class ExecutorMixin: worker_count = 5 @@ -396,6 +402,22 @@ class ExecutorTest(unittest.TestCase): self.executor.map(str, [2] * (self.worker_count + 1)) self.executor.shutdown() + @test.support.cpython_only + def test_no_stale_references(self): + # Issue #16284: check that the executors don't unnecessarily hang onto + # references. + my_object = MyObject() + my_object_collected = threading.Event() + my_object_callback = weakref.ref( + my_object, lambda obj: my_object_collected.set()) + # Deliberately discarding the future. + self.executor.submit(my_object.my_method) + del my_object + + collected = my_object_collected.wait(timeout=5.0) + self.assertTrue(collected, + "Stale reference not collected within timeout.") + class ThreadPoolExecutorTest(ThreadPoolMixin, ExecutorTest): def test_map_submits_without_iteration(self): diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 9e45f70f85..48f8fa9637 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -1,5 +1,6 @@ """Unit tests for contextlib.py, and other context managers.""" +import io import sys import tempfile import unittest @@ -631,10 +632,36 @@ class TestExitStack(unittest.TestCase): stack.push(cm) self.assertIs(stack._exit_callbacks[-1], cm) +class TestIgnore(unittest.TestCase): -# This is needed to make the test actually run under regrtest.py! -def test_main(): - support.run_unittest(__name__) + def test_no_exception(self): + + with ignore(ValueError): + self.assertEqual(pow(2, 5), 32) + + def test_exact_exception(self): + + with ignore(TypeError): + len(5) + + def test_multiple_exception_args(self): + + with ignore(ZeroDivisionError, TypeError): + len(5) + + def test_exception_hierarchy(self): + + with ignore(LookupError): + 'Hello'[50] + +class TestRedirectStdout(unittest.TestCase): + + def test_redirect_to_string_io(self): + f = io.StringIO() + with redirect_stdout(f): + help(pow) + s = f.getvalue() + self.assertIn('pow', s) if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 56766682b3..c3eb7faf8f 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -6,9 +6,11 @@ from test.support import run_unittest, TESTFN, unlink # rip off all interesting stuff from test_profile import cProfile from test.test_profile import ProfileTest, regenerate_expected_output +from test.profilee import testfunc class CProfileTest(ProfileTest): profilerclass = cProfile.Profile + profilermodule = cProfile expected_max_output = "{built-in method max}" def get_expected_output(self): diff --git a/Lib/test/test_crypt.py b/Lib/test/test_crypt.py index cfb7341e20..624d702f99 100644 --- a/Lib/test/test_crypt.py +++ b/Lib/test/test_crypt.py @@ -1,11 +1,7 @@ from test import support import unittest -def setUpModule(): - # this import will raise unittest.SkipTest if _crypt doesn't exist, - # so it has to be done in setUpModule for test discovery to work - global crypt - crypt = support.import_module('crypt') +crypt = support.import_module('crypt') class CryptTestCase(unittest.TestCase): diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 5e285865a7..559e51f47b 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -136,12 +136,12 @@ class Test_Csv(unittest.TestCase): return 10; def __getitem__(self, i): if i > 2: - raise IOError - self.assertRaises(IOError, self._write_test, BadList(), '') + raise OSError + self.assertRaises(OSError, self._write_test, BadList(), '') class BadItem: def __str__(self): - raise IOError - self.assertRaises(IOError, self._write_test, [BadItem()], '') + raise OSError + self.assertRaises(OSError, self._write_test, [BadItem()], '') def test_write_bigfield(self): # This exercises the buffer realloc functionality @@ -186,9 +186,9 @@ class Test_Csv(unittest.TestCase): def test_writerows(self): class BrokenFile: def write(self, buf): - raise IOError + raise OSError writer = csv.writer(BrokenFile()) - self.assertRaises(IOError, writer.writerows, [['a']]) + self.assertRaises(OSError, writer.writerows, [['a']]) with TemporaryFile("w+", newline='') as fileobj: writer = csv.writer(fileobj) @@ -308,6 +308,15 @@ class Test_Csv(unittest.TestCase): for i, row in enumerate(csv.reader(fileobj)): self.assertEqual(row, rows[i]) + def test_roundtrip_escaped_unquoted_newlines(self): + with TemporaryFile("w+", newline='') as fileobj: + writer = csv.writer(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\") + rows = [['a\nb','b'],['c','x\r\nd']] + writer.writerows(rows) + fileobj.seek(0) + for i, row in enumerate(csv.reader(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\")): + self.assertEqual(row,rows[i]) + class TestDialectRegistry(unittest.TestCase): def test_registry_badargs(self): self.assertRaises(TypeError, csv.list_dialects, None) diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index 243a0af103..1c57770242 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -62,7 +62,7 @@ class AnyDBMTestCase: return keys def test_error(self): - self.assertTrue(issubclass(self.module.error, IOError)) + self.assertTrue(issubclass(self.module.error, OSError)) def test_anydbm_not_existing(self): self.assertRaises(dbm.error, dbm.open, _fname) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 71b93a661b..ac8a7bebc1 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -290,7 +290,6 @@ class IBMTestCases(unittest.TestCase): global skip_expected if skip_expected: raise unittest.SkipTest - return with open(file) as f: for line in f: line = line.replace('\r\n', '').replace('\n', '') diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index a8487d2d95..7bff1d2798 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -543,8 +543,8 @@ class TestBasic(unittest.TestCase): check = self.check_sizeof check(deque(), basesize + blocksize) check(deque('a'), basesize + blocksize) - check(deque('a' * (BLOCKLEN // 2)), basesize + blocksize) - check(deque('a' * (BLOCKLEN // 2 + 1)), basesize + 2 * blocksize) + check(deque('a' * (BLOCKLEN - 1)), basesize + blocksize) + check(deque('a' * BLOCKLEN), basesize + 2 * blocksize) check(deque('a' * (42 * BLOCKLEN)), basesize + 43 * blocksize) class TestVariousIteratorArgs(unittest.TestCase): diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 3776389ebb..f0d6d1c4ef 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1789,6 +1789,7 @@ order (MRO) for bases """ ("__trunc__", int, zero, set(), {}), ("__ceil__", math.ceil, zero, set(), {}), ("__dir__", dir, empty_seq, set(), {}), + ("__round__", round, zero, set(), {}), ] class Checker(object): @@ -2249,7 +2250,9 @@ order (MRO) for bases """ minstance = M("m") minstance.b = 2 minstance.a = 1 - names = [x for x in dir(minstance) if x not in ["__name__", "__doc__"]] + default_attributes = ['__name__', '__doc__', '__package__', + '__loader__'] + names = [x for x in dir(minstance) if x not in default_attributes] self.assertEqual(names, ['a', 'b']) class M2(M): @@ -3733,18 +3736,8 @@ order (MRO) for bases """ # bug). del c - # If that didn't blow up, it's also interesting to see whether clearing - # the last container slot works: that will attempt to delete c again, - # which will cause c to get appended back to the container again - # "during" the del. (On non-CPython implementations, however, __del__ - # is typically not called again.) support.gc_collect() self.assertEqual(len(C.container), 1) - del C.container[-1] - if support.check_impl_detail(): - support.gc_collect() - self.assertEqual(len(C.container), 1) - self.assertEqual(C.container[-1].attr, 42) # Make c mortal again, so that the test framework with -l doesn't report # it as a leak. diff --git a/Lib/test/test_devpoll.py b/Lib/test/test_devpoll.py index bef4e1846b..40ebeee597 100644 --- a/Lib/test/test_devpoll.py +++ b/Lib/test/test_devpoll.py @@ -2,13 +2,17 @@ # Initial tests are copied as is from "test_poll.py" -import os, select, random, unittest, sys +import os +import random +import select +import sys +import unittest from test.support import TESTFN, run_unittest try: select.devpoll except AttributeError: - raise unittest.SkipTest("select.devpoll not defined -- skipping test_devpoll") + raise unittest.SkipTest("select.devpoll not defined") def find_ready_matching(ready, flag): @@ -87,6 +91,36 @@ class DevPollTests(unittest.TestCase): self.assertRaises(OverflowError, pollster.poll, 1 << 63) self.assertRaises(OverflowError, pollster.poll, 1 << 64) + def test_close(self): + open_file = open(__file__, "rb") + self.addCleanup(open_file.close) + fd = open_file.fileno() + devpoll = select.devpoll() + + # test fileno() method and closed attribute + self.assertIsInstance(devpoll.fileno(), int) + self.assertFalse(devpoll.closed) + + # test close() + devpoll.close() + self.assertTrue(devpoll.closed) + self.assertRaises(ValueError, devpoll.fileno) + + # close() can be called more than once + devpoll.close() + + # operations must fail with ValueError("I/O operation on closed ...") + self.assertRaises(ValueError, devpoll.modify, fd, select.POLLIN) + self.assertRaises(ValueError, devpoll.poll) + self.assertRaises(ValueError, devpoll.register, fd, fd, select.POLLIN) + self.assertRaises(ValueError, devpoll.unregister, fd) + + def test_fd_non_inheritable(self): + devpoll = select.devpoll() + self.addCleanup(devpoll.close) + self.assertEqual(os.get_inheritable(devpoll.fileno()), False) + + def test_main(): run_unittest(DevPollTests) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index d1355bbb96..8610c5dcd8 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1,11 +1,13 @@ # Minimal tests for dis module from test.support import run_unittest, captured_stdout +from test.bytecode_helper import BytecodeTestCase import difflib import unittest import sys import dis import io +import types class _C: def __init__(self, x): @@ -22,12 +24,12 @@ dis_c_instance_method = """\ """ % (_C.__init__.__code__.co_firstlineno + 1,) dis_c_instance_method_bytes = """\ - 0 LOAD_FAST 1 (1) - 3 LOAD_CONST 1 (1) - 6 COMPARE_OP 2 (==) - 9 LOAD_FAST 0 (0) - 12 STORE_ATTR 0 (0) - 15 LOAD_CONST 0 (0) + 0 LOAD_FAST 1 (1) + 3 LOAD_CONST 1 (1) + 6 COMPARE_OP 2 (==) + 9 LOAD_FAST 0 (0) + 12 STORE_ATTR 0 (0) + 15 LOAD_CONST 0 (0) 18 RETURN_VALUE """ @@ -48,11 +50,11 @@ dis_f = """\ dis_f_co_code = """\ - 0 LOAD_GLOBAL 0 (0) - 3 LOAD_FAST 0 (0) - 6 CALL_FUNCTION 1 (1 positional, 0 keyword pair) + 0 LOAD_GLOBAL 0 (0) + 3 LOAD_FAST 0 (0) + 6 CALL_FUNCTION 1 (1 positional, 0 keyword pair) 9 POP_TOP - 10 LOAD_CONST 1 (1) + 10 LOAD_CONST 1 (1) 13 RETURN_VALUE """ @@ -294,29 +296,16 @@ Filename: (.*) Argument count: 1 Kw-only arguments: 0 Number of locals: 1 -Stack size: 4 +Stack size: 3 Flags: OPTIMIZED, NEWLOCALS, NOFREE Constants: 0: %r - 1: '__func__' - 2: '__code__' - 3: '<code_info>' - 4: 'co_code' - 5: "don't know how to disassemble %%s objects" -%sNames: - 0: hasattr - 1: __func__ - 2: __code__ - 3: isinstance - 4: str - 5: _try_compile - 6: _format_code_info - 7: TypeError - 8: type - 9: __name__ +Names: + 0: _format_code_info + 1: _get_code_object Variable names: - 0: x""" % (('Formatted details of methods, functions, or code.', ' 6: None\n') - if sys.flags.optimize < 2 else (None, '')) + 0: x""" % (('Formatted details of methods, functions, or code.',) + if sys.flags.optimize < 2 else (None,)) @staticmethod def tricky(x, y, z=True, *args, c, d, e=[], **kwds): @@ -380,7 +369,7 @@ Free variables: code_info_expr_str = """\ Name: <module> -Filename: <code_info> +Filename: <disassembly> Argument count: 0 Kw-only arguments: 0 Number of locals: 0 @@ -393,7 +382,7 @@ Names: code_info_simple_stmt_str = """\ Name: <module> -Filename: <code_info> +Filename: <disassembly> Argument count: 0 Kw-only arguments: 0 Number of locals: 0 @@ -407,7 +396,7 @@ Names: code_info_compound_stmt_str = """\ Name: <module> -Filename: <code_info> +Filename: <disassembly> Argument count: 0 Kw-only arguments: 0 Number of locals: 0 @@ -441,6 +430,9 @@ class CodeInfoTests(unittest.TestCase): with captured_stdout() as output: dis.show_code(x) self.assertRegex(output.getvalue(), expected+"\n") + output = io.StringIO() + dis.show_code(x, file=output) + self.assertRegex(output.getvalue(), expected) def test_code_info_object(self): self.assertRaises(TypeError, dis.code_info, object()) @@ -449,8 +441,289 @@ class CodeInfoTests(unittest.TestCase): self.assertEqual(dis.pretty_flags(0), '0x0') +# Fodder for instruction introspection tests +# Editing any of these may require recalculating the expected output +def outer(a=1, b=2): + def f(c=3, d=4): + def inner(e=5, f=6): + print(a, b, c, d, e, f) + print(a, b, c, d) + return inner + print(a, b, '', 1, [], {}, "Hello world!") + return f + +def jumpy(): + # This won't actually run (but that's OK, we only disassemble it) + for i in range(10): + print(i) + if i < 4: + continue + if i > 6: + break + else: + print("I can haz else clause?") + while i: + print(i) + i -= 1 + if i > 6: + continue + if i < 4: + break + else: + print("Who let lolcatz into this test suite?") + try: + 1 / 0 + except ZeroDivisionError: + print("Here we go, here we go, here we go...") + else: + with i as dodgy: + print("Never reach this") + finally: + print("OK, now we're done") + +# End fodder for opinfo generation tests +expected_outer_offset = 1 - outer.__code__.co_firstlineno +expected_jumpy_offset = 1 - jumpy.__code__.co_firstlineno +code_object_f = outer.__code__.co_consts[3] +code_object_inner = code_object_f.co_consts[3] + +# The following lines are useful to regenerate the expected results after +# either the fodder is modified or the bytecode generation changes +# After regeneration, update the references to code_object_f and +# code_object_inner before rerunning the tests + +#_instructions = dis.get_instructions(outer, line_offset=expected_outer_offset) +#print('expected_opinfo_outer = [\n ', + #',\n '.join(map(str, _instructions)), ',\n]', sep='') +#_instructions = dis.get_instructions(outer(), line_offset=expected_outer_offset) +#print('expected_opinfo_f = [\n ', + #',\n '.join(map(str, _instructions)), ',\n]', sep='') +#_instructions = dis.get_instructions(outer()(), line_offset=expected_outer_offset) +#print('expected_opinfo_inner = [\n ', + #',\n '.join(map(str, _instructions)), ',\n]', sep='') +#_instructions = dis.get_instructions(jumpy, line_offset=expected_jumpy_offset) +#print('expected_opinfo_jumpy = [\n ', + #',\n '.join(map(str, _instructions)), ',\n]', sep='') + + +Instruction = dis.Instruction +expected_opinfo_outer = [ + Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=3, argrepr='3', offset=0, starts_line=2, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=3, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='a', argrepr='a', offset=6, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='b', argrepr='b', offset=9, starts_line=None, is_jump_target=False), + Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=12, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=15, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=18, starts_line=None, is_jump_target=False), + Instruction(opname='MAKE_CLOSURE', opcode=134, arg=2, argval=2, argrepr='', offset=21, starts_line=None, is_jump_target=False), + Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=24, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=27, starts_line=7, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=30, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='b', argrepr='b', offset=33, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval='', argrepr="''", offset=36, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval=1, argrepr='1', offset=39, starts_line=None, is_jump_target=False), + Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=42, starts_line=None, is_jump_target=False), + Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=45, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval='Hello world!', argrepr="'Hello world!'", offset=48, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=7, argval=7, argrepr='7 positional, 0 keyword pair', offset=51, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=55, starts_line=8, is_jump_target=False), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False), +] + +expected_opinfo_f = [ + Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=5, argrepr='5', offset=0, starts_line=3, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=6, argrepr='6', offset=3, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CLOSURE', opcode=135, arg=2, argval='a', argrepr='a', offset=6, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='b', argrepr='b', offset=9, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='c', argrepr='c', offset=12, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='d', argrepr='d', offset=15, starts_line=None, is_jump_target=False), + Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=18, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=21, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=24, starts_line=None, is_jump_target=False), + Instruction(opname='MAKE_CLOSURE', opcode=134, arg=2, argval=2, argrepr='', offset=27, starts_line=None, is_jump_target=False), + Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=30, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=33, starts_line=5, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='a', argrepr='a', offset=36, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='b', argrepr='b', offset=39, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='c', argrepr='c', offset=42, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='d', argrepr='d', offset=45, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=4, argval=4, argrepr='4 positional, 0 keyword pair', offset=48, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=51, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=52, starts_line=6, is_jump_target=False), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=55, starts_line=None, is_jump_target=False), +] + +expected_opinfo_inner = [ + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=0, starts_line=4, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=3, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='b', argrepr='b', offset=6, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='c', argrepr='c', offset=9, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='d', argrepr='d', offset=12, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=15, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=18, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=6, argval=6, argrepr='6 positional, 0 keyword pair', offset=21, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=24, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=25, starts_line=None, is_jump_target=False), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=28, starts_line=None, is_jump_target=False), +] + +expected_opinfo_jumpy = [ + Instruction(opname='SETUP_LOOP', opcode=120, arg=74, argval=77, argrepr='to 77', offset=0, starts_line=3, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='range', argrepr='range', offset=3, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=6, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=9, starts_line=None, is_jump_target=False), + Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=12, starts_line=None, is_jump_target=False), + Instruction(opname='FOR_ITER', opcode=93, arg=50, argval=66, argrepr='to 66', offset=13, starts_line=None, is_jump_target=True), + Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=16, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=19, starts_line=4, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=22, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=25, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=28, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=29, starts_line=5, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=32, starts_line=None, is_jump_target=False), + Instruction(opname='COMPARE_OP', opcode=107, arg=0, argval='<', argrepr='<', offset=35, starts_line=None, is_jump_target=False), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=47, argval=47, argrepr='', offset=38, starts_line=None, is_jump_target=False), + Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=13, argval=13, argrepr='', offset=41, starts_line=6, is_jump_target=False), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=0, argval=47, argrepr='to 47', offset=44, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=47, starts_line=7, is_jump_target=True), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=50, starts_line=None, is_jump_target=False), + Instruction(opname='COMPARE_OP', opcode=107, arg=4, argval='>', argrepr='>', offset=53, starts_line=None, is_jump_target=False), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=13, argval=13, argrepr='', offset=56, starts_line=None, is_jump_target=False), + Instruction(opname='BREAK_LOOP', opcode=80, arg=None, argval=None, argrepr='', offset=59, starts_line=8, is_jump_target=False), + Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=13, argval=13, argrepr='', offset=60, starts_line=None, is_jump_target=False), + Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=13, argval=13, argrepr='', offset=63, starts_line=None, is_jump_target=False), + Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=66, starts_line=None, is_jump_target=True), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=67, starts_line=10, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=70, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=73, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=76, starts_line=None, is_jump_target=False), + Instruction(opname='SETUP_LOOP', opcode=120, arg=74, argval=154, argrepr='to 154', offset=77, starts_line=11, is_jump_target=True), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=80, starts_line=None, is_jump_target=True), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=143, argval=143, argrepr='', offset=83, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=86, starts_line=12, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=89, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=92, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=95, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=96, starts_line=13, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=99, starts_line=None, is_jump_target=False), + Instruction(opname='INPLACE_SUBTRACT', opcode=56, arg=None, argval=None, argrepr='', offset=102, starts_line=None, is_jump_target=False), + Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=103, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=106, starts_line=14, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=109, starts_line=None, is_jump_target=False), + Instruction(opname='COMPARE_OP', opcode=107, arg=4, argval='>', argrepr='>', offset=112, starts_line=None, is_jump_target=False), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=124, argval=124, argrepr='', offset=115, starts_line=None, is_jump_target=False), + Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=80, argval=80, argrepr='', offset=118, starts_line=15, is_jump_target=False), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=0, argval=124, argrepr='to 124', offset=121, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=124, starts_line=16, is_jump_target=True), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=127, starts_line=None, is_jump_target=False), + Instruction(opname='COMPARE_OP', opcode=107, arg=0, argval='<', argrepr='<', offset=130, starts_line=None, is_jump_target=False), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=80, argval=80, argrepr='', offset=133, starts_line=None, is_jump_target=False), + Instruction(opname='BREAK_LOOP', opcode=80, arg=None, argval=None, argrepr='', offset=136, starts_line=17, is_jump_target=False), + Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=80, argval=80, argrepr='', offset=137, starts_line=None, is_jump_target=False), + Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=80, argval=80, argrepr='', offset=140, starts_line=None, is_jump_target=False), + Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=143, starts_line=None, is_jump_target=True), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=144, starts_line=19, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=147, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=150, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=153, starts_line=None, is_jump_target=False), + Instruction(opname='SETUP_FINALLY', opcode=122, arg=72, argval=229, argrepr='to 229', offset=154, starts_line=20, is_jump_target=True), + Instruction(opname='SETUP_EXCEPT', opcode=121, arg=12, argval=172, argrepr='to 172', offset=157, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=160, starts_line=21, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=163, starts_line=None, is_jump_target=False), + Instruction(opname='BINARY_TRUE_DIVIDE', opcode=27, arg=None, argval=None, argrepr='', offset=166, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=167, starts_line=None, is_jump_target=False), + Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=168, starts_line=None, is_jump_target=False), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=28, argval=200, argrepr='to 200', offset=169, starts_line=None, is_jump_target=False), + Instruction(opname='DUP_TOP', opcode=4, arg=None, argval=None, argrepr='', offset=172, starts_line=22, is_jump_target=True), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=2, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=173, starts_line=None, is_jump_target=False), + Instruction(opname='COMPARE_OP', opcode=107, arg=10, argval='exception match', argrepr='exception match', offset=176, starts_line=None, is_jump_target=False), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=199, argval=199, argrepr='', offset=179, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=182, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=183, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=184, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=185, starts_line=23, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=188, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=191, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=194, starts_line=None, is_jump_target=False), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=195, starts_line=None, is_jump_target=False), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=26, argval=225, argrepr='to 225', offset=196, starts_line=None, is_jump_target=False), + Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=199, starts_line=None, is_jump_target=True), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=200, starts_line=25, is_jump_target=True), + Instruction(opname='SETUP_WITH', opcode=143, arg=17, argval=223, argrepr='to 223', offset=203, starts_line=None, is_jump_target=False), + Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=206, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=209, starts_line=26, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Never reach this', argrepr="'Never reach this'", offset=212, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=215, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=218, starts_line=None, is_jump_target=False), + Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=219, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=220, starts_line=None, is_jump_target=False), + Instruction(opname='WITH_CLEANUP', opcode=81, arg=None, argval=None, argrepr='', offset=223, starts_line=None, is_jump_target=True), + Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=224, starts_line=None, is_jump_target=False), + Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=225, starts_line=None, is_jump_target=True), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=226, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=229, starts_line=28, is_jump_target=True), + Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=232, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=235, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=238, starts_line=None, is_jump_target=False), + Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=239, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=240, starts_line=None, is_jump_target=False), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=243, starts_line=None, is_jump_target=False), +] + +class InstructionTests(BytecodeTestCase): + def test_outer(self): + self.assertBytecodeExactlyMatches(outer, expected_opinfo_outer, + line_offset=expected_outer_offset) + + def test_nested(self): + with captured_stdout(): + f = outer() + self.assertBytecodeExactlyMatches(f, expected_opinfo_f, + line_offset=expected_outer_offset) + + def test_doubly_nested(self): + with captured_stdout(): + inner = outer()() + self.assertBytecodeExactlyMatches(inner, expected_opinfo_inner, + line_offset=expected_outer_offset) + + def test_jumpy(self): + self.assertBytecodeExactlyMatches(jumpy, expected_opinfo_jumpy, + line_offset=expected_jumpy_offset) + +class BytecodeTests(unittest.TestCase): + def test_instantiation(self): + # Test with function, method, code string and code object + for obj in [_f, _C(1).__init__, "a=1", _f.__code__]: + b = dis.Bytecode(obj) + self.assertIsInstance(b.codeobj, types.CodeType) + + self.assertRaises(TypeError, dis.Bytecode, object()) + + def test_iteration(self): + b = dis.Bytecode(_f) + for instr in b: + self.assertIsInstance(instr, dis.Instruction) + + assert len(list(b)) > 0 # Iterating should yield at least 1 instruction + + def test_info(self): + self.maxDiff = 1000 + for x, expected in CodeInfoTests.test_pairs: + b = dis.Bytecode(x) + self.assertRegex(b.info(), expected) + + def test_display_code(self): + b = dis.Bytecode(_f) + output = io.StringIO() + b.display_code(file=output) + result = [line.rstrip() for line in output.getvalue().splitlines()] + self.assertEqual(result, dis_f.splitlines()) + + def test_main(): - run_unittest(DisTests, CodeInfoTests) + run_unittest(DisTests, CodeInfoTests, InstructionTests, BytecodeTests) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 8f8c7c7e82..d4ff049fca 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -1409,8 +1409,40 @@ However, output from `report_start` is not suppressed: 2 TestResults(failed=3, attempted=5) -For the purposes of REPORT_ONLY_FIRST_FAILURE, unexpected exceptions -count as failures: +The FAIL_FAST flag causes the runner to exit after the first failing example, +so subsequent examples are not even attempted: + + >>> flags = doctest.FAIL_FAST + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 5, in f + Failed example: + print(2) # first failure + Expected: + 200 + Got: + 2 + TestResults(failed=1, attempted=2) + +Specifying both FAIL_FAST and REPORT_ONLY_FIRST_FAILURE is equivalent to +FAIL_FAST only: + + >>> flags = doctest.FAIL_FAST | doctest.REPORT_ONLY_FIRST_FAILURE + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 5, in f + Failed example: + print(2) # first failure + Expected: + 200 + Got: + 2 + TestResults(failed=1, attempted=2) + +For the purposes of both REPORT_ONLY_FIRST_FAILURE and FAIL_FAST, unexpected +exceptions count as failures: >>> def f(x): ... r''' @@ -1437,6 +1469,17 @@ count as failures: ... ValueError: 2 TestResults(failed=3, attempted=5) + >>> flags = doctest.FAIL_FAST + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 5, in f + Failed example: + raise ValueError(2) # first failure + Exception raised: + ... + ValueError: 2 + TestResults(failed=1, attempted=2) New option flags can also be registered, via register_optionflag(). Here we reach into doctest's internals a bit. @@ -2553,6 +2596,240 @@ Check doctest with a non-ascii filename: TestResults(failed=1, attempted=1) """ +def test_CLI(): r""" +The doctest module can be used to run doctests against an arbitrary file. +These tests test this CLI functionality. + +We'll use the support module's script_helpers for this, and write a test files +to a temp dir to run the command against. Due to a current limitation in +script_helpers, though, we need a little utility function to turn the returned +output into something we can doctest against: + + >>> def normalize(s): + ... return '\n'.join(s.decode().splitlines()) + +Note: we also pass TERM='' to all the assert_python calls to avoid a bug +in the readline library that is triggered in these tests because we are +running them in a new python process. See: + + http://lists.gnu.org/archive/html/bug-readline/2013-06/msg00000.html + +With those preliminaries out of the way, we'll start with a file with two +simple tests and no errors. We'll run both the unadorned doctest command, and +the verbose version, and then check the output: + + >>> from test import script_helper + >>> with script_helper.temp_dir() as tmpdir: + ... fn = os.path.join(tmpdir, 'myfile.doc') + ... with open(fn, 'w') as f: + ... _ = f.write('This is a very simple test file.\n') + ... _ = f.write(' >>> 1 + 1\n') + ... _ = f.write(' 2\n') + ... _ = f.write(' >>> "a"\n') + ... _ = f.write(" 'a'\n") + ... _ = f.write('\n') + ... _ = f.write('And that is it.\n') + ... rc1, out1, err1 = script_helper.assert_python_ok( + ... '-m', 'doctest', fn, TERM='') + ... rc2, out2, err2 = script_helper.assert_python_ok( + ... '-m', 'doctest', '-v', fn, TERM='') + +With no arguments and passing tests, we should get no output: + + >>> rc1, out1, err1 + (0, b'', b'') + +With the verbose flag, we should see the test output, but no error output: + + >>> rc2, err2 + (0, b'') + >>> print(normalize(out2)) + Trying: + 1 + 1 + Expecting: + 2 + ok + Trying: + "a" + Expecting: + 'a' + ok + 1 items passed all tests: + 2 tests in myfile.doc + 2 tests in 1 items. + 2 passed and 0 failed. + Test passed. + +Now we'll write a couple files, one with three tests, the other a python module +with two tests, both of the files having "errors" in the tests that can be made +non-errors by applying the appropriate doctest options to the run (ELLIPSIS in +the first file, NORMALIZE_WHITESPACE in the second). This combination will +allow to thoroughly test the -f and -o flags, as well as the doctest command's +ability to process more than one file on the command line and, since the second +file ends in '.py', its handling of python module files (as opposed to straight +text files). + + >>> from test import script_helper + >>> with script_helper.temp_dir() as tmpdir: + ... fn = os.path.join(tmpdir, 'myfile.doc') + ... with open(fn, 'w') as f: + ... _ = f.write('This is another simple test file.\n') + ... _ = f.write(' >>> 1 + 1\n') + ... _ = f.write(' 2\n') + ... _ = f.write(' >>> "abcdef"\n') + ... _ = f.write(" 'a...f'\n") + ... _ = f.write(' >>> "ajkml"\n') + ... _ = f.write(" 'a...l'\n") + ... _ = f.write('\n') + ... _ = f.write('And that is it.\n') + ... fn2 = os.path.join(tmpdir, 'myfile2.py') + ... with open(fn2, 'w') as f: + ... _ = f.write('def test_func():\n') + ... _ = f.write(' \"\"\"\n') + ... _ = f.write(' This is simple python test function.\n') + ... _ = f.write(' >>> 1 + 1\n') + ... _ = f.write(' 2\n') + ... _ = f.write(' >>> "abc def"\n') + ... _ = f.write(" 'abc def'\n") + ... _ = f.write("\n") + ... _ = f.write(' \"\"\"\n') + ... import shutil + ... rc1, out1, err1 = script_helper.assert_python_failure( + ... '-m', 'doctest', fn, fn2, TERM='') + ... rc2, out2, err2 = script_helper.assert_python_ok( + ... '-m', 'doctest', '-o', 'ELLIPSIS', fn, TERM='') + ... rc3, out3, err3 = script_helper.assert_python_ok( + ... '-m', 'doctest', '-o', 'ELLIPSIS', + ... '-o', 'NORMALIZE_WHITESPACE', fn, fn2, TERM='') + ... rc4, out4, err4 = script_helper.assert_python_failure( + ... '-m', 'doctest', '-f', fn, fn2, TERM='') + ... rc5, out5, err5 = script_helper.assert_python_ok( + ... '-m', 'doctest', '-v', '-o', 'ELLIPSIS', + ... '-o', 'NORMALIZE_WHITESPACE', fn, fn2, TERM='') + +Our first test run will show the errors from the first file (doctest stops if a +file has errors). Note that doctest test-run error output appears on stdout, +not stderr: + + >>> rc1, err1 + (1, b'') + >>> print(normalize(out1)) # doctest: +ELLIPSIS + ********************************************************************** + File "...myfile.doc", line 4, in myfile.doc + Failed example: + "abcdef" + Expected: + 'a...f' + Got: + 'abcdef' + ********************************************************************** + File "...myfile.doc", line 6, in myfile.doc + Failed example: + "ajkml" + Expected: + 'a...l' + Got: + 'ajkml' + ********************************************************************** + 1 items had failures: + 2 of 3 in myfile.doc + ***Test Failed*** 2 failures. + +With -o ELLIPSIS specified, the second run, against just the first file, should +produce no errors, and with -o NORMALIZE_WHITESPACE also specified, neither +should the third, which ran against both files: + + >>> rc2, out2, err2 + (0, b'', b'') + >>> rc3, out3, err3 + (0, b'', b'') + +The fourth run uses FAIL_FAST, so we should see only one error: + + >>> rc4, err4 + (1, b'') + >>> print(normalize(out4)) # doctest: +ELLIPSIS + ********************************************************************** + File "...myfile.doc", line 4, in myfile.doc + Failed example: + "abcdef" + Expected: + 'a...f' + Got: + 'abcdef' + ********************************************************************** + 1 items had failures: + 1 of 2 in myfile.doc + ***Test Failed*** 1 failures. + +The fifth test uses verbose with the two options, so we should get verbose +success output for the tests in both files: + + >>> rc5, err5 + (0, b'') + >>> print(normalize(out5)) + Trying: + 1 + 1 + Expecting: + 2 + ok + Trying: + "abcdef" + Expecting: + 'a...f' + ok + Trying: + "ajkml" + Expecting: + 'a...l' + ok + 1 items passed all tests: + 3 tests in myfile.doc + 3 tests in 1 items. + 3 passed and 0 failed. + Test passed. + Trying: + 1 + 1 + Expecting: + 2 + ok + Trying: + "abc def" + Expecting: + 'abc def' + ok + 1 items had no tests: + myfile2 + 1 items passed all tests: + 2 tests in myfile2.test_func + 2 tests in 2 items. + 2 passed and 0 failed. + Test passed. + +We should also check some typical error cases. + +Invalid file name: + + >>> rc, out, err = script_helper.assert_python_failure( + ... '-m', 'doctest', 'nosuchfile', TERM='') + >>> rc, out + (1, b'') + >>> print(normalize(err)) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + FileNotFoundError: [Errno ...] No such file or directory: 'nosuchfile' + +Invalid doctest option: + + >>> rc, out, err = script_helper.assert_python_failure( + ... '-m', 'doctest', '-o', 'nosuchoption', TERM='') + >>> rc, out + (2, b'') + >>> print(normalize(err)) # doctest: +ELLIPSIS + usage...invalid...nosuchoption... + +""" + ###################################################################### ## Main ###################################################################### diff --git a/Lib/test/test_dynamicclassattribute.py b/Lib/test/test_dynamicclassattribute.py new file mode 100644 index 0000000000..079d6c3224 --- /dev/null +++ b/Lib/test/test_dynamicclassattribute.py @@ -0,0 +1,304 @@ +# Test case for DynamicClassAttribute +# more tests are in test_descr + +import abc +import sys +import unittest +from test.support import run_unittest +from types import DynamicClassAttribute + +class PropertyBase(Exception): + pass + +class PropertyGet(PropertyBase): + pass + +class PropertySet(PropertyBase): + pass + +class PropertyDel(PropertyBase): + pass + +class BaseClass(object): + def __init__(self): + self._spam = 5 + + @DynamicClassAttribute + def spam(self): + """BaseClass.getter""" + return self._spam + + @spam.setter + def spam(self, value): + self._spam = value + + @spam.deleter + def spam(self): + del self._spam + +class SubClass(BaseClass): + + spam = BaseClass.__dict__['spam'] + + @spam.getter + def spam(self): + """SubClass.getter""" + raise PropertyGet(self._spam) + + @spam.setter + def spam(self, value): + raise PropertySet(self._spam) + + @spam.deleter + def spam(self): + raise PropertyDel(self._spam) + +class PropertyDocBase(object): + _spam = 1 + def _get_spam(self): + return self._spam + spam = DynamicClassAttribute(_get_spam, doc="spam spam spam") + +class PropertyDocSub(PropertyDocBase): + spam = PropertyDocBase.__dict__['spam'] + @spam.getter + def spam(self): + """The decorator does not use this doc string""" + return self._spam + +class PropertySubNewGetter(BaseClass): + spam = BaseClass.__dict__['spam'] + @spam.getter + def spam(self): + """new docstring""" + return 5 + +class PropertyNewGetter(object): + @DynamicClassAttribute + def spam(self): + """original docstring""" + return 1 + @spam.getter + def spam(self): + """new docstring""" + return 8 + +class ClassWithAbstractVirtualProperty(metaclass=abc.ABCMeta): + @DynamicClassAttribute + @abc.abstractmethod + def color(): + pass + +class ClassWithPropertyAbstractVirtual(metaclass=abc.ABCMeta): + @abc.abstractmethod + @DynamicClassAttribute + def color(): + pass + +class PropertyTests(unittest.TestCase): + def test_property_decorator_baseclass(self): + # see #1620 + base = BaseClass() + self.assertEqual(base.spam, 5) + self.assertEqual(base._spam, 5) + base.spam = 10 + self.assertEqual(base.spam, 10) + self.assertEqual(base._spam, 10) + delattr(base, "spam") + self.assertTrue(not hasattr(base, "spam")) + self.assertTrue(not hasattr(base, "_spam")) + base.spam = 20 + self.assertEqual(base.spam, 20) + self.assertEqual(base._spam, 20) + + def test_property_decorator_subclass(self): + # see #1620 + sub = SubClass() + self.assertRaises(PropertyGet, getattr, sub, "spam") + self.assertRaises(PropertySet, setattr, sub, "spam", None) + self.assertRaises(PropertyDel, delattr, sub, "spam") + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_property_decorator_subclass_doc(self): + sub = SubClass() + self.assertEqual(sub.__class__.__dict__['spam'].__doc__, "SubClass.getter") + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_property_decorator_baseclass_doc(self): + base = BaseClass() + self.assertEqual(base.__class__.__dict__['spam'].__doc__, "BaseClass.getter") + + def test_property_decorator_doc(self): + base = PropertyDocBase() + sub = PropertyDocSub() + self.assertEqual(base.__class__.__dict__['spam'].__doc__, "spam spam spam") + self.assertEqual(sub.__class__.__dict__['spam'].__doc__, "spam spam spam") + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_property_getter_doc_override(self): + newgettersub = PropertySubNewGetter() + self.assertEqual(newgettersub.spam, 5) + self.assertEqual(newgettersub.__class__.__dict__['spam'].__doc__, "new docstring") + newgetter = PropertyNewGetter() + self.assertEqual(newgetter.spam, 8) + self.assertEqual(newgetter.__class__.__dict__['spam'].__doc__, "new docstring") + + def test_property___isabstractmethod__descriptor(self): + for val in (True, False, [], [1], '', '1'): + class C(object): + def foo(self): + pass + foo.__isabstractmethod__ = val + foo = DynamicClassAttribute(foo) + self.assertIs(C.__dict__['foo'].__isabstractmethod__, bool(val)) + + # check that the DynamicClassAttribute's __isabstractmethod__ descriptor does the + # right thing when presented with a value that fails truth testing: + class NotBool(object): + def __nonzero__(self): + raise ValueError() + __len__ = __nonzero__ + with self.assertRaises(ValueError): + class C(object): + def foo(self): + pass + foo.__isabstractmethod__ = NotBool() + foo = DynamicClassAttribute(foo) + + def test_abstract_virtual(self): + self.assertRaises(TypeError, ClassWithAbstractVirtualProperty) + self.assertRaises(TypeError, ClassWithPropertyAbstractVirtual) + class APV(ClassWithPropertyAbstractVirtual): + pass + self.assertRaises(TypeError, APV) + class AVP(ClassWithAbstractVirtualProperty): + pass + self.assertRaises(TypeError, AVP) + class Okay1(ClassWithAbstractVirtualProperty): + @DynamicClassAttribute + def color(self): + return self._color + def __init__(self): + self._color = 'cyan' + with self.assertRaises(AttributeError): + Okay1.color + self.assertEqual(Okay1().color, 'cyan') + class Okay2(ClassWithAbstractVirtualProperty): + @DynamicClassAttribute + def color(self): + return self._color + def __init__(self): + self._color = 'magenta' + with self.assertRaises(AttributeError): + Okay2.color + self.assertEqual(Okay2().color, 'magenta') + + +# Issue 5890: subclasses of DynamicClassAttribute do not preserve method __doc__ strings +class PropertySub(DynamicClassAttribute): + """This is a subclass of DynamicClassAttribute""" + +class PropertySubSlots(DynamicClassAttribute): + """This is a subclass of DynamicClassAttribute that defines __slots__""" + __slots__ = () + +class PropertySubclassTests(unittest.TestCase): + + @unittest.skipIf(hasattr(PropertySubSlots, '__doc__'), + "__doc__ is already present, __slots__ will have no effect") + def test_slots_docstring_copy_exception(self): + try: + class Foo(object): + @PropertySubSlots + def spam(self): + """Trying to copy this docstring will raise an exception""" + return 1 + print('\n',spam.__doc__) + except AttributeError: + pass + else: + raise Exception("AttributeError not raised") + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_docstring_copy(self): + class Foo(object): + @PropertySub + def spam(self): + """spam wrapped in DynamicClassAttribute subclass""" + return 1 + self.assertEqual( + Foo.__dict__['spam'].__doc__, + "spam wrapped in DynamicClassAttribute subclass") + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_property_setter_copies_getter_docstring(self): + class Foo(object): + def __init__(self): self._spam = 1 + @PropertySub + def spam(self): + """spam wrapped in DynamicClassAttribute subclass""" + return self._spam + @spam.setter + def spam(self, value): + """this docstring is ignored""" + self._spam = value + foo = Foo() + self.assertEqual(foo.spam, 1) + foo.spam = 2 + self.assertEqual(foo.spam, 2) + self.assertEqual( + Foo.__dict__['spam'].__doc__, + "spam wrapped in DynamicClassAttribute subclass") + class FooSub(Foo): + spam = Foo.__dict__['spam'] + @spam.setter + def spam(self, value): + """another ignored docstring""" + self._spam = 'eggs' + foosub = FooSub() + self.assertEqual(foosub.spam, 1) + foosub.spam = 7 + self.assertEqual(foosub.spam, 'eggs') + self.assertEqual( + FooSub.__dict__['spam'].__doc__, + "spam wrapped in DynamicClassAttribute subclass") + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_property_new_getter_new_docstring(self): + + class Foo(object): + @PropertySub + def spam(self): + """a docstring""" + return 1 + @spam.getter + def spam(self): + """a new docstring""" + return 2 + self.assertEqual(Foo.__dict__['spam'].__doc__, "a new docstring") + class FooBase(object): + @PropertySub + def spam(self): + """a docstring""" + return 1 + class Foo2(FooBase): + spam = FooBase.__dict__['spam'] + @spam.getter + def spam(self): + """a new docstring""" + return 2 + self.assertEqual(Foo.__dict__['spam'].__doc__, "a new docstring") + + + +def test_main(): + run_unittest(PropertyTests, PropertySubclassTests) + +if __name__ == '__main__': + test_main() diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 904c06a18c..b62a889238 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -249,15 +249,42 @@ class TestMessageAPI(TestEmailBase): self.assertTrue('TO' in msg) def test_as_string(self): - eq = self.ndiffAssertEqual msg = self._msgobj('msg_01.txt') with openfile('msg_01.txt') as fp: text = fp.read() - eq(text, str(msg)) + self.assertEqual(text, str(msg)) fullrepr = msg.as_string(unixfrom=True) lines = fullrepr.split('\n') self.assertTrue(lines[0].startswith('From ')) - eq(text, NL.join(lines[1:])) + self.assertEqual(text, NL.join(lines[1:])) + + def test_as_string_policy(self): + msg = self._msgobj('msg_01.txt') + newpolicy = msg.policy.clone(linesep='\r\n') + fullrepr = msg.as_string(policy=newpolicy) + s = StringIO() + g = Generator(s, policy=newpolicy) + g.flatten(msg) + self.assertEqual(fullrepr, s.getvalue()) + + def test_as_bytes(self): + msg = self._msgobj('msg_01.txt') + with openfile('msg_01.txt') as fp: + data = fp.read().encode('ascii') + self.assertEqual(data, bytes(msg)) + fullrepr = msg.as_bytes(unixfrom=True) + lines = fullrepr.split(b'\n') + self.assertTrue(lines[0].startswith(b'From ')) + self.assertEqual(data, b'\n'.join(lines[1:])) + + def test_as_bytes_policy(self): + msg = self._msgobj('msg_01.txt') + newpolicy = msg.policy.clone(linesep='\r\n') + fullrepr = msg.as_bytes(policy=newpolicy) + s = BytesIO() + g = BytesGenerator(s,policy=newpolicy) + g.flatten(msg) + self.assertEqual(fullrepr, s.getvalue()) # test_headerregistry.TestContentTypeHeader.bad_params def test_bad_param(self): diff --git a/Lib/test/test_email/torture_test.py b/Lib/test/test_email/torture_test.py index 544b1bbb39..19cf64f0c7 100644 --- a/Lib/test/test_email/torture_test.py +++ b/Lib/test/test_email/torture_test.py @@ -27,7 +27,7 @@ def openfile(filename): # Prevent this test from running in the Python distro try: openfile('crispin-torture.txt') -except IOError: +except OSError: raise TestSkipped diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py new file mode 100644 index 0000000000..3d962f9688 --- /dev/null +++ b/Lib/test/test_enum.py @@ -0,0 +1,1197 @@ +import enum +import unittest +from collections import OrderedDict +from pickle import dumps, loads, PicklingError +from enum import Enum, IntEnum, unique + +# for pickle tests +try: + class Stooges(Enum): + LARRY = 1 + CURLY = 2 + MOE = 3 +except Exception as exc: + Stooges = exc + +try: + class IntStooges(int, Enum): + LARRY = 1 + CURLY = 2 + MOE = 3 +except Exception as exc: + IntStooges = exc + +try: + class FloatStooges(float, Enum): + LARRY = 1.39 + CURLY = 2.72 + MOE = 3.142596 +except Exception as exc: + FloatStooges = exc + +# for pickle test and subclass tests +try: + class StrEnum(str, Enum): + 'accepts only string values' + class Name(StrEnum): + BDFL = 'Guido van Rossum' + FLUFL = 'Barry Warsaw' +except Exception as exc: + Name = exc + +try: + Question = Enum('Question', 'who what when where why', module=__name__) +except Exception as exc: + Question = exc + +try: + Answer = Enum('Answer', 'him this then there because') +except Exception as exc: + Answer = exc + +# for doctests +try: + class Fruit(Enum): + tomato = 1 + banana = 2 + cherry = 3 +except Exception: + pass + + +class TestHelpers(unittest.TestCase): + # _is_descriptor, _is_sunder, _is_dunder + + def test_is_descriptor(self): + class foo: + pass + for attr in ('__get__','__set__','__delete__'): + obj = foo() + self.assertFalse(enum._is_descriptor(obj)) + setattr(obj, attr, 1) + self.assertTrue(enum._is_descriptor(obj)) + + def test_is_sunder(self): + for s in ('_a_', '_aa_'): + self.assertTrue(enum._is_sunder(s)) + + for s in ('a', 'a_', '_a', '__a', 'a__', '__a__', '_a__', '__a_', '_', + '__', '___', '____', '_____',): + self.assertFalse(enum._is_sunder(s)) + + def test_is_dunder(self): + for s in ('__a__', '__aa__'): + self.assertTrue(enum._is_dunder(s)) + for s in ('a', 'a_', '_a', '__a', 'a__', '_a_', '_a__', '__a_', '_', + '__', '___', '____', '_____',): + self.assertFalse(enum._is_dunder(s)) + + +class TestEnum(unittest.TestCase): + def setUp(self): + class Season(Enum): + SPRING = 1 + SUMMER = 2 + AUTUMN = 3 + WINTER = 4 + self.Season = Season + + class Konstants(float, Enum): + E = 2.7182818 + PI = 3.1415926 + TAU = 2 * PI + self.Konstants = Konstants + + class Grades(IntEnum): + A = 5 + B = 4 + C = 3 + D = 2 + F = 0 + self.Grades = Grades + + class Directional(str, Enum): + EAST = 'east' + WEST = 'west' + NORTH = 'north' + SOUTH = 'south' + self.Directional = Directional + + from datetime import date + class Holiday(date, Enum): + NEW_YEAR = 2013, 1, 1 + IDES_OF_MARCH = 2013, 3, 15 + self.Holiday = Holiday + + def test_dir_on_class(self): + Season = self.Season + self.assertEqual( + set(dir(Season)), + set(['__class__', '__doc__', '__members__', '__module__', + 'SPRING', 'SUMMER', 'AUTUMN', 'WINTER']), + ) + + def test_dir_on_item(self): + Season = self.Season + self.assertEqual( + set(dir(Season.WINTER)), + set(['__class__', '__doc__', '__module__', 'name', 'value']), + ) + + def test_dir_with_added_behavior(self): + class Test(Enum): + this = 'that' + these = 'those' + def wowser(self): + return ("Wowser! I'm %s!" % self.name) + self.assertEqual( + set(dir(Test)), + set(['__class__', '__doc__', '__members__', '__module__', 'this', 'these']), + ) + self.assertEqual( + set(dir(Test.this)), + set(['__class__', '__doc__', '__module__', 'name', 'value', 'wowser']), + ) + + def test_enum_in_enum_out(self): + Season = self.Season + self.assertIs(Season(Season.WINTER), Season.WINTER) + + def test_enum_value(self): + Season = self.Season + self.assertEqual(Season.SPRING.value, 1) + + def test_intenum_value(self): + self.assertEqual(IntStooges.CURLY.value, 2) + + def test_enum(self): + Season = self.Season + lst = list(Season) + self.assertEqual(len(lst), len(Season)) + self.assertEqual(len(Season), 4, Season) + self.assertEqual( + [Season.SPRING, Season.SUMMER, Season.AUTUMN, Season.WINTER], lst) + + for i, season in enumerate('SPRING SUMMER AUTUMN WINTER'.split(), 1): + e = Season(i) + self.assertEqual(e, getattr(Season, season)) + self.assertEqual(e.value, i) + self.assertNotEqual(e, i) + self.assertEqual(e.name, season) + self.assertIn(e, Season) + self.assertIs(type(e), Season) + self.assertIsInstance(e, Season) + self.assertEqual(str(e), 'Season.' + season) + self.assertEqual( + repr(e), + '<Season.{0}: {1}>'.format(season, i), + ) + + def test_value_name(self): + Season = self.Season + self.assertEqual(Season.SPRING.name, 'SPRING') + self.assertEqual(Season.SPRING.value, 1) + with self.assertRaises(AttributeError): + Season.SPRING.name = 'invierno' + with self.assertRaises(AttributeError): + Season.SPRING.value = 2 + + def test_changing_member(self): + Season = self.Season + with self.assertRaises(AttributeError): + Season.WINTER = 'really cold' + + def test_attribute_deletion(self): + class Season(Enum): + SPRING = 1 + SUMMER = 2 + AUTUMN = 3 + WINTER = 4 + + def spam(cls): + pass + + self.assertTrue(hasattr(Season, 'spam')) + del Season.spam + self.assertFalse(hasattr(Season, 'spam')) + + with self.assertRaises(AttributeError): + del Season.SPRING + with self.assertRaises(AttributeError): + del Season.DRY + with self.assertRaises(AttributeError): + del Season.SPRING.name + + def test_invalid_names(self): + with self.assertRaises(ValueError): + class Wrong(Enum): + mro = 9 + with self.assertRaises(ValueError): + class Wrong(Enum): + _create_= 11 + with self.assertRaises(ValueError): + class Wrong(Enum): + _get_mixins_ = 9 + with self.assertRaises(ValueError): + class Wrong(Enum): + _find_new_ = 1 + with self.assertRaises(ValueError): + class Wrong(Enum): + _any_name_ = 9 + + def test_contains(self): + Season = self.Season + self.assertIn(Season.AUTUMN, Season) + self.assertNotIn(3, Season) + + val = Season(3) + self.assertIn(val, Season) + + class OtherEnum(Enum): + one = 1; two = 2 + self.assertNotIn(OtherEnum.two, Season) + + def test_comparisons(self): + Season = self.Season + with self.assertRaises(TypeError): + Season.SPRING < Season.WINTER + with self.assertRaises(TypeError): + Season.SPRING > 4 + + self.assertNotEqual(Season.SPRING, 1) + + class Part(Enum): + SPRING = 1 + CLIP = 2 + BARREL = 3 + + self.assertNotEqual(Season.SPRING, Part.SPRING) + with self.assertRaises(TypeError): + Season.SPRING < Part.CLIP + + def test_enum_duplicates(self): + class Season(Enum): + SPRING = 1 + SUMMER = 2 + AUTUMN = FALL = 3 + WINTER = 4 + ANOTHER_SPRING = 1 + lst = list(Season) + self.assertEqual( + lst, + [Season.SPRING, Season.SUMMER, + Season.AUTUMN, Season.WINTER, + ]) + self.assertIs(Season.FALL, Season.AUTUMN) + self.assertEqual(Season.FALL.value, 3) + self.assertEqual(Season.AUTUMN.value, 3) + self.assertIs(Season(3), Season.AUTUMN) + self.assertIs(Season(1), Season.SPRING) + self.assertEqual(Season.FALL.name, 'AUTUMN') + self.assertEqual( + [k for k,v in Season.__members__.items() if v.name != k], + ['FALL', 'ANOTHER_SPRING'], + ) + + def test_duplicate_name(self): + with self.assertRaises(TypeError): + class Color(Enum): + red = 1 + green = 2 + blue = 3 + red = 4 + + with self.assertRaises(TypeError): + class Color(Enum): + red = 1 + green = 2 + blue = 3 + def red(self): + return 'red' + + with self.assertRaises(TypeError): + class Color(Enum): + @property + def red(self): + return 'redder' + red = 1 + green = 2 + blue = 3 + + + def test_enum_with_value_name(self): + class Huh(Enum): + name = 1 + value = 2 + self.assertEqual( + list(Huh), + [Huh.name, Huh.value], + ) + self.assertIs(type(Huh.name), Huh) + self.assertEqual(Huh.name.name, 'name') + self.assertEqual(Huh.name.value, 1) + + def test_format_enum(self): + Season = self.Season + self.assertEqual('{}'.format(Season.SPRING), + '{}'.format(str(Season.SPRING))) + self.assertEqual( '{:}'.format(Season.SPRING), + '{:}'.format(str(Season.SPRING))) + self.assertEqual('{:20}'.format(Season.SPRING), + '{:20}'.format(str(Season.SPRING))) + self.assertEqual('{:^20}'.format(Season.SPRING), + '{:^20}'.format(str(Season.SPRING))) + self.assertEqual('{:>20}'.format(Season.SPRING), + '{:>20}'.format(str(Season.SPRING))) + self.assertEqual('{:<20}'.format(Season.SPRING), + '{:<20}'.format(str(Season.SPRING))) + + def test_format_enum_custom(self): + class TestFloat(float, Enum): + one = 1.0 + two = 2.0 + def __format__(self, spec): + return 'TestFloat success!' + self.assertEqual('{}'.format(TestFloat.one), 'TestFloat success!') + + def assertFormatIsValue(self, spec, member): + self.assertEqual(spec.format(member), spec.format(member.value)) + + def test_format_enum_date(self): + Holiday = self.Holiday + self.assertFormatIsValue('{}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:^20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:>20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:<20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:%Y %m}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:%Y %m %M:00}', Holiday.IDES_OF_MARCH) + + def test_format_enum_float(self): + Konstants = self.Konstants + self.assertFormatIsValue('{}', Konstants.TAU) + self.assertFormatIsValue('{:}', Konstants.TAU) + self.assertFormatIsValue('{:20}', Konstants.TAU) + self.assertFormatIsValue('{:^20}', Konstants.TAU) + self.assertFormatIsValue('{:>20}', Konstants.TAU) + self.assertFormatIsValue('{:<20}', Konstants.TAU) + self.assertFormatIsValue('{:n}', Konstants.TAU) + self.assertFormatIsValue('{:5.2}', Konstants.TAU) + self.assertFormatIsValue('{:f}', Konstants.TAU) + + def test_format_enum_int(self): + Grades = self.Grades + self.assertFormatIsValue('{}', Grades.C) + self.assertFormatIsValue('{:}', Grades.C) + self.assertFormatIsValue('{:20}', Grades.C) + self.assertFormatIsValue('{:^20}', Grades.C) + self.assertFormatIsValue('{:>20}', Grades.C) + self.assertFormatIsValue('{:<20}', Grades.C) + self.assertFormatIsValue('{:+}', Grades.C) + self.assertFormatIsValue('{:08X}', Grades.C) + self.assertFormatIsValue('{:b}', Grades.C) + + def test_format_enum_str(self): + Directional = self.Directional + self.assertFormatIsValue('{}', Directional.WEST) + self.assertFormatIsValue('{:}', Directional.WEST) + self.assertFormatIsValue('{:20}', Directional.WEST) + self.assertFormatIsValue('{:^20}', Directional.WEST) + self.assertFormatIsValue('{:>20}', Directional.WEST) + self.assertFormatIsValue('{:<20}', Directional.WEST) + + def test_hash(self): + Season = self.Season + dates = {} + dates[Season.WINTER] = '1225' + dates[Season.SPRING] = '0315' + dates[Season.SUMMER] = '0704' + dates[Season.AUTUMN] = '1031' + self.assertEqual(dates[Season.AUTUMN], '1031') + + def test_intenum_from_scratch(self): + class phy(int, Enum): + pi = 3 + tau = 2 * pi + self.assertTrue(phy.pi < phy.tau) + + def test_intenum_inherited(self): + class IntEnum(int, Enum): + pass + class phy(IntEnum): + pi = 3 + tau = 2 * pi + self.assertTrue(phy.pi < phy.tau) + + def test_floatenum_from_scratch(self): + class phy(float, Enum): + pi = 3.1415926 + tau = 2 * pi + self.assertTrue(phy.pi < phy.tau) + + def test_floatenum_inherited(self): + class FloatEnum(float, Enum): + pass + class phy(FloatEnum): + pi = 3.1415926 + tau = 2 * pi + self.assertTrue(phy.pi < phy.tau) + + def test_strenum_from_scratch(self): + class phy(str, Enum): + pi = 'Pi' + tau = 'Tau' + self.assertTrue(phy.pi < phy.tau) + + def test_strenum_inherited(self): + class StrEnum(str, Enum): + pass + class phy(StrEnum): + pi = 'Pi' + tau = 'Tau' + self.assertTrue(phy.pi < phy.tau) + + + def test_intenum(self): + class WeekDay(IntEnum): + SUNDAY = 1 + MONDAY = 2 + TUESDAY = 3 + WEDNESDAY = 4 + THURSDAY = 5 + FRIDAY = 6 + SATURDAY = 7 + + self.assertEqual(['a', 'b', 'c'][WeekDay.MONDAY], 'c') + self.assertEqual([i for i in range(WeekDay.TUESDAY)], [0, 1, 2]) + + lst = list(WeekDay) + self.assertEqual(len(lst), len(WeekDay)) + self.assertEqual(len(WeekDay), 7) + target = 'SUNDAY MONDAY TUESDAY WEDNESDAY THURSDAY FRIDAY SATURDAY' + target = target.split() + for i, weekday in enumerate(target, 1): + e = WeekDay(i) + self.assertEqual(e, i) + self.assertEqual(int(e), i) + self.assertEqual(e.name, weekday) + self.assertIn(e, WeekDay) + self.assertEqual(lst.index(e)+1, i) + self.assertTrue(0 < e < 8) + self.assertIs(type(e), WeekDay) + self.assertIsInstance(e, int) + self.assertIsInstance(e, Enum) + + def test_intenum_duplicates(self): + class WeekDay(IntEnum): + SUNDAY = 1 + MONDAY = 2 + TUESDAY = TEUSDAY = 3 + WEDNESDAY = 4 + THURSDAY = 5 + FRIDAY = 6 + SATURDAY = 7 + self.assertIs(WeekDay.TEUSDAY, WeekDay.TUESDAY) + self.assertEqual(WeekDay(3).name, 'TUESDAY') + self.assertEqual([k for k,v in WeekDay.__members__.items() + if v.name != k], ['TEUSDAY', ]) + + def test_pickle_enum(self): + if isinstance(Stooges, Exception): + raise Stooges + self.assertIs(Stooges.CURLY, loads(dumps(Stooges.CURLY))) + self.assertIs(Stooges, loads(dumps(Stooges))) + + def test_pickle_int(self): + if isinstance(IntStooges, Exception): + raise IntStooges + self.assertIs(IntStooges.CURLY, loads(dumps(IntStooges.CURLY))) + self.assertIs(IntStooges, loads(dumps(IntStooges))) + + def test_pickle_float(self): + if isinstance(FloatStooges, Exception): + raise FloatStooges + self.assertIs(FloatStooges.CURLY, loads(dumps(FloatStooges.CURLY))) + self.assertIs(FloatStooges, loads(dumps(FloatStooges))) + + def test_pickle_enum_function(self): + if isinstance(Answer, Exception): + raise Answer + self.assertIs(Answer.him, loads(dumps(Answer.him))) + self.assertIs(Answer, loads(dumps(Answer))) + + def test_pickle_enum_function_with_module(self): + if isinstance(Question, Exception): + raise Question + self.assertIs(Question.who, loads(dumps(Question.who))) + self.assertIs(Question, loads(dumps(Question))) + + def test_exploding_pickle(self): + BadPickle = Enum('BadPickle', 'dill sweet bread-n-butter') + enum._make_class_unpicklable(BadPickle) + globals()['BadPickle'] = BadPickle + with self.assertRaises(TypeError): + dumps(BadPickle.dill) + with self.assertRaises(PicklingError): + dumps(BadPickle) + + def test_string_enum(self): + class SkillLevel(str, Enum): + master = 'what is the sound of one hand clapping?' + journeyman = 'why did the chicken cross the road?' + apprentice = 'knock, knock!' + self.assertEqual(SkillLevel.apprentice, 'knock, knock!') + + def test_getattr_getitem(self): + class Period(Enum): + morning = 1 + noon = 2 + evening = 3 + night = 4 + self.assertIs(Period(2), Period.noon) + self.assertIs(getattr(Period, 'night'), Period.night) + self.assertIs(Period['morning'], Period.morning) + + def test_getattr_dunder(self): + Season = self.Season + self.assertTrue(getattr(Season, '__eq__')) + + def test_iteration_order(self): + class Season(Enum): + SUMMER = 2 + WINTER = 4 + AUTUMN = 3 + SPRING = 1 + self.assertEqual( + list(Season), + [Season.SUMMER, Season.WINTER, Season.AUTUMN, Season.SPRING], + ) + + def test_reversed_iteration_order(self): + self.assertEqual( + list(reversed(self.Season)), + [self.Season.WINTER, self.Season.AUTUMN, self.Season.SUMMER, + self.Season.SPRING] + ) + + def test_programatic_function_string(self): + SummerMonth = Enum('SummerMonth', 'june july august') + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate('june july august'.split(), 1): + e = SummerMonth(i) + self.assertEqual(int(e.value), i) + self.assertNotEqual(e, i) + self.assertEqual(e.name, month) + self.assertIn(e, SummerMonth) + self.assertIs(type(e), SummerMonth) + + def test_programatic_function_string_list(self): + SummerMonth = Enum('SummerMonth', ['june', 'july', 'august']) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate('june july august'.split(), 1): + e = SummerMonth(i) + self.assertEqual(int(e.value), i) + self.assertNotEqual(e, i) + self.assertEqual(e.name, month) + self.assertIn(e, SummerMonth) + self.assertIs(type(e), SummerMonth) + + def test_programatic_function_iterable(self): + SummerMonth = Enum( + 'SummerMonth', + (('june', 1), ('july', 2), ('august', 3)) + ) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate('june july august'.split(), 1): + e = SummerMonth(i) + self.assertEqual(int(e.value), i) + self.assertNotEqual(e, i) + self.assertEqual(e.name, month) + self.assertIn(e, SummerMonth) + self.assertIs(type(e), SummerMonth) + + def test_programatic_function_from_dict(self): + SummerMonth = Enum( + 'SummerMonth', + OrderedDict((('june', 1), ('july', 2), ('august', 3))) + ) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate('june july august'.split(), 1): + e = SummerMonth(i) + self.assertEqual(int(e.value), i) + self.assertNotEqual(e, i) + self.assertEqual(e.name, month) + self.assertIn(e, SummerMonth) + self.assertIs(type(e), SummerMonth) + + def test_programatic_function_type(self): + SummerMonth = Enum('SummerMonth', 'june july august', type=int) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate('june july august'.split(), 1): + e = SummerMonth(i) + self.assertEqual(e, i) + self.assertEqual(e.name, month) + self.assertIn(e, SummerMonth) + self.assertIs(type(e), SummerMonth) + + def test_programatic_function_type_from_subclass(self): + SummerMonth = IntEnum('SummerMonth', 'june july august') + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate('june july august'.split(), 1): + e = SummerMonth(i) + self.assertEqual(e, i) + self.assertEqual(e.name, month) + self.assertIn(e, SummerMonth) + self.assertIs(type(e), SummerMonth) + + def test_subclassing(self): + if isinstance(Name, Exception): + raise Name + self.assertEqual(Name.BDFL, 'Guido van Rossum') + self.assertTrue(Name.BDFL, Name('Guido van Rossum')) + self.assertIs(Name.BDFL, getattr(Name, 'BDFL')) + self.assertIs(Name.BDFL, loads(dumps(Name.BDFL))) + + def test_extending(self): + class Color(Enum): + red = 1 + green = 2 + blue = 3 + with self.assertRaises(TypeError): + class MoreColor(Color): + cyan = 4 + magenta = 5 + yellow = 6 + + def test_exclude_methods(self): + class whatever(Enum): + this = 'that' + these = 'those' + def really(self): + return 'no, not %s' % self.value + self.assertIsNot(type(whatever.really), whatever) + self.assertEqual(whatever.this.really(), 'no, not that') + + def test_wrong_inheritance_order(self): + with self.assertRaises(TypeError): + class Wrong(Enum, str): + NotHere = 'error before this point' + + def test_intenum_transitivity(self): + class number(IntEnum): + one = 1 + two = 2 + three = 3 + class numero(IntEnum): + uno = 1 + dos = 2 + tres = 3 + self.assertEqual(number.one, numero.uno) + self.assertEqual(number.two, numero.dos) + self.assertEqual(number.three, numero.tres) + + def test_wrong_enum_in_call(self): + class Monochrome(Enum): + black = 0 + white = 1 + class Gender(Enum): + male = 0 + female = 1 + self.assertRaises(ValueError, Monochrome, Gender.male) + + def test_wrong_enum_in_mixed_call(self): + class Monochrome(IntEnum): + black = 0 + white = 1 + class Gender(Enum): + male = 0 + female = 1 + self.assertRaises(ValueError, Monochrome, Gender.male) + + def test_mixed_enum_in_call_1(self): + class Monochrome(IntEnum): + black = 0 + white = 1 + class Gender(IntEnum): + male = 0 + female = 1 + self.assertIs(Monochrome(Gender.female), Monochrome.white) + + def test_mixed_enum_in_call_2(self): + class Monochrome(Enum): + black = 0 + white = 1 + class Gender(IntEnum): + male = 0 + female = 1 + self.assertIs(Monochrome(Gender.male), Monochrome.black) + + def test_flufl_enum(self): + class Fluflnum(Enum): + def __int__(self): + return int(self.value) + class MailManOptions(Fluflnum): + option1 = 1 + option2 = 2 + option3 = 3 + self.assertEqual(int(MailManOptions.option1), 1) + + def test_introspection(self): + class Number(IntEnum): + one = 100 + two = 200 + self.assertIs(Number.one._member_type_, int) + self.assertIs(Number._member_type_, int) + class String(str, Enum): + yarn = 'soft' + rope = 'rough' + wire = 'hard' + self.assertIs(String.yarn._member_type_, str) + self.assertIs(String._member_type_, str) + class Plain(Enum): + vanilla = 'white' + one = 1 + self.assertIs(Plain.vanilla._member_type_, object) + self.assertIs(Plain._member_type_, object) + + def test_no_such_enum_member(self): + class Color(Enum): + red = 1 + green = 2 + blue = 3 + with self.assertRaises(ValueError): + Color(4) + with self.assertRaises(KeyError): + Color['chartreuse'] + + def test_new_repr(self): + class Color(Enum): + red = 1 + green = 2 + blue = 3 + def __repr__(self): + return "don't you just love shades of %s?" % self.name + self.assertEqual( + repr(Color.blue), + "don't you just love shades of blue?", + ) + + def test_inherited_repr(self): + class MyEnum(Enum): + def __repr__(self): + return "My name is %s." % self.name + class MyIntEnum(int, MyEnum): + this = 1 + that = 2 + theother = 3 + self.assertEqual(repr(MyIntEnum.that), "My name is that.") + + def test_multiple_mixin_mro(self): + class auto_enum(type(Enum)): + def __new__(metacls, cls, bases, classdict): + temp = type(classdict)() + names = set(classdict._member_names) + i = 0 + for k in classdict._member_names: + v = classdict[k] + if v is Ellipsis: + v = i + else: + i = v + i += 1 + temp[k] = v + for k, v in classdict.items(): + if k not in names: + temp[k] = v + return super(auto_enum, metacls).__new__( + metacls, cls, bases, temp) + + class AutoNumberedEnum(Enum, metaclass=auto_enum): + pass + + class AutoIntEnum(IntEnum, metaclass=auto_enum): + pass + + class TestAutoNumber(AutoNumberedEnum): + a = ... + b = 3 + c = ... + + class TestAutoInt(AutoIntEnum): + a = ... + b = 3 + c = ... + + def test_subclasses_with_getnewargs(self): + class NamedInt(int): + def __new__(cls, *args): + _args = args + name, *args = args + if len(args) == 0: + raise TypeError("name and value must be specified") + self = int.__new__(cls, *args) + self._intname = name + self._args = _args + return self + def __getnewargs__(self): + return self._args + @property + def __name__(self): + return self._intname + def __repr__(self): + # repr() is updated to include the name and type info + return "{}({!r}, {})".format(type(self).__name__, + self.__name__, + int.__repr__(self)) + def __str__(self): + # str() is unchanged, even if it relies on the repr() fallback + base = int + base_str = base.__str__ + if base_str.__objclass__ is object: + return base.__repr__(self) + return base_str(self) + # for simplicity, we only define one operator that + # propagates expressions + def __add__(self, other): + temp = int(self) + int( other) + if isinstance(self, NamedInt) and isinstance(other, NamedInt): + return NamedInt( + '({0} + {1})'.format(self.__name__, other.__name__), + temp ) + else: + return temp + + class NEI(NamedInt, Enum): + x = ('the-x', 1) + y = ('the-y', 2) + + + self.assertIs(NEI.__new__, Enum.__new__) + self.assertEqual(repr(NEI.x + NEI.y), "NamedInt('(the-x + the-y)', 3)") + globals()['NamedInt'] = NamedInt + globals()['NEI'] = NEI + NI5 = NamedInt('test', 5) + self.assertEqual(NI5, 5) + self.assertEqual(loads(dumps(NI5)), 5) + self.assertEqual(NEI.y.value, 2) + self.assertIs(loads(dumps(NEI.y)), NEI.y) + + def test_subclasses_without_getnewargs(self): + class NamedInt(int): + def __new__(cls, *args): + _args = args + name, *args = args + if len(args) == 0: + raise TypeError("name and value must be specified") + self = int.__new__(cls, *args) + self._intname = name + self._args = _args + return self + @property + def __name__(self): + return self._intname + def __repr__(self): + # repr() is updated to include the name and type info + return "{}({!r}, {})".format(type(self).__name__, + self.__name__, + int.__repr__(self)) + def __str__(self): + # str() is unchanged, even if it relies on the repr() fallback + base = int + base_str = base.__str__ + if base_str.__objclass__ is object: + return base.__repr__(self) + return base_str(self) + # for simplicity, we only define one operator that + # propagates expressions + def __add__(self, other): + temp = int(self) + int( other) + if isinstance(self, NamedInt) and isinstance(other, NamedInt): + return NamedInt( + '({0} + {1})'.format(self.__name__, other.__name__), + temp ) + else: + return temp + + class NEI(NamedInt, Enum): + x = ('the-x', 1) + y = ('the-y', 2) + + self.assertIs(NEI.__new__, Enum.__new__) + self.assertEqual(repr(NEI.x + NEI.y), "NamedInt('(the-x + the-y)', 3)") + globals()['NamedInt'] = NamedInt + globals()['NEI'] = NEI + NI5 = NamedInt('test', 5) + self.assertEqual(NI5, 5) + self.assertEqual(NEI.y.value, 2) + with self.assertRaises(TypeError): + dumps(NEI.x) + with self.assertRaises(PicklingError): + dumps(NEI) + + def test_tuple_subclass(self): + class SomeTuple(tuple, Enum): + first = (1, 'for the money') + second = (2, 'for the show') + third = (3, 'for the music') + self.assertIs(type(SomeTuple.first), SomeTuple) + self.assertIsInstance(SomeTuple.second, tuple) + self.assertEqual(SomeTuple.third, (3, 'for the music')) + globals()['SomeTuple'] = SomeTuple + self.assertIs(loads(dumps(SomeTuple.first)), SomeTuple.first) + + def test_duplicate_values_give_unique_enum_items(self): + class AutoNumber(Enum): + first = () + second = () + third = () + def __new__(cls): + value = len(cls.__members__) + 1 + obj = object.__new__(cls) + obj._value_ = value + return obj + def __int__(self): + return int(self._value_) + self.assertEqual( + list(AutoNumber), + [AutoNumber.first, AutoNumber.second, AutoNumber.third], + ) + self.assertEqual(int(AutoNumber.second), 2) + self.assertEqual(AutoNumber.third.value, 3) + self.assertIs(AutoNumber(1), AutoNumber.first) + + def test_inherited_new_from_enhanced_enum(self): + class AutoNumber(Enum): + def __new__(cls): + value = len(cls.__members__) + 1 + obj = object.__new__(cls) + obj._value_ = value + return obj + def __int__(self): + return int(self._value_) + class Color(AutoNumber): + red = () + green = () + blue = () + self.assertEqual(list(Color), [Color.red, Color.green, Color.blue]) + self.assertEqual(list(map(int, Color)), [1, 2, 3]) + + def test_inherited_new_from_mixed_enum(self): + class AutoNumber(IntEnum): + def __new__(cls): + value = len(cls.__members__) + 1 + obj = int.__new__(cls, value) + obj._value_ = value + return obj + class Color(AutoNumber): + red = () + green = () + blue = () + self.assertEqual(list(Color), [Color.red, Color.green, Color.blue]) + self.assertEqual(list(map(int, Color)), [1, 2, 3]) + + def test_ordered_mixin(self): + class OrderedEnum(Enum): + def __ge__(self, other): + if self.__class__ is other.__class__: + return self._value_ >= other._value_ + return NotImplemented + def __gt__(self, other): + if self.__class__ is other.__class__: + return self._value_ > other._value_ + return NotImplemented + def __le__(self, other): + if self.__class__ is other.__class__: + return self._value_ <= other._value_ + return NotImplemented + def __lt__(self, other): + if self.__class__ is other.__class__: + return self._value_ < other._value_ + return NotImplemented + class Grade(OrderedEnum): + A = 5 + B = 4 + C = 3 + D = 2 + F = 1 + self.assertGreater(Grade.A, Grade.B) + self.assertLessEqual(Grade.F, Grade.C) + self.assertLess(Grade.D, Grade.A) + self.assertGreaterEqual(Grade.B, Grade.B) + + def test_extending2(self): + class Shade(Enum): + def shade(self): + print(self.name) + class Color(Shade): + red = 1 + green = 2 + blue = 3 + with self.assertRaises(TypeError): + class MoreColor(Color): + cyan = 4 + magenta = 5 + yellow = 6 + + def test_extending3(self): + class Shade(Enum): + def shade(self): + return self.name + class Color(Shade): + def hex(self): + return '%s hexlified!' % self.value + class MoreColor(Color): + cyan = 4 + magenta = 5 + yellow = 6 + self.assertEqual(MoreColor.magenta.hex(), '5 hexlified!') + + + def test_no_duplicates(self): + class UniqueEnum(Enum): + def __init__(self, *args): + cls = self.__class__ + if any(self.value == e.value for e in cls): + a = self.name + e = cls(self.value).name + raise ValueError( + "aliases not allowed in UniqueEnum: %r --> %r" + % (a, e) + ) + class Color(UniqueEnum): + red = 1 + green = 2 + blue = 3 + with self.assertRaises(ValueError): + class Color(UniqueEnum): + red = 1 + green = 2 + blue = 3 + grene = 2 + + def test_init(self): + class Planet(Enum): + MERCURY = (3.303e+23, 2.4397e6) + VENUS = (4.869e+24, 6.0518e6) + EARTH = (5.976e+24, 6.37814e6) + MARS = (6.421e+23, 3.3972e6) + JUPITER = (1.9e+27, 7.1492e7) + SATURN = (5.688e+26, 6.0268e7) + URANUS = (8.686e+25, 2.5559e7) + NEPTUNE = (1.024e+26, 2.4746e7) + def __init__(self, mass, radius): + self.mass = mass # in kilograms + self.radius = radius # in meters + @property + def surface_gravity(self): + # universal gravitational constant (m3 kg-1 s-2) + G = 6.67300E-11 + return G * self.mass / (self.radius * self.radius) + self.assertEqual(round(Planet.EARTH.surface_gravity, 2), 9.80) + self.assertEqual(Planet.EARTH.value, (5.976e+24, 6.37814e6)) + + def test_nonhash_value(self): + class AutoNumberInAList(Enum): + def __new__(cls): + value = [len(cls.__members__) + 1] + obj = object.__new__(cls) + obj._value_ = value + return obj + class ColorInAList(AutoNumberInAList): + red = () + green = () + blue = () + self.assertEqual(list(ColorInAList), [ColorInAList.red, ColorInAList.green, ColorInAList.blue]) + self.assertEqual(ColorInAList.red.value, [1]) + self.assertEqual(ColorInAList([1]), ColorInAList.red) + + def test_conflicting_types_resolved_in_new(self): + class LabelledIntEnum(int, Enum): + def __new__(cls, *args): + value, label = args + obj = int.__new__(cls, value) + obj.label = label + obj._value_ = value + return obj + + class LabelledList(LabelledIntEnum): + unprocessed = (1, "Unprocessed") + payment_complete = (2, "Payment Complete") + + self.assertEqual(list(LabelledList), [LabelledList.unprocessed, LabelledList.payment_complete]) + self.assertEqual(LabelledList.unprocessed, 1) + self.assertEqual(LabelledList(1), LabelledList.unprocessed) + + +class TestUnique(unittest.TestCase): + + def test_unique_clean(self): + @unique + class Clean(Enum): + one = 1 + two = 'dos' + tres = 4.0 + @unique + class Cleaner(IntEnum): + single = 1 + double = 2 + triple = 3 + + def test_unique_dirty(self): + with self.assertRaisesRegex(ValueError, 'tres.*one'): + @unique + class Dirty(Enum): + one = 1 + two = 'dos' + tres = 1 + with self.assertRaisesRegex( + ValueError, + 'double.*single.*turkey.*triple', + ): + @unique + class Dirtier(IntEnum): + single = 1 + double = 1 + triple = 3 + turkey = 3 + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_enumerate.py b/Lib/test/test_enumerate.py index 2e904cf878..a2d18d01ad 100644 --- a/Lib/test/test_enumerate.py +++ b/Lib/test/test_enumerate.py @@ -1,4 +1,5 @@ import unittest +import operator import sys import pickle @@ -168,15 +169,12 @@ class TestReversed(unittest.TestCase, PickleTest): x = range(1) self.assertEqual(type(reversed(x)), type(iter(x))) - @support.cpython_only def test_len(self): - # This is an implementation detail, not an interface requirement - from test.test_iterlen import len for s in ('hello', tuple('hello'), list('hello'), range(5)): - self.assertEqual(len(reversed(s)), len(s)) + self.assertEqual(operator.length_hint(reversed(s)), len(s)) r = reversed(s) list(r) - self.assertEqual(len(r), 0) + self.assertEqual(operator.length_hint(r), 0) class SeqWithWeirdLen: called = False def __len__(self): @@ -187,7 +185,7 @@ class TestReversed(unittest.TestCase, PickleTest): def __getitem__(self, index): return index r = reversed(SeqWithWeirdLen()) - self.assertRaises(ZeroDivisionError, len, r) + self.assertRaises(ZeroDivisionError, operator.length_hint, r) def test_gc(self): diff --git a/Lib/test/test_epoll.py b/Lib/test/test_epoll.py index 7f9547ff95..6459fbafc8 100644 --- a/Lib/test/test_epoll.py +++ b/Lib/test/test_epoll.py @@ -21,10 +21,11 @@ """ Tests for epoll wrapper. """ -import socket import errno -import time +import os import select +import socket +import time import unittest from test import support @@ -33,7 +34,7 @@ if not hasattr(select, "epoll"): try: select.epoll() -except IOError as e: +except OSError as e: if e.errno == errno.ENOSYS: raise unittest.SkipTest("kernel doesn't support epoll()") raise @@ -56,7 +57,7 @@ class TestEPoll(unittest.TestCase): client.setblocking(False) try: client.connect(('127.0.0.1', self.serverSocket.getsockname()[1])) - except socket.error as e: + except OSError as e: self.assertEqual(e.args[0], errno.EINPROGRESS) else: raise AssertionError("Connect should have raised EINPROGRESS") @@ -87,6 +88,13 @@ class TestEPoll(unittest.TestCase): self.assertRaises(TypeError, select.epoll, ['foo']) self.assertRaises(TypeError, select.epoll, {}) + def test_context_manager(self): + with select.epoll(16) as ep: + self.assertGreater(ep.fileno(), 0) + self.assertFalse(ep.closed) + self.assertTrue(ep.closed) + self.assertRaises(ValueError, ep.fileno) + def test_add(self): server, client = self._connected_pair() @@ -115,12 +123,12 @@ class TestEPoll(unittest.TestCase): # ValueError: file descriptor cannot be a negative integer (-1) self.assertRaises(ValueError, ep.register, -1, select.EPOLLIN | select.EPOLLOUT) - # IOError: [Errno 9] Bad file descriptor - self.assertRaises(IOError, ep.register, 10000, + # OSError: [Errno 9] Bad file descriptor + self.assertRaises(OSError, ep.register, 10000, select.EPOLLIN | select.EPOLLOUT) # registering twice also raises an exception ep.register(server, select.EPOLLIN | select.EPOLLOUT) - self.assertRaises(IOError, ep.register, server, + self.assertRaises(OSError, ep.register, server, select.EPOLLIN | select.EPOLLOUT) finally: ep.close() @@ -142,7 +150,7 @@ class TestEPoll(unittest.TestCase): ep.close() try: ep2.poll(1, 4) - except IOError as e: + except OSError as e: self.assertEqual(e.args[0], errno.EBADF, e) else: self.fail("epoll on closed fd didn't raise EBADF") @@ -218,6 +226,36 @@ class TestEPoll(unittest.TestCase): server.close() ep.unregister(fd) + def test_close(self): + open_file = open(__file__, "rb") + self.addCleanup(open_file.close) + fd = open_file.fileno() + epoll = select.epoll() + + # test fileno() method and closed attribute + self.assertIsInstance(epoll.fileno(), int) + self.assertFalse(epoll.closed) + + # test close() + epoll.close() + self.assertTrue(epoll.closed) + self.assertRaises(ValueError, epoll.fileno) + + # close() can be called more than once + epoll.close() + + # operations must fail with ValueError("I/O operation on closed ...") + self.assertRaises(ValueError, epoll.modify, fd, select.EPOLLIN) + self.assertRaises(ValueError, epoll.poll, 1.0) + self.assertRaises(ValueError, epoll.register, fd, select.EPOLLIN) + self.assertRaises(ValueError, epoll.unregister, fd) + + def test_fd_non_inheritable(self): + epoll = select.epoll() + self.addCleanup(epoll.close) + self.assertEqual(os.get_inheritable(epoll.fileno()), False) + + def test_main(): support.run_unittest(TestEPoll) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 1ad7f97b74..f0851bdbe2 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -8,8 +8,8 @@ import weakref import errno from test.support import (TESTFN, captured_output, check_impl_detail, - cpython_only, gc_collect, run_unittest, no_tracing, - unlink) + check_warnings, cpython_only, gc_collect, run_unittest, + no_tracing, unlink) class NaiveException(Exception): def __init__(self, x): @@ -244,22 +244,22 @@ class ExceptionTests(unittest.TestCase): {'args' : ('foo', 1)}), (SystemExit, ('foo',), {'args' : ('foo',), 'code' : 'foo'}), - (IOError, ('foo',), + (OSError, ('foo',), {'args' : ('foo',), 'filename' : None, 'errno' : None, 'strerror' : None}), - (IOError, ('foo', 'bar'), + (OSError, ('foo', 'bar'), {'args' : ('foo', 'bar'), 'filename' : None, 'errno' : 'foo', 'strerror' : 'bar'}), - (IOError, ('foo', 'bar', 'baz'), + (OSError, ('foo', 'bar', 'baz'), {'args' : ('foo', 'bar'), 'filename' : 'baz', 'errno' : 'foo', 'strerror' : 'bar'}), - (IOError, ('foo', 'bar', 'baz', 'quux'), + (OSError, ('foo', 'bar', 'baz', 'quux'), {'args' : ('foo', 'bar', 'baz', 'quux')}), - (EnvironmentError, ('errnoStr', 'strErrorStr', 'filenameStr'), + (OSError, ('errnoStr', 'strErrorStr', 'filenameStr'), {'args' : ('errnoStr', 'strErrorStr'), 'strerror' : 'strErrorStr', 'errno' : 'errnoStr', 'filename' : 'filenameStr'}), - (EnvironmentError, (1, 'strErrorStr', 'filenameStr'), + (OSError, (1, 'strErrorStr', 'filenameStr'), {'args' : (1, 'strErrorStr'), 'errno' : 1, 'strerror' : 'strErrorStr', 'filename' : 'filenameStr'}), (SyntaxError, (), {'msg' : None, 'text' : None, @@ -409,7 +409,7 @@ class ExceptionTests(unittest.TestCase): self.assertIsNone(e.__context__) self.assertIsNone(e.__cause__) - class MyException(EnvironmentError): + class MyException(OSError): pass e = MyException() @@ -947,13 +947,11 @@ class ImportErrorTests(unittest.TestCase): def test_non_str_argument(self): # Issue #15778 - arg = b'abc' - exc = ImportError(arg) - self.assertEqual(str(arg), str(exc)) + with check_warnings(('', BytesWarning), quiet=True): + arg = b'abc' + exc = ImportError(arg) + self.assertEqual(str(arg), str(exc)) -def test_main(): - run_unittest(ExceptionTests, ImportErrorTests) - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 770e70c77d..2d374b9d6c 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -19,18 +19,6 @@ except ImportError: TIMEOUT = 0.5 -try: - from resource import setrlimit, RLIMIT_CORE, error as resource_error -except ImportError: - prepare_subprocess = None -else: - def prepare_subprocess(): - # don't create core file - try: - setrlimit(RLIMIT_CORE, (0, 0)) - except (ValueError, resource_error): - pass - def expected_traceback(lineno1, lineno2, header, min_count=1): regex = header regex += ' File "<string>", line %s in func\n' % lineno1 @@ -59,10 +47,8 @@ class FaultHandlerTests(unittest.TestCase): build, and replace "Current thread 0x00007f8d8fbd9700" by "Current thread XXX". """ - options = {} - if prepare_subprocess: - options['preexec_fn'] = prepare_subprocess - process = script_helper.spawn_python('-c', code, **options) + with support.SuppressCrashReport(): + process = script_helper.spawn_python('-c', code) stdout, stderr = process.communicate() exitcode = process.wait() output = support.strip_python_stderr(stdout) @@ -101,8 +87,7 @@ class FaultHandlerTests(unittest.TestCase): header=re.escape(header)) if other_regex: regex += '|' + other_regex - with support.suppress_crash_popup(): - output, exitcode = self.get_output(code, filename) + output, exitcode = self.get_output(code, filename) output = '\n'.join(output) self.assertRegex(output, regex) self.assertNotEqual(exitcode, 0) @@ -232,8 +217,7 @@ faulthandler.disable() faulthandler._sigsegv() """.strip() not_expected = 'Fatal Python error' - with support.suppress_crash_popup(): - stderr, exitcode = self.get_output(code) + stderr, exitcode = self.get_output(code) stder = '\n'.join(stderr) self.assertTrue(not_expected not in stderr, "%r is present in %r" % (not_expected, stderr)) @@ -264,16 +248,34 @@ faulthandler._sigsegv() def test_disabled_by_default(self): # By default, the module should be disabled code = "import faulthandler; print(faulthandler.is_enabled())" - rc, stdout, stderr = assert_python_ok("-c", code) - stdout = (stdout + stderr).strip() - self.assertEqual(stdout, b"False") + args = (sys.executable, '-E', '-c', code) + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args) + self.assertEqual(output.rstrip(), b"False") def test_sys_xoptions(self): # Test python -X faulthandler code = "import faulthandler; print(faulthandler.is_enabled())" - rc, stdout, stderr = assert_python_ok("-X", "faulthandler", "-c", code) - stdout = (stdout + stderr).strip() - self.assertEqual(stdout, b"True") + args = (sys.executable, "-E", "-X", "faulthandler", "-c", code) + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args) + self.assertEqual(output.rstrip(), b"True") + + def test_env_var(self): + # empty env var + code = "import faulthandler; print(faulthandler.is_enabled())" + args = (sys.executable, "-c", code) + env = os.environ.copy() + env['PYTHONFAULTHANDLER'] = '' + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args, env=env) + self.assertEqual(output.rstrip(), b"False") + + # non-empty env var + env = os.environ.copy() + env['PYTHONFAULTHANDLER'] = '1' + output = subprocess.check_output(args, env=env) + self.assertEqual(output.rstrip(), b"True") def check_dump_traceback(self, filename): """ @@ -590,8 +592,5 @@ sys.exit(exitcode) self.check_register(chain=True) -def test_main(): - support.run_unittest(FaultHandlerTests) - if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index b8cda2f108..c816d970d7 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -1,7 +1,4 @@ """Test program for the fcntl C module. - -OS/2+EMX doesn't support the file locking operations. - """ import platform import os @@ -39,8 +36,6 @@ def get_lockdata(): lockdata = struct.pack('qqihhi', 0, 0, 0, fcntl.F_WRLCK, 0, 0) elif sys.platform in ['aix3', 'aix4', 'hp-uxB', 'unixware7']: lockdata = struct.pack('hhlllii', fcntl.F_WRLCK, 0, 0, 0, 0, 0, 0) - elif sys.platform in ['os2emx']: - lockdata = None else: lockdata = struct.pack('hh'+start_len+'hh', fcntl.F_WRLCK, 0, 0, 0, 0, 0) if lockdata: @@ -66,18 +61,20 @@ class TestFcntl(unittest.TestCase): rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETFL, os.O_NONBLOCK) if verbose: print('Status from fcntl with O_NONBLOCK: ', rv) - if sys.platform not in ['os2emx']: - rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETLKW, lockdata) - if verbose: - print('String from fcntl with F_SETLKW: ', repr(rv)) + rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETLKW, lockdata) + if verbose: + print('String from fcntl with F_SETLKW: ', repr(rv)) self.f.close() def test_fcntl_file_descriptor(self): # again, but pass the file rather than numeric descriptor self.f = open(TESTFN, 'wb') rv = fcntl.fcntl(self.f, fcntl.F_SETFL, os.O_NONBLOCK) - if sys.platform not in ['os2emx']: - rv = fcntl.fcntl(self.f, fcntl.F_SETLKW, lockdata) + if verbose: + print('Status from fcntl with O_NONBLOCK: ', rv) + rv = fcntl.fcntl(self.f, fcntl.F_SETLKW, lockdata) + if verbose: + print('String from fcntl with F_SETLKW: ', repr(rv)) self.f.close() def test_fcntl_bad_file(self): diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py index 76b1694e98..d54e976143 100644 --- a/Lib/test/test_file.py +++ b/Lib/test/test_file.py @@ -87,7 +87,7 @@ class AutoFileTests: self.assertTrue(not f.closed) if hasattr(f, "readinto"): - self.assertRaises((IOError, TypeError), f.readinto, "") + self.assertRaises((OSError, TypeError), f.readinto, "") f.close() self.assertTrue(f.closed) @@ -126,7 +126,7 @@ class AutoFileTests: self.assertEqual(self.f.__exit__(*sys.exc_info()), None) def testReadWhenWriting(self): - self.assertRaises(IOError, self.f.read) + self.assertRaises(OSError, self.f.read) class CAutoFileTests(AutoFileTests, unittest.TestCase): open = io.open @@ -177,7 +177,7 @@ class OtherFileTests: d = int(f.read().decode("ascii")) f.close() f.close() - except IOError as msg: + except OSError as msg: self.fail('error setting buffer size %d: %s' % (s, str(msg))) self.assertEqual(d, s) diff --git a/Lib/test/test_filecmp.py b/Lib/test/test_filecmp.py index 09599794ad..c7cb3fcf36 100644 --- a/Lib/test/test_filecmp.py +++ b/Lib/test/test_filecmp.py @@ -1,4 +1,3 @@ - import os, filecmp, shutil, tempfile import unittest from test import support @@ -40,15 +39,27 @@ class FileCompareTestCase(unittest.TestCase): self.assertFalse(filecmp.cmp(self.name, self.dir), "File and directory compare as equal") + def test_cache_clear(self): + first_compare = filecmp.cmp(self.name, self.name_same, shallow=False) + second_compare = filecmp.cmp(self.name, self.name_diff, shallow=False) + filecmp.clear_cache() + self.assertTrue(len(filecmp._cache) == 0, + "Cache not cleared after calling clear_cache") + class DirCompareTestCase(unittest.TestCase): def setUp(self): tmpdir = tempfile.gettempdir() self.dir = os.path.join(tmpdir, 'dir') self.dir_same = os.path.join(tmpdir, 'dir-same') self.dir_diff = os.path.join(tmpdir, 'dir-diff') + + # Another dir is created under dir_same, but it has a name from the + # ignored list so it should not affect testing results. + self.dir_ignored = os.path.join(self.dir_same, '.hg') + self.caseinsensitive = os.path.normcase('A') == os.path.normcase('a') data = 'Contents of file go here.\n' - for dir in [self.dir, self.dir_same, self.dir_diff]: + for dir in (self.dir, self.dir_same, self.dir_diff, self.dir_ignored): shutil.rmtree(dir, True) os.mkdir(dir) if self.caseinsensitive and dir is self.dir_same: @@ -64,9 +75,11 @@ class DirCompareTestCase(unittest.TestCase): output.close() def tearDown(self): - shutil.rmtree(self.dir) - shutil.rmtree(self.dir_same) - shutil.rmtree(self.dir_diff) + for dir in (self.dir, self.dir_same, self.dir_diff): + shutil.rmtree(dir) + + def test_default_ignores(self): + self.assertIn('.hg', filecmp.DEFAULT_IGNORES) def test_cmpfiles(self): self.assertTrue(filecmp.cmpfiles(self.dir, self.dir, ['file']) == diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index 1e70641150..c5e57d4715 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -275,8 +275,8 @@ class FileInputTests(unittest.TestCase): try: t1 = writeTmp(1, [""]) with FileInput(files=t1) as fi: - raise IOError - except IOError: + raise OSError + except OSError: self.assertEqual(fi._files, ()) finally: remove_tempfiles(t1) @@ -835,22 +835,6 @@ class Test_hook_encoded(unittest.TestCase): self.assertIs(kwargs.pop('encoding'), encoding) self.assertFalse(kwargs) -def test_main(): - run_unittest( - BufferSizesTests, - FileInputTests, - Test_fileinput_input, - Test_fileinput_close, - Test_fileinput_nextfile, - Test_fileinput_filename, - Test_fileinput_lineno, - Test_fileinput_filelineno, - Test_fileinput_fileno, - Test_fileinput_isfirstline, - Test_fileinput_isstdin, - Test_hook_compressed, - Test_hook_encoded, - ) if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index f91ccaa427..e67876985e 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -145,16 +145,16 @@ class AutoFileTests(unittest.TestCase): # Unix calls dircheck() and returns "[Errno 21]: Is a directory" try: _FileIO('.', 'r') - except IOError as e: + except OSError as e: self.assertNotEqual(e.errno, 0) self.assertEqual(e.filename, ".") else: - self.fail("Should have raised IOError") + self.fail("Should have raised OSError") @unittest.skipIf(os.name == 'nt', "test only works on a POSIX-like system") def testOpenDirFD(self): fd = os.open('.', os.O_RDONLY) - with self.assertRaises(IOError) as cm: + with self.assertRaises(OSError) as cm: _FileIO(fd, 'r') os.close(fd) self.assertEqual(cm.exception.errno, errno.EISDIR) @@ -172,7 +172,7 @@ class AutoFileTests(unittest.TestCase): finally: try: self.f.close() - except IOError: + except OSError: pass return wrapper @@ -184,14 +184,14 @@ class AutoFileTests(unittest.TestCase): os.close(f.fileno()) try: func(self, f) - except IOError as e: + except OSError as e: self.assertEqual(e.errno, errno.EBADF) else: - self.fail("Should have raised IOError") + self.fail("Should have raised OSError") finally: try: self.f.close() - except IOError: + except OSError: pass return wrapper @@ -238,7 +238,7 @@ class AutoFileTests(unittest.TestCase): def ReopenForRead(self): try: self.f.close() - except IOError: + except OSError: pass self.f = _FileIO(TESTFN, 'r') os.close(self.f.fileno()) @@ -286,7 +286,7 @@ class OtherFileTests(unittest.TestCase): if sys.platform != "win32": try: f = _FileIO("/dev/tty", "a") - except EnvironmentError: + except OSError: # When run in a cron job there just aren't any # ttys, so skip the test. This also handles other # OS'es that don't support /dev/tty. @@ -362,7 +362,7 @@ class OtherFileTests(unittest.TestCase): self.assertRaises(OSError, _FileIO, make_bad_fd()) if sys.platform == 'win32': import msvcrt - self.assertRaises(IOError, msvcrt.get_osfhandle, make_bad_fd()) + self.assertRaises(OSError, msvcrt.get_osfhandle, make_bad_fd()) # Issue 15989 self.assertRaises(TypeError, _FileIO, _testcapi.INT_MAX + 1) self.assertRaises(TypeError, _FileIO, _testcapi.INT_MIN - 1) diff --git a/Lib/test/test_finalization.py b/Lib/test/test_finalization.py new file mode 100644 index 0000000000..80c9b87988 --- /dev/null +++ b/Lib/test/test_finalization.py @@ -0,0 +1,513 @@ +""" +Tests for object finalization semantics, as outlined in PEP 442. +""" + +import contextlib +import gc +import unittest +import weakref + +import _testcapi +from test import support + + +class NonGCSimpleBase: + """ + The base class for all the objects under test, equipped with various + testing features. + """ + + survivors = [] + del_calls = [] + tp_del_calls = [] + errors = [] + + _cleaning = False + + __slots__ = () + + @classmethod + def _cleanup(cls): + cls.survivors.clear() + cls.errors.clear() + gc.garbage.clear() + gc.collect() + cls.del_calls.clear() + cls.tp_del_calls.clear() + + @classmethod + @contextlib.contextmanager + def test(cls): + """ + A context manager to use around all finalization tests. + """ + with support.disable_gc(): + cls.del_calls.clear() + cls.tp_del_calls.clear() + NonGCSimpleBase._cleaning = False + try: + yield + if cls.errors: + raise cls.errors[0] + finally: + NonGCSimpleBase._cleaning = True + cls._cleanup() + + def check_sanity(self): + """ + Check the object is sane (non-broken). + """ + + def __del__(self): + """ + PEP 442 finalizer. Record that this was called, check the + object is in a sane state, and invoke a side effect. + """ + try: + if not self._cleaning: + self.del_calls.append(id(self)) + self.check_sanity() + self.side_effect() + except Exception as e: + self.errors.append(e) + + def side_effect(self): + """ + A side effect called on destruction. + """ + + +class SimpleBase(NonGCSimpleBase): + + def __init__(self): + self.id_ = id(self) + + def check_sanity(self): + assert self.id_ == id(self) + + +class NonGC(NonGCSimpleBase): + __slots__ = () + +class NonGCResurrector(NonGCSimpleBase): + __slots__ = () + + def side_effect(self): + """ + Resurrect self by storing self in a class-wide list. + """ + self.survivors.append(self) + +class Simple(SimpleBase): + pass + +class SimpleResurrector(NonGCResurrector, SimpleBase): + pass + + +class TestBase: + + def setUp(self): + self.old_garbage = gc.garbage[:] + gc.garbage[:] = [] + + def tearDown(self): + # None of the tests here should put anything in gc.garbage + try: + self.assertEqual(gc.garbage, []) + finally: + del self.old_garbage + gc.collect() + + def assert_del_calls(self, ids): + self.assertEqual(sorted(SimpleBase.del_calls), sorted(ids)) + + def assert_tp_del_calls(self, ids): + self.assertEqual(sorted(SimpleBase.tp_del_calls), sorted(ids)) + + def assert_survivors(self, ids): + self.assertEqual(sorted(id(x) for x in SimpleBase.survivors), sorted(ids)) + + def assert_garbage(self, ids): + self.assertEqual(sorted(id(x) for x in gc.garbage), sorted(ids)) + + def clear_survivors(self): + SimpleBase.survivors.clear() + + +class SimpleFinalizationTest(TestBase, unittest.TestCase): + """ + Test finalization without refcycles. + """ + + def test_simple(self): + with SimpleBase.test(): + s = Simple() + ids = [id(s)] + wr = weakref.ref(s) + del s + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors([]) + self.assertIs(wr(), None) + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors([]) + + def test_simple_resurrect(self): + with SimpleBase.test(): + s = SimpleResurrector() + ids = [id(s)] + wr = weakref.ref(s) + del s + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors(ids) + self.assertIsNot(wr(), None) + self.clear_survivors() + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors([]) + self.assertIs(wr(), None) + + def test_non_gc(self): + with SimpleBase.test(): + s = NonGC() + self.assertFalse(gc.is_tracked(s)) + ids = [id(s)] + del s + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors([]) + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors([]) + + def test_non_gc_resurrect(self): + with SimpleBase.test(): + s = NonGCResurrector() + self.assertFalse(gc.is_tracked(s)) + ids = [id(s)] + del s + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors(ids) + self.clear_survivors() + gc.collect() + self.assert_del_calls(ids * 2) + self.assert_survivors(ids) + + +class SelfCycleBase: + + def __init__(self): + super().__init__() + self.ref = self + + def check_sanity(self): + super().check_sanity() + assert self.ref is self + +class SimpleSelfCycle(SelfCycleBase, Simple): + pass + +class SelfCycleResurrector(SelfCycleBase, SimpleResurrector): + pass + +class SuicidalSelfCycle(SelfCycleBase, Simple): + + def side_effect(self): + """ + Explicitly break the reference cycle. + """ + self.ref = None + + +class SelfCycleFinalizationTest(TestBase, unittest.TestCase): + """ + Test finalization of an object having a single cyclic reference to + itself. + """ + + def test_simple(self): + with SimpleBase.test(): + s = SimpleSelfCycle() + ids = [id(s)] + wr = weakref.ref(s) + del s + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors([]) + self.assertIs(wr(), None) + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors([]) + + def test_simple_resurrect(self): + # Test that __del__ can resurrect the object being finalized. + with SimpleBase.test(): + s = SelfCycleResurrector() + ids = [id(s)] + wr = weakref.ref(s) + del s + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors(ids) + # XXX is this desirable? + self.assertIs(wr(), None) + # When trying to destroy the object a second time, __del__ + # isn't called anymore (and the object isn't resurrected). + self.clear_survivors() + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors([]) + self.assertIs(wr(), None) + + def test_simple_suicide(self): + # Test the GC is able to deal with an object that kills its last + # reference during __del__. + with SimpleBase.test(): + s = SuicidalSelfCycle() + ids = [id(s)] + wr = weakref.ref(s) + del s + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors([]) + self.assertIs(wr(), None) + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors([]) + self.assertIs(wr(), None) + + +class ChainedBase: + + def chain(self, left): + self.suicided = False + self.left = left + left.right = self + + def check_sanity(self): + super().check_sanity() + if self.suicided: + assert self.left is None + assert self.right is None + else: + left = self.left + if left.suicided: + assert left.right is None + else: + assert left.right is self + right = self.right + if right.suicided: + assert right.left is None + else: + assert right.left is self + +class SimpleChained(ChainedBase, Simple): + pass + +class ChainedResurrector(ChainedBase, SimpleResurrector): + pass + +class SuicidalChained(ChainedBase, Simple): + + def side_effect(self): + """ + Explicitly break the reference cycle. + """ + self.suicided = True + self.left = None + self.right = None + + +class CycleChainFinalizationTest(TestBase, unittest.TestCase): + """ + Test finalization of a cyclic chain. These tests are similar in + spirit to the self-cycle tests above, but the collectable object + graph isn't trivial anymore. + """ + + def build_chain(self, classes): + nodes = [cls() for cls in classes] + for i in range(len(nodes)): + nodes[i].chain(nodes[i-1]) + return nodes + + def check_non_resurrecting_chain(self, classes): + N = len(classes) + with SimpleBase.test(): + nodes = self.build_chain(classes) + ids = [id(s) for s in nodes] + wrs = [weakref.ref(s) for s in nodes] + del nodes + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors([]) + self.assertEqual([wr() for wr in wrs], [None] * N) + gc.collect() + self.assert_del_calls(ids) + + def check_resurrecting_chain(self, classes): + N = len(classes) + with SimpleBase.test(): + nodes = self.build_chain(classes) + N = len(nodes) + ids = [id(s) for s in nodes] + survivor_ids = [id(s) for s in nodes if isinstance(s, SimpleResurrector)] + wrs = [weakref.ref(s) for s in nodes] + del nodes + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors(survivor_ids) + # XXX desirable? + self.assertEqual([wr() for wr in wrs], [None] * N) + self.clear_survivors() + gc.collect() + self.assert_del_calls(ids) + self.assert_survivors([]) + + def test_homogenous(self): + self.check_non_resurrecting_chain([SimpleChained] * 3) + + def test_homogenous_resurrect(self): + self.check_resurrecting_chain([ChainedResurrector] * 3) + + def test_homogenous_suicidal(self): + self.check_non_resurrecting_chain([SuicidalChained] * 3) + + def test_heterogenous_suicidal_one(self): + self.check_non_resurrecting_chain([SuicidalChained, SimpleChained] * 2) + + def test_heterogenous_suicidal_two(self): + self.check_non_resurrecting_chain( + [SuicidalChained] * 2 + [SimpleChained] * 2) + + def test_heterogenous_resurrect_one(self): + self.check_resurrecting_chain([ChainedResurrector, SimpleChained] * 2) + + def test_heterogenous_resurrect_two(self): + self.check_resurrecting_chain( + [ChainedResurrector, SimpleChained, SuicidalChained] * 2) + + def test_heterogenous_resurrect_three(self): + self.check_resurrecting_chain( + [ChainedResurrector] * 2 + [SimpleChained] * 2 + [SuicidalChained] * 2) + + +# NOTE: the tp_del slot isn't automatically inherited, so we have to call +# with_tp_del() for each instantiated class. + +class LegacyBase(SimpleBase): + + def __del__(self): + try: + # Do not invoke side_effect here, since we are now exercising + # the tp_del slot. + if not self._cleaning: + self.del_calls.append(id(self)) + self.check_sanity() + except Exception as e: + self.errors.append(e) + + def __tp_del__(self): + """ + Legacy (pre-PEP 442) finalizer, mapped to a tp_del slot. + """ + try: + if not self._cleaning: + self.tp_del_calls.append(id(self)) + self.check_sanity() + self.side_effect() + except Exception as e: + self.errors.append(e) + +@_testcapi.with_tp_del +class Legacy(LegacyBase): + pass + +@_testcapi.with_tp_del +class LegacyResurrector(LegacyBase): + + def side_effect(self): + """ + Resurrect self by storing self in a class-wide list. + """ + self.survivors.append(self) + +@_testcapi.with_tp_del +class LegacySelfCycle(SelfCycleBase, LegacyBase): + pass + + +class LegacyFinalizationTest(TestBase, unittest.TestCase): + """ + Test finalization of objects with a tp_del. + """ + + def tearDown(self): + # These tests need to clean up a bit more, since they create + # uncollectable objects. + gc.garbage.clear() + gc.collect() + super().tearDown() + + def test_legacy(self): + with SimpleBase.test(): + s = Legacy() + ids = [id(s)] + wr = weakref.ref(s) + del s + gc.collect() + self.assert_del_calls(ids) + self.assert_tp_del_calls(ids) + self.assert_survivors([]) + self.assertIs(wr(), None) + gc.collect() + self.assert_del_calls(ids) + self.assert_tp_del_calls(ids) + + def test_legacy_resurrect(self): + with SimpleBase.test(): + s = LegacyResurrector() + ids = [id(s)] + wr = weakref.ref(s) + del s + gc.collect() + self.assert_del_calls(ids) + self.assert_tp_del_calls(ids) + self.assert_survivors(ids) + # weakrefs are cleared before tp_del is called. + self.assertIs(wr(), None) + self.clear_survivors() + gc.collect() + self.assert_del_calls(ids) + self.assert_tp_del_calls(ids * 2) + self.assert_survivors(ids) + self.assertIs(wr(), None) + + def test_legacy_self_cycle(self): + # Self-cycles with legacy finalizers end up in gc.garbage. + with SimpleBase.test(): + s = LegacySelfCycle() + ids = [id(s)] + wr = weakref.ref(s) + del s + gc.collect() + self.assert_del_calls([]) + self.assert_tp_del_calls([]) + self.assert_survivors([]) + self.assert_garbage(ids) + self.assertIsNot(wr(), None) + # Break the cycle to allow collection + gc.garbage[0].ref = None + self.assert_garbage([]) + self.assertIs(wr(), None) + + +def test_main(): + support.run_unittest(__name__) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_fork1.py b/Lib/test/test_fork1.py index 8192c38a44..e0626dffdc 100644 --- a/Lib/test/test_fork1.py +++ b/Lib/test/test_fork1.py @@ -1,7 +1,7 @@ """This test checks for correct fork() behavior. """ -import imp +import _imp as imp import os import signal import sys diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index bd159f5689..29330f9171 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -307,27 +307,41 @@ class FormatTest(unittest.TestCase): finally: locale.setlocale(locale.LC_ALL, oldloc) + @support.cpython_only + def test_optimisations(self): + text = "abcde" # 5 characters + self.assertIs("%s" % text, text) + self.assertIs("%.5s" % text, text) + self.assertIs("%.10s" % text, text) + self.assertIs("%1s" % text, text) + self.assertIs("%5s" % text, text) -def test_main(): - support.run_unittest(FormatTest) + self.assertIs("{0}".format(text), text) + self.assertIs("{0:s}".format(text), text) + self.assertIs("{0:.5s}".format(text), text) + self.assertIs("{0:.10s}".format(text), text) + self.assertIs("{0:1s}".format(text), text) + self.assertIs("{0:5s}".format(text), text) + self.assertIs(text % (), text) + self.assertIs(text.format(), text) + + @support.cpython_only def test_precision(self): - INT_MAX = 2147483647 + from _testcapi import INT_MAX f = 1.2 self.assertEqual(format(f, ".0f"), "1") self.assertEqual(format(f, ".3f"), "1.200") with self.assertRaises(ValueError) as cm: format(f, ".%sf" % (INT_MAX + 1)) - self.assertEqual(str(cm.exception), "precision too big") c = complex(f) - self.assertEqual(format(f, ".0f"), "1") - self.assertEqual(format(f, ".3f"), "1.200") + self.assertEqual(format(c, ".0f"), "1+0j") + self.assertEqual(format(c, ".3f"), "1.200+0.000j") with self.assertRaises(ValueError) as cm: - format(f, ".%sf" % (INT_MAX + 1)) - self.assertEqual(str(cm.exception), "precision too big") + format(c, ".%sf" % (INT_MAX + 1)) if __name__ == "__main__": diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 1fad9215c0..33365322b9 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -146,9 +146,10 @@ class FractionTest(unittest.TestCase): self.assertEqual((0, 1), _components(F(-0.0))) self.assertEqual((3602879701896397, 36028797018963968), _components(F(0.1))) - self.assertRaises(TypeError, F, float('nan')) - self.assertRaises(TypeError, F, float('inf')) - self.assertRaises(TypeError, F, float('-inf')) + # bug 16469: error types should be consistent with float -> int + self.assertRaises(ValueError, F, float('nan')) + self.assertRaises(OverflowError, F, float('inf')) + self.assertRaises(OverflowError, F, float('-inf')) def testInitFromDecimal(self): self.assertEqual((11, 10), @@ -157,10 +158,11 @@ class FractionTest(unittest.TestCase): _components(F(Decimal('3.5e-2')))) self.assertEqual((0, 1), _components(F(Decimal('.000e20')))) - self.assertRaises(TypeError, F, Decimal('nan')) - self.assertRaises(TypeError, F, Decimal('snan')) - self.assertRaises(TypeError, F, Decimal('inf')) - self.assertRaises(TypeError, F, Decimal('-inf')) + # bug 16469: error types should be consistent with decimal -> int + self.assertRaises(ValueError, F, Decimal('nan')) + self.assertRaises(ValueError, F, Decimal('snan')) + self.assertRaises(OverflowError, F, Decimal('inf')) + self.assertRaises(OverflowError, F, Decimal('-inf')) def testFromString(self): self.assertEqual((5, 1), _components(F("5"))) @@ -248,14 +250,15 @@ class FractionTest(unittest.TestCase): inf = 1e1000 nan = inf - inf + # bug 16469: error types should be consistent with float -> int self.assertRaisesMessage( - TypeError, "Cannot convert inf to Fraction.", + OverflowError, "Cannot convert inf to Fraction.", F.from_float, inf) self.assertRaisesMessage( - TypeError, "Cannot convert -inf to Fraction.", + OverflowError, "Cannot convert -inf to Fraction.", F.from_float, -inf) self.assertRaisesMessage( - TypeError, "Cannot convert nan to Fraction.", + ValueError, "Cannot convert nan to Fraction.", F.from_float, nan) def testFromDecimal(self): @@ -268,17 +271,18 @@ class FractionTest(unittest.TestCase): self.assertEqual(1 - F(1, 10**30), F.from_decimal(Decimal("0." + "9" * 30))) + # bug 16469: error types should be consistent with decimal -> int self.assertRaisesMessage( - TypeError, "Cannot convert Infinity to Fraction.", + OverflowError, "Cannot convert Infinity to Fraction.", F.from_decimal, Decimal("inf")) self.assertRaisesMessage( - TypeError, "Cannot convert -Infinity to Fraction.", + OverflowError, "Cannot convert -Infinity to Fraction.", F.from_decimal, Decimal("-inf")) self.assertRaisesMessage( - TypeError, "Cannot convert NaN to Fraction.", + ValueError, "Cannot convert NaN to Fraction.", F.from_decimal, Decimal("nan")) self.assertRaisesMessage( - TypeError, "Cannot convert sNaN to Fraction.", + ValueError, "Cannot convert sNaN to Fraction.", F.from_decimal, Decimal("snan")) def testLimitDenominator(self): diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py new file mode 100644 index 0000000000..2dd5780c88 --- /dev/null +++ b/Lib/test/test_frame.py @@ -0,0 +1,116 @@ +import gc +import sys +import unittest +import weakref + +from test import support + + +class ClearTest(unittest.TestCase): + """ + Tests for frame.clear(). + """ + + def inner(self, x=5, **kwargs): + 1/0 + + def outer(self, **kwargs): + try: + self.inner(**kwargs) + except ZeroDivisionError as e: + exc = e + return exc + + def clear_traceback_frames(self, tb): + """ + Clear all frames in a traceback. + """ + while tb is not None: + tb.tb_frame.clear() + tb = tb.tb_next + + def test_clear_locals(self): + class C: + pass + c = C() + wr = weakref.ref(c) + exc = self.outer(c=c) + del c + support.gc_collect() + # A reference to c is held through the frames + self.assertIsNot(None, wr()) + self.clear_traceback_frames(exc.__traceback__) + support.gc_collect() + # The reference was released by .clear() + self.assertIs(None, wr()) + + def test_clear_generator(self): + endly = False + def g(): + nonlocal endly + try: + yield + inner() + finally: + endly = True + gen = g() + next(gen) + self.assertFalse(endly) + # Clearing the frame closes the generator + gen.gi_frame.clear() + self.assertTrue(endly) + + def test_clear_executing(self): + # Attempting to clear an executing frame is forbidden. + try: + 1/0 + except ZeroDivisionError as e: + f = e.__traceback__.tb_frame + with self.assertRaises(RuntimeError): + f.clear() + with self.assertRaises(RuntimeError): + f.f_back.clear() + + def test_clear_executing_generator(self): + # Attempting to clear an executing generator frame is forbidden. + endly = False + def g(): + nonlocal endly + try: + 1/0 + except ZeroDivisionError as e: + f = e.__traceback__.tb_frame + with self.assertRaises(RuntimeError): + f.clear() + with self.assertRaises(RuntimeError): + f.f_back.clear() + yield f + finally: + endly = True + gen = g() + f = next(gen) + self.assertFalse(endly) + # Clearing the frame closes the generator + f.clear() + self.assertTrue(endly) + + @support.cpython_only + def test_clear_refcycles(self): + # .clear() doesn't leave any refcycle behind + with support.disable_gc(): + class C: + pass + c = C() + wr = weakref.ref(c) + exc = self.outer(c=c) + del c + self.assertIsNot(None, wr()) + self.clear_traceback_frames(exc.__traceback__) + self.assertIs(None, wr()) + + +def test_main(): + support.run_unittest(__name__) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_frozen.py b/Lib/test/test_frozen.py index fd6761c651..4c50cb7510 100644 --- a/Lib/test/test_frozen.py +++ b/Lib/test/test_frozen.py @@ -36,7 +36,7 @@ class FrozenTests(unittest.TestCase): else: expect.add('spam') self.assertEqual(set(dir(__phello__)), expect) - self.assertEqual(__phello__.__path__, [__phello__.__name__]) + self.assertEqual(__phello__.__path__, []) self.assertEqual(stdout.getvalue(), 'Hello world!\n') with captured_stdout() as stdout: @@ -72,8 +72,6 @@ class FrozenTests(unittest.TestCase): del sys.modules['__phello__'] del sys.modules['__phello__.spam'] -def test_main(): - run_unittest(FrozenTests) if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 2c45af3145..47563c806a 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -21,6 +21,7 @@ from test import support from test.support import HOST, HOSTv6 threading = support.import_module('threading') +TIMEOUT = 3 # the dummy data returned by server over the data channel when # RETR, LIST, NLST, MLSD commands are issued RETR_DATA = 'abcde12345\r\n' * 1000 @@ -125,7 +126,7 @@ class DummyFTPHandler(asynchat.async_chat): addr = list(map(int, arg.split(','))) ip = '%d.%d.%d.%d' %tuple(addr[:4]) port = (addr[4] * 256) + addr[5] - s = socket.create_connection((ip, port), timeout=2) + s = socket.create_connection((ip, port), timeout=TIMEOUT) self.dtp = self.dtp_handler(s, baseclass=self) self.push('200 active data connection established') @@ -133,7 +134,7 @@ class DummyFTPHandler(asynchat.async_chat): with socket.socket() as sock: sock.bind((self.socket.getsockname()[0], 0)) sock.listen(5) - sock.settimeout(10) + sock.settimeout(TIMEOUT) ip, port = sock.getsockname()[:2] ip = ip.replace('.', ','); p1 = port / 256; p2 = port % 256 self.push('227 entering passive mode (%s,%d,%d)' %(ip, p1, p2)) @@ -143,7 +144,7 @@ class DummyFTPHandler(asynchat.async_chat): def cmd_eprt(self, arg): af, ip, port = arg.split(arg[0])[1:-1] port = int(port) - s = socket.create_connection((ip, port), timeout=2) + s = socket.create_connection((ip, port), timeout=TIMEOUT) self.dtp = self.dtp_handler(s, baseclass=self) self.push('200 active data connection established') @@ -151,7 +152,7 @@ class DummyFTPHandler(asynchat.async_chat): with socket.socket(socket.AF_INET6) as sock: sock.bind((self.socket.getsockname()[0], 0)) sock.listen(5) - sock.settimeout(10) + sock.settimeout(TIMEOUT) port = sock.getsockname()[1] self.push('229 entering extended passive mode (|||%d|)' %port) conn, addr = sock.accept() @@ -321,7 +322,7 @@ if ssl is not None: elif err.args[0] == ssl.SSL_ERROR_EOF: return self.handle_close() raise - except socket.error as err: + except OSError as err: if err.args[0] == errno.ECONNABORTED: return self.handle_close() else: @@ -335,7 +336,7 @@ if ssl is not None: if err.args[0] in (ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE): return - except socket.error as err: + except OSError as err: # Any "socket error" corresponds to a SSL_ERROR_SYSCALL return # from OpenSSL's SSL_shutdown(), corresponding to a # closed socket condition. See also: @@ -454,7 +455,7 @@ class TestFTPClass(TestCase): def setUp(self): self.server = DummyFTPServer((HOST, 0)) self.server.start() - self.client = ftplib.FTP(timeout=2) + self.client = ftplib.FTP(timeout=TIMEOUT) self.client.connect(self.server.host, self.server.port) def tearDown(self): @@ -482,7 +483,7 @@ class TestFTPClass(TestCase): def test_all_errors(self): exceptions = (ftplib.error_reply, ftplib.error_temp, ftplib.error_perm, - ftplib.error_proto, ftplib.Error, IOError, EOFError) + ftplib.error_proto, ftplib.Error, OSError, EOFError) for x in exceptions: try: raise x('exception not included in all_errors set') @@ -672,7 +673,7 @@ class TestFTPClass(TestCase): def test_makepasv(self): host, port = self.client.makepasv() - conn = socket.create_connection((host, port), 10) + conn = socket.create_connection((host, port), timeout=TIMEOUT) conn.close() # IPv4 is in use, just make sure send_epsv has not been used self.assertEqual(self.server.handler_instance.last_received_cmd, 'pasv') @@ -685,12 +686,12 @@ class TestFTPClass(TestCase): return False try: self.client.sendcmd('noop') - except (socket.error, EOFError): + except (OSError, EOFError): return False return True # base test - with ftplib.FTP(timeout=2) as self.client: + with ftplib.FTP(timeout=TIMEOUT) as self.client: self.client.connect(self.server.host, self.server.port) self.client.sendcmd('noop') self.assertTrue(is_client_connected()) @@ -698,7 +699,7 @@ class TestFTPClass(TestCase): self.assertFalse(is_client_connected()) # QUIT sent inside the with block - with ftplib.FTP(timeout=2) as self.client: + with ftplib.FTP(timeout=TIMEOUT) as self.client: self.client.connect(self.server.host, self.server.port) self.client.sendcmd('noop') self.client.quit() @@ -708,7 +709,7 @@ class TestFTPClass(TestCase): # force a wrong response code to be sent on QUIT: error_perm # is expected and the connection is supposed to be closed try: - with ftplib.FTP(timeout=2) as self.client: + with ftplib.FTP(timeout=TIMEOUT) as self.client: self.client.connect(self.server.host, self.server.port) self.client.sendcmd('noop') self.server.handler_instance.next_response = '550 error on quit' @@ -730,7 +731,7 @@ class TestFTPClass(TestCase): source_address=(HOST, port)) self.assertEqual(self.client.sock.getsockname()[1], port) self.client.quit() - except IOError as e: + except OSError as e: if e.errno == errno.EADDRINUSE: self.skipTest("couldn't bind to port %d" % port) raise @@ -741,7 +742,7 @@ class TestFTPClass(TestCase): try: with self.client.transfercmd('list') as sock: self.assertEqual(sock.getsockname()[1], port) - except IOError as e: + except OSError as e: if e.errno == errno.EADDRINUSE: self.skipTest("couldn't bind to port %d" % port) raise @@ -764,7 +765,7 @@ class TestIPv6Environment(TestCase): def setUp(self): self.server = DummyFTPServer((HOSTv6, 0), af=socket.AF_INET6) self.server.start() - self.client = ftplib.FTP() + self.client = ftplib.FTP(timeout=TIMEOUT) self.client.connect(self.server.host, self.server.port) def tearDown(self): @@ -781,7 +782,7 @@ class TestIPv6Environment(TestCase): def test_makepasv(self): host, port = self.client.makepasv() - conn = socket.create_connection((host, port), 10) + conn = socket.create_connection((host, port), timeout=TIMEOUT) conn.close() self.assertEqual(self.server.handler_instance.last_received_cmd, 'epsv') @@ -807,7 +808,7 @@ class TestTLS_FTPClassMixin(TestFTPClass): def setUp(self): self.server = DummyTLS_FTPServer((HOST, 0)) self.server.start() - self.client = ftplib.FTP_TLS(timeout=2) + self.client = ftplib.FTP_TLS(timeout=TIMEOUT) self.client.connect(self.server.host, self.server.port) # enable TLS self.client.auth() @@ -820,7 +821,7 @@ class TestTLS_FTPClass(TestCase): def setUp(self): self.server = DummyTLS_FTPServer((HOST, 0)) self.server.start() - self.client = ftplib.FTP_TLS(timeout=2) + self.client = ftplib.FTP_TLS(timeout=TIMEOUT) self.client.connect(self.server.host, self.server.port) def tearDown(self): @@ -880,7 +881,7 @@ class TestTLS_FTPClass(TestCase): self.assertRaises(ValueError, ftplib.FTP_TLS, certfile=CERTFILE, keyfile=CERTFILE, context=ctx) - self.client = ftplib.FTP_TLS(context=ctx, timeout=2) + self.client = ftplib.FTP_TLS(context=ctx, timeout=TIMEOUT) self.client.connect(self.server.host, self.server.port) self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) self.client.auth() @@ -994,8 +995,19 @@ class TestTimeouts(TestCase): ftp.close() +class TestNetrcDeprecation(TestCase): + + def test_deprecation(self): + with support.temp_cwd(), support.EnvironmentVarGuard() as env: + env['HOME'] = os.getcwd() + open('.netrc', 'w').close() + with self.assertWarns(DeprecationWarning): + ftplib.Netrc() + + + def test_main(): - tests = [TestFTPClass, TestTimeouts] + tests = [TestFTPClass, TestTimeouts, TestNetrcDeprecation] if support.IPV6_ENABLED: tests.append(TestIPv6Environment) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index db1e9348dd..cb493bff52 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1,57 +1,54 @@ -import functools import collections +from itertools import permutations +import pickle +from random import choice import sys -import unittest from test import support +import unittest from weakref import proxy -import pickle -from random import choice -@staticmethod -def PythonPartial(func, *args, **keywords): - 'Pure Python approximation of partial()' - def newfunc(*fargs, **fkeywords): - newkeywords = keywords.copy() - newkeywords.update(fkeywords) - return func(*(args + fargs), **newkeywords) - newfunc.func = func - newfunc.args = args - newfunc.keywords = keywords - return newfunc +import functools + +py_functools = support.import_fresh_module('functools', blocked=['_functools']) +c_functools = support.import_fresh_module('functools', fresh=['_functools']) + +decimal = support.import_fresh_module('decimal', fresh=['_decimal']) + def capture(*args, **kw): """capture all positional and keyword arguments""" return args, kw + def signature(part): """ return the signature of a partial object """ return (part.func, part.args, part.keywords, part.__dict__) -class TestPartial(unittest.TestCase): - thetype = functools.partial +class TestPartial: def test_basic_examples(self): - p = self.thetype(capture, 1, 2, a=10, b=20) + p = self.partial(capture, 1, 2, a=10, b=20) + self.assertTrue(callable(p)) self.assertEqual(p(3, 4, b=30, c=40), ((1, 2, 3, 4), dict(a=10, b=30, c=40))) - p = self.thetype(map, lambda x: x*10) + p = self.partial(map, lambda x: x*10) self.assertEqual(list(p([1,2,3,4])), [10, 20, 30, 40]) def test_attributes(self): - p = self.thetype(capture, 1, 2, a=10, b=20) + p = self.partial(capture, 1, 2, a=10, b=20) # attributes should be readable self.assertEqual(p.func, capture) self.assertEqual(p.args, (1, 2)) self.assertEqual(p.keywords, dict(a=10, b=20)) # attributes should not be writable - if not isinstance(self.thetype, type): + if not isinstance(self.partial, type): return self.assertRaises(AttributeError, setattr, p, 'func', map) self.assertRaises(AttributeError, setattr, p, 'args', (1, 2)) self.assertRaises(AttributeError, setattr, p, 'keywords', dict(a=1, b=2)) - p = self.thetype(hex) + p = self.partial(hex) try: del p.__dict__ except TypeError: @@ -60,9 +57,9 @@ class TestPartial(unittest.TestCase): self.fail('partial object allowed __dict__ to be deleted') def test_argument_checking(self): - self.assertRaises(TypeError, self.thetype) # need at least a func arg + self.assertRaises(TypeError, self.partial) # need at least a func arg try: - self.thetype(2)() + self.partial(2)() except TypeError: pass else: @@ -73,7 +70,7 @@ class TestPartial(unittest.TestCase): def func(a=10, b=20): return a d = {'a':3} - p = self.thetype(func, a=5) + p = self.partial(func, a=5) self.assertEqual(p(**d), 3) self.assertEqual(d, {'a':3}) p(b=7) @@ -82,20 +79,20 @@ class TestPartial(unittest.TestCase): def test_arg_combinations(self): # exercise special code paths for zero args in either partial # object or the caller - p = self.thetype(capture) + p = self.partial(capture) self.assertEqual(p(), ((), {})) self.assertEqual(p(1,2), ((1,2), {})) - p = self.thetype(capture, 1, 2) + p = self.partial(capture, 1, 2) self.assertEqual(p(), ((1,2), {})) self.assertEqual(p(3,4), ((1,2,3,4), {})) def test_kw_combinations(self): # exercise special code paths for no keyword args in # either the partial object or the caller - p = self.thetype(capture) + p = self.partial(capture) self.assertEqual(p(), ((), {})) self.assertEqual(p(a=1), ((), {'a':1})) - p = self.thetype(capture, a=1) + p = self.partial(capture, a=1) self.assertEqual(p(), ((), {'a':1})) self.assertEqual(p(b=2), ((), {'a':1, 'b':2})) # keyword args in the call override those in the partial object @@ -104,7 +101,7 @@ class TestPartial(unittest.TestCase): def test_positional(self): # make sure positional arguments are captured correctly for args in [(), (0,), (0,1), (0,1,2), (0,1,2,3)]: - p = self.thetype(capture, *args) + p = self.partial(capture, *args) expected = args + ('x',) got, empty = p('x') self.assertTrue(expected == got and empty == {}) @@ -112,14 +109,14 @@ class TestPartial(unittest.TestCase): def test_keyword(self): # make sure keyword arguments are captured correctly for a in ['a', 0, None, 3.5]: - p = self.thetype(capture, a=a) + p = self.partial(capture, a=a) expected = {'a':a,'x':None} empty, got = p(x=None) self.assertTrue(expected == got and empty == ()) def test_no_side_effects(self): # make sure there are no side effects that affect subsequent calls - p = self.thetype(capture, 0, a=1) + p = self.partial(capture, 0, a=1) args1, kw1 = p(1, b=2) self.assertTrue(args1 == (0,1) and kw1 == {'a':1,'b':2}) args2, kw2 = p() @@ -128,13 +125,13 @@ class TestPartial(unittest.TestCase): def test_error_propagation(self): def f(x, y): x / y - self.assertRaises(ZeroDivisionError, self.thetype(f, 1, 0)) - self.assertRaises(ZeroDivisionError, self.thetype(f, 1), 0) - self.assertRaises(ZeroDivisionError, self.thetype(f), 1, 0) - self.assertRaises(ZeroDivisionError, self.thetype(f, y=0), 1) + self.assertRaises(ZeroDivisionError, self.partial(f, 1, 0)) + self.assertRaises(ZeroDivisionError, self.partial(f, 1), 0) + self.assertRaises(ZeroDivisionError, self.partial(f), 1, 0) + self.assertRaises(ZeroDivisionError, self.partial(f, y=0), 1) def test_weakref(self): - f = self.thetype(int, base=16) + f = self.partial(int, base=16) p = proxy(f) self.assertEqual(f.func, p.func) f = None @@ -142,39 +139,45 @@ class TestPartial(unittest.TestCase): def test_with_bound_and_unbound_methods(self): data = list(map(str, range(10))) - join = self.thetype(str.join, '') + join = self.partial(str.join, '') self.assertEqual(join(data), '0123456789') - join = self.thetype(''.join) + join = self.partial(''.join) self.assertEqual(join(data), '0123456789') + +@unittest.skipUnless(c_functools, 'requires the C _functools module') +class TestPartialC(TestPartial, unittest.TestCase): + if c_functools: + partial = c_functools.partial + def test_repr(self): args = (object(), object()) args_repr = ', '.join(repr(a) for a in args) kwargs = {'a': object(), 'b': object()} kwargs_repr = ', '.join("%s=%r" % (k, v) for k, v in kwargs.items()) - if self.thetype is functools.partial: + if self.partial is c_functools.partial: name = 'functools.partial' else: - name = self.thetype.__name__ + name = self.partial.__name__ - f = self.thetype(capture) + f = self.partial(capture) self.assertEqual('{}({!r})'.format(name, capture), repr(f)) - f = self.thetype(capture, *args) + f = self.partial(capture, *args) self.assertEqual('{}({!r}, {})'.format(name, capture, args_repr), repr(f)) - f = self.thetype(capture, **kwargs) + f = self.partial(capture, **kwargs) self.assertEqual('{}({!r}, {})'.format(name, capture, kwargs_repr), repr(f)) - f = self.thetype(capture, *args, **kwargs) + f = self.partial(capture, *args, **kwargs) self.assertEqual('{}({!r}, {}, {})'.format(name, capture, args_repr, kwargs_repr), repr(f)) def test_pickle(self): - f = self.thetype(signature, 'asdf', bar=True) + f = self.partial(signature, 'asdf', bar=True) f.add_something_to__dict__ = True f_copy = pickle.loads(pickle.dumps(f)) self.assertEqual(signature(f), signature(f_copy)) @@ -193,28 +196,26 @@ class TestPartial(unittest.TestCase): return {} raise IndexError - f = self.thetype(object) + f = self.partial(object) self.assertRaisesRegex(SystemError, "new style getargs format but argument is not a tuple", f.__setstate__, BadSequence()) -class PartialSubclass(functools.partial): - pass -class TestPartialSubclass(TestPartial): +class TestPartialPy(TestPartial, unittest.TestCase): + partial = staticmethod(py_functools.partial) - thetype = PartialSubclass -class TestPythonPartial(TestPartial): +if c_functools: + class PartialSubclass(c_functools.partial): + pass - thetype = PythonPartial - # the python version hasn't a nice repr - def test_repr(self): pass +@unittest.skipUnless(c_functools, 'requires the C _functools module') +class TestPartialCSubclass(TestPartialC): + if c_functools: + partial = PartialSubclass - # the python version isn't picklable - def test_pickle(self): pass - def test_setstate_refcount(self): pass class TestUpdateWrapper(unittest.TestCase): @@ -223,19 +224,26 @@ class TestUpdateWrapper(unittest.TestCase): updated=functools.WRAPPER_UPDATES): # Check attributes were assigned for name in assigned: - self.assertTrue(getattr(wrapper, name) is getattr(wrapped, name)) + self.assertIs(getattr(wrapper, name), getattr(wrapped, name)) # Check attributes were updated for name in updated: wrapper_attr = getattr(wrapper, name) wrapped_attr = getattr(wrapped, name) for key in wrapped_attr: - self.assertTrue(wrapped_attr[key] is wrapper_attr[key]) + if name == "__dict__" and key == "__wrapped__": + # __wrapped__ is overwritten by the update code + continue + self.assertIs(wrapped_attr[key], wrapper_attr[key]) + # Check __wrapped__ + self.assertIs(wrapper.__wrapped__, wrapped) + def _default_update(self): def f(a:'This is a new annotation'): """This is a test""" pass f.attr = 'This is also a test' + f.__wrapped__ = "This is a bald faced lie" def wrapper(b:'This is the prior annotation'): pass functools.update_wrapper(wrapper, f) @@ -322,6 +330,7 @@ class TestUpdateWrapper(unittest.TestCase): self.assertTrue(wrapper.__doc__.startswith('max(')) self.assertEqual(wrapper.__annotations__, {}) + class TestWraps(TestUpdateWrapper): def _default_update(self): @@ -329,14 +338,15 @@ class TestWraps(TestUpdateWrapper): """This is a test""" pass f.attr = 'This is also a test' + f.__wrapped__ = "This is still a bald faced lie" @functools.wraps(f) def wrapper(): pass - self.check_wrapper(wrapper, f) return wrapper, f def test_default_update(self): wrapper, f = self._default_update() + self.check_wrapper(wrapper, f) self.assertEqual(wrapper.__name__, 'f') self.assertEqual(wrapper.__qualname__, f.__qualname__) self.assertEqual(wrapper.attr, 'This is also a test') @@ -382,6 +392,7 @@ class TestWraps(TestUpdateWrapper): self.assertEqual(wrapper.attr, 'This is a different test') self.assertEqual(wrapper.dict_attr, f.dict_attr) + class TestReduce(unittest.TestCase): func = functools.reduce @@ -462,24 +473,29 @@ class TestReduce(unittest.TestCase): d = {"one": 1, "two": 2, "three": 3} self.assertEqual(self.func(add, d), "".join(d.keys())) -class TestCmpToKey(unittest.TestCase): + +class TestCmpToKey: def test_cmp_to_key(self): def cmp1(x, y): return (x > y) - (x < y) - key = functools.cmp_to_key(cmp1) + key = self.cmp_to_key(cmp1) self.assertEqual(key(3), key(3)) self.assertGreater(key(3), key(1)) + self.assertGreaterEqual(key(3), key(3)) + def cmp2(x, y): return int(x) - int(y) - key = functools.cmp_to_key(cmp2) + key = self.cmp_to_key(cmp2) self.assertEqual(key(4.0), key('4')) self.assertLess(key(2), key('35')) + self.assertLessEqual(key(2), key('35')) + self.assertNotEqual(key(2), key('35')) def test_cmp_to_key_arguments(self): def cmp1(x, y): return (x > y) - (x < y) - key = functools.cmp_to_key(mycmp=cmp1) + key = self.cmp_to_key(mycmp=cmp1) self.assertEqual(key(obj=3), key(obj=3)) self.assertGreater(key(obj=3), key(obj=1)) with self.assertRaises((TypeError, AttributeError)): @@ -487,10 +503,10 @@ class TestCmpToKey(unittest.TestCase): with self.assertRaises((TypeError, AttributeError)): 1 < key(3) # lhs is not a K object with self.assertRaises(TypeError): - key = functools.cmp_to_key() # too few args + key = self.cmp_to_key() # too few args with self.assertRaises(TypeError): - key = functools.cmp_to_key(cmp1, None) # too many args - key = functools.cmp_to_key(cmp1) + key = self.cmp_to_key(cmp1, None) # too many args + key = self.cmp_to_key(cmp1) with self.assertRaises(TypeError): key() # too few args with self.assertRaises(TypeError): @@ -499,7 +515,7 @@ class TestCmpToKey(unittest.TestCase): def test_bad_cmp(self): def cmp1(x, y): raise ZeroDivisionError - key = functools.cmp_to_key(cmp1) + key = self.cmp_to_key(cmp1) with self.assertRaises(ZeroDivisionError): key(3) > key(1) @@ -514,13 +530,13 @@ class TestCmpToKey(unittest.TestCase): def test_obj_field(self): def cmp1(x, y): return (x > y) - (x < y) - key = functools.cmp_to_key(mycmp=cmp1) + key = self.cmp_to_key(mycmp=cmp1) self.assertEqual(key(50).obj, 50) def test_sort_int(self): def mycmp(x, y): return y - x - self.assertEqual(sorted(range(5), key=functools.cmp_to_key(mycmp)), + self.assertEqual(sorted(range(5), key=self.cmp_to_key(mycmp)), [4, 3, 2, 1, 0]) def test_sort_int_str(self): @@ -528,18 +544,29 @@ class TestCmpToKey(unittest.TestCase): x, y = int(x), int(y) return (x > y) - (x < y) values = [5, '3', 7, 2, '0', '1', 4, '10', 1] - values = sorted(values, key=functools.cmp_to_key(mycmp)) + values = sorted(values, key=self.cmp_to_key(mycmp)) self.assertEqual([int(value) for value in values], [0, 1, 1, 2, 3, 4, 5, 7, 10]) def test_hash(self): def mycmp(x, y): return y - x - key = functools.cmp_to_key(mycmp) + key = self.cmp_to_key(mycmp) k = key(10) self.assertRaises(TypeError, hash, k) self.assertNotIsInstance(k, collections.Hashable) + +@unittest.skipUnless(c_functools, 'requires the C _functools module') +class TestCmpToKeyC(TestCmpToKey, unittest.TestCase): + if c_functools: + cmp_to_key = c_functools.cmp_to_key + + +class TestCmpToKeyPy(TestCmpToKey, unittest.TestCase): + cmp_to_key = staticmethod(py_functools.cmp_to_key) + + class TestTotalOrdering(unittest.TestCase): def test_total_ordering_lt(self): @@ -557,6 +584,7 @@ class TestTotalOrdering(unittest.TestCase): self.assertTrue(A(2) >= A(1)) self.assertTrue(A(2) <= A(2)) self.assertTrue(A(2) >= A(2)) + self.assertFalse(A(1) > A(2)) def test_total_ordering_le(self): @functools.total_ordering @@ -573,6 +601,7 @@ class TestTotalOrdering(unittest.TestCase): self.assertTrue(A(2) >= A(1)) self.assertTrue(A(2) <= A(2)) self.assertTrue(A(2) >= A(2)) + self.assertFalse(A(1) >= A(2)) def test_total_ordering_gt(self): @functools.total_ordering @@ -589,6 +618,7 @@ class TestTotalOrdering(unittest.TestCase): self.assertTrue(A(2) >= A(1)) self.assertTrue(A(2) <= A(2)) self.assertTrue(A(2) >= A(2)) + self.assertFalse(A(2) < A(1)) def test_total_ordering_ge(self): @functools.total_ordering @@ -605,6 +635,7 @@ class TestTotalOrdering(unittest.TestCase): self.assertTrue(A(2) >= A(1)) self.assertTrue(A(2) <= A(2)) self.assertTrue(A(2) >= A(2)) + self.assertFalse(A(2) <= A(1)) def test_total_ordering_no_overwrite(self): # new methods should not overwrite existing @@ -624,27 +655,118 @@ class TestTotalOrdering(unittest.TestCase): class A: pass - def test_bug_10042(self): + def test_type_error_when_not_implemented(self): + # bug 10042; ensure stack overflow does not occur + # when decorated types return NotImplemented @functools.total_ordering - class TestTO: + class ImplementsLessThan: def __init__(self, value): self.value = value def __eq__(self, other): - if isinstance(other, TestTO): + if isinstance(other, ImplementsLessThan): return self.value == other.value return False def __lt__(self, other): - if isinstance(other, TestTO): + if isinstance(other, ImplementsLessThan): return self.value < other.value - raise TypeError - with self.assertRaises(TypeError): - TestTO(8) <= () + return NotImplemented + + @functools.total_ordering + class ImplementsGreaterThan: + def __init__(self, value): + self.value = value + def __eq__(self, other): + if isinstance(other, ImplementsGreaterThan): + return self.value == other.value + return False + def __gt__(self, other): + if isinstance(other, ImplementsGreaterThan): + return self.value > other.value + return NotImplemented + + @functools.total_ordering + class ImplementsLessThanEqualTo: + def __init__(self, value): + self.value = value + def __eq__(self, other): + if isinstance(other, ImplementsLessThanEqualTo): + return self.value == other.value + return False + def __le__(self, other): + if isinstance(other, ImplementsLessThanEqualTo): + return self.value <= other.value + return NotImplemented + + @functools.total_ordering + class ImplementsGreaterThanEqualTo: + def __init__(self, value): + self.value = value + def __eq__(self, other): + if isinstance(other, ImplementsGreaterThanEqualTo): + return self.value == other.value + return False + def __ge__(self, other): + if isinstance(other, ImplementsGreaterThanEqualTo): + return self.value >= other.value + return NotImplemented + + @functools.total_ordering + class ComparatorNotImplemented: + def __init__(self, value): + self.value = value + def __eq__(self, other): + if isinstance(other, ComparatorNotImplemented): + return self.value == other.value + return False + def __lt__(self, other): + return NotImplemented + + with self.subTest("LT < 1"), self.assertRaises(TypeError): + ImplementsLessThan(-1) < 1 + + with self.subTest("LT < LE"), self.assertRaises(TypeError): + ImplementsLessThan(0) < ImplementsLessThanEqualTo(0) + + with self.subTest("LT < GT"), self.assertRaises(TypeError): + ImplementsLessThan(1) < ImplementsGreaterThan(1) + + with self.subTest("LE <= LT"), self.assertRaises(TypeError): + ImplementsLessThanEqualTo(2) <= ImplementsLessThan(2) + + with self.subTest("LE <= GE"), self.assertRaises(TypeError): + ImplementsLessThanEqualTo(3) <= ImplementsGreaterThanEqualTo(3) + + with self.subTest("GT > GE"), self.assertRaises(TypeError): + ImplementsGreaterThan(4) > ImplementsGreaterThanEqualTo(4) + + with self.subTest("GT > LT"), self.assertRaises(TypeError): + ImplementsGreaterThan(5) > ImplementsLessThan(5) + + with self.subTest("GE >= GT"), self.assertRaises(TypeError): + ImplementsGreaterThanEqualTo(6) >= ImplementsGreaterThan(6) + + with self.subTest("GE >= LE"), self.assertRaises(TypeError): + ImplementsGreaterThanEqualTo(7) >= ImplementsLessThanEqualTo(7) + + with self.subTest("GE when equal"): + a = ComparatorNotImplemented(8) + b = ComparatorNotImplemented(8) + self.assertEqual(a, b) + with self.assertRaises(TypeError): + a >= b + + with self.subTest("LE when equal"): + a = ComparatorNotImplemented(9) + b = ComparatorNotImplemented(9) + self.assertEqual(a, b) + with self.assertRaises(TypeError): + a <= b class TestLRU(unittest.TestCase): def test_lru(self): def orig(x, y): - return 3*x+y + return 3 * x + y f = functools.lru_cache(maxsize=20)(orig) hits, misses, maxsize, currsize = f.cache_info() self.assertEqual(maxsize, 20) @@ -749,7 +871,7 @@ class TestLRU(unittest.TestCase): # Verify that user_function exceptions get passed through without # creating a hard-to-read chained exception. # http://bugs.python.org/issue13177 - for maxsize in (None, 100): + for maxsize in (None, 128): @functools.lru_cache(maxsize) def func(i): return 'abc'[i] @@ -762,7 +884,7 @@ class TestLRU(unittest.TestCase): func(15) def test_lru_with_types(self): - for maxsize in (None, 100): + for maxsize in (None, 128): @functools.lru_cache(maxsize=maxsize, typed=True) def square(x): return x * x @@ -777,6 +899,36 @@ class TestLRU(unittest.TestCase): self.assertEqual(square.cache_info().hits, 4) self.assertEqual(square.cache_info().misses, 4) + def test_lru_with_keyword_args(self): + @functools.lru_cache() + def fib(n): + if n < 2: + return n + return fib(n=n-1) + fib(n=n-2) + self.assertEqual( + [fib(n=number) for number in range(16)], + [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610] + ) + self.assertEqual(fib.cache_info(), + functools._CacheInfo(hits=28, misses=16, maxsize=128, currsize=16)) + fib.cache_clear() + self.assertEqual(fib.cache_info(), + functools._CacheInfo(hits=0, misses=0, maxsize=128, currsize=0)) + + def test_lru_with_keyword_args_maxsize_none(self): + @functools.lru_cache(maxsize=None) + def fib(n): + if n < 2: + return n + return fib(n=n-1) + fib(n=n-2) + self.assertEqual([fib(n=number) for number in range(16)], + [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610]) + self.assertEqual(fib.cache_info(), + functools._CacheInfo(hits=28, misses=16, maxsize=None, currsize=16)) + fib.cache_clear() + self.assertEqual(fib.cache_info(), + functools._CacheInfo(hits=0, misses=0, maxsize=None, currsize=0)) + def test_need_for_rlock(self): # This will deadlock on an LRU cache that uses a regular lock @@ -802,17 +954,493 @@ class TestLRU(unittest.TestCase): DoubleEq(2)) # Verify the correct return value +class TestSingleDispatch(unittest.TestCase): + def test_simple_overloads(self): + @functools.singledispatch + def g(obj): + return "base" + def g_int(i): + return "integer" + g.register(int, g_int) + self.assertEqual(g("str"), "base") + self.assertEqual(g(1), "integer") + self.assertEqual(g([1,2,3]), "base") + + def test_mro(self): + @functools.singledispatch + def g(obj): + return "base" + class A: + pass + class C(A): + pass + class B(A): + pass + class D(C, B): + pass + def g_A(a): + return "A" + def g_B(b): + return "B" + g.register(A, g_A) + g.register(B, g_B) + self.assertEqual(g(A()), "A") + self.assertEqual(g(B()), "B") + self.assertEqual(g(C()), "A") + self.assertEqual(g(D()), "B") + + def test_register_decorator(self): + @functools.singledispatch + def g(obj): + return "base" + @g.register(int) + def g_int(i): + return "int %s" % (i,) + self.assertEqual(g(""), "base") + self.assertEqual(g(12), "int 12") + self.assertIs(g.dispatch(int), g_int) + self.assertIs(g.dispatch(object), g.dispatch(str)) + # Note: in the assert above this is not g. + # @singledispatch returns the wrapper. + + def test_wrapping_attributes(self): + @functools.singledispatch + def g(obj): + "Simple test" + return "Test" + self.assertEqual(g.__name__, "g") + self.assertEqual(g.__doc__, "Simple test") + + @unittest.skipUnless(decimal, 'requires _decimal') + @support.cpython_only + def test_c_classes(self): + @functools.singledispatch + def g(obj): + return "base" + @g.register(decimal.DecimalException) + def _(obj): + return obj.args + subn = decimal.Subnormal("Exponent < Emin") + rnd = decimal.Rounded("Number got rounded") + self.assertEqual(g(subn), ("Exponent < Emin",)) + self.assertEqual(g(rnd), ("Number got rounded",)) + @g.register(decimal.Subnormal) + def _(obj): + return "Too small to care." + self.assertEqual(g(subn), "Too small to care.") + self.assertEqual(g(rnd), ("Number got rounded",)) + + def test_compose_mro(self): + # None of the examples in this test depend on haystack ordering. + c = collections + mro = functools._compose_mro + bases = [c.Sequence, c.MutableMapping, c.Mapping, c.Set] + for haystack in permutations(bases): + m = mro(dict, haystack) + self.assertEqual(m, [dict, c.MutableMapping, c.Mapping, c.Sized, + c.Iterable, c.Container, object]) + bases = [c.Container, c.Mapping, c.MutableMapping, c.OrderedDict] + for haystack in permutations(bases): + m = mro(c.ChainMap, haystack) + self.assertEqual(m, [c.ChainMap, c.MutableMapping, c.Mapping, + c.Sized, c.Iterable, c.Container, object]) + + # If there's a generic function with implementations registered for + # both Sized and Container, passing a defaultdict to it results in an + # ambiguous dispatch which will cause a RuntimeError (see + # test_mro_conflicts). + bases = [c.Container, c.Sized, str] + for haystack in permutations(bases): + m = mro(c.defaultdict, [c.Sized, c.Container, str]) + self.assertEqual(m, [c.defaultdict, dict, c.Sized, c.Container, + object]) + + # MutableSequence below is registered directly on D. In other words, it + # preceeds MutableMapping which means single dispatch will always + # choose MutableSequence here. + class D(c.defaultdict): + pass + c.MutableSequence.register(D) + bases = [c.MutableSequence, c.MutableMapping] + for haystack in permutations(bases): + m = mro(D, bases) + self.assertEqual(m, [D, c.MutableSequence, c.Sequence, + c.defaultdict, dict, c.MutableMapping, + c.Mapping, c.Sized, c.Iterable, c.Container, + object]) + + # Container and Callable are registered on different base classes and + # a generic function supporting both should always pick the Callable + # implementation if a C instance is passed. + class C(c.defaultdict): + def __call__(self): + pass + bases = [c.Sized, c.Callable, c.Container, c.Mapping] + for haystack in permutations(bases): + m = mro(C, haystack) + self.assertEqual(m, [C, c.Callable, c.defaultdict, dict, c.Mapping, + c.Sized, c.Iterable, c.Container, object]) + + def test_register_abc(self): + c = collections + d = {"a": "b"} + l = [1, 2, 3] + s = {object(), None} + f = frozenset(s) + t = (1, 2, 3) + @functools.singledispatch + def g(obj): + return "base" + self.assertEqual(g(d), "base") + self.assertEqual(g(l), "base") + self.assertEqual(g(s), "base") + self.assertEqual(g(f), "base") + self.assertEqual(g(t), "base") + g.register(c.Sized, lambda obj: "sized") + self.assertEqual(g(d), "sized") + self.assertEqual(g(l), "sized") + self.assertEqual(g(s), "sized") + self.assertEqual(g(f), "sized") + self.assertEqual(g(t), "sized") + g.register(c.MutableMapping, lambda obj: "mutablemapping") + self.assertEqual(g(d), "mutablemapping") + self.assertEqual(g(l), "sized") + self.assertEqual(g(s), "sized") + self.assertEqual(g(f), "sized") + self.assertEqual(g(t), "sized") + g.register(c.ChainMap, lambda obj: "chainmap") + self.assertEqual(g(d), "mutablemapping") # irrelevant ABCs registered + self.assertEqual(g(l), "sized") + self.assertEqual(g(s), "sized") + self.assertEqual(g(f), "sized") + self.assertEqual(g(t), "sized") + g.register(c.MutableSequence, lambda obj: "mutablesequence") + self.assertEqual(g(d), "mutablemapping") + self.assertEqual(g(l), "mutablesequence") + self.assertEqual(g(s), "sized") + self.assertEqual(g(f), "sized") + self.assertEqual(g(t), "sized") + g.register(c.MutableSet, lambda obj: "mutableset") + self.assertEqual(g(d), "mutablemapping") + self.assertEqual(g(l), "mutablesequence") + self.assertEqual(g(s), "mutableset") + self.assertEqual(g(f), "sized") + self.assertEqual(g(t), "sized") + g.register(c.Mapping, lambda obj: "mapping") + self.assertEqual(g(d), "mutablemapping") # not specific enough + self.assertEqual(g(l), "mutablesequence") + self.assertEqual(g(s), "mutableset") + self.assertEqual(g(f), "sized") + self.assertEqual(g(t), "sized") + g.register(c.Sequence, lambda obj: "sequence") + self.assertEqual(g(d), "mutablemapping") + self.assertEqual(g(l), "mutablesequence") + self.assertEqual(g(s), "mutableset") + self.assertEqual(g(f), "sized") + self.assertEqual(g(t), "sequence") + g.register(c.Set, lambda obj: "set") + self.assertEqual(g(d), "mutablemapping") + self.assertEqual(g(l), "mutablesequence") + self.assertEqual(g(s), "mutableset") + self.assertEqual(g(f), "set") + self.assertEqual(g(t), "sequence") + g.register(dict, lambda obj: "dict") + self.assertEqual(g(d), "dict") + self.assertEqual(g(l), "mutablesequence") + self.assertEqual(g(s), "mutableset") + self.assertEqual(g(f), "set") + self.assertEqual(g(t), "sequence") + g.register(list, lambda obj: "list") + self.assertEqual(g(d), "dict") + self.assertEqual(g(l), "list") + self.assertEqual(g(s), "mutableset") + self.assertEqual(g(f), "set") + self.assertEqual(g(t), "sequence") + g.register(set, lambda obj: "concrete-set") + self.assertEqual(g(d), "dict") + self.assertEqual(g(l), "list") + self.assertEqual(g(s), "concrete-set") + self.assertEqual(g(f), "set") + self.assertEqual(g(t), "sequence") + g.register(frozenset, lambda obj: "frozen-set") + self.assertEqual(g(d), "dict") + self.assertEqual(g(l), "list") + self.assertEqual(g(s), "concrete-set") + self.assertEqual(g(f), "frozen-set") + self.assertEqual(g(t), "sequence") + g.register(tuple, lambda obj: "tuple") + self.assertEqual(g(d), "dict") + self.assertEqual(g(l), "list") + self.assertEqual(g(s), "concrete-set") + self.assertEqual(g(f), "frozen-set") + self.assertEqual(g(t), "tuple") + + def test_c3_abc(self): + c = collections + mro = functools._c3_mro + class A(object): + pass + class B(A): + def __len__(self): + return 0 # implies Sized + @c.Container.register + class C(object): + pass + class D(object): + pass # unrelated + class X(D, C, B): + def __call__(self): + pass # implies Callable + expected = [X, c.Callable, D, C, c.Container, B, c.Sized, A, object] + for abcs in permutations([c.Sized, c.Callable, c.Container]): + self.assertEqual(mro(X, abcs=abcs), expected) + # unrelated ABCs don't appear in the resulting MRO + many_abcs = [c.Mapping, c.Sized, c.Callable, c.Container, c.Iterable] + self.assertEqual(mro(X, abcs=many_abcs), expected) + + def test_mro_conflicts(self): + c = collections + @functools.singledispatch + def g(arg): + return "base" + class O(c.Sized): + def __len__(self): + return 0 + o = O() + self.assertEqual(g(o), "base") + g.register(c.Iterable, lambda arg: "iterable") + g.register(c.Container, lambda arg: "container") + g.register(c.Sized, lambda arg: "sized") + g.register(c.Set, lambda arg: "set") + self.assertEqual(g(o), "sized") + c.Iterable.register(O) + self.assertEqual(g(o), "sized") # because it's explicitly in __mro__ + c.Container.register(O) + self.assertEqual(g(o), "sized") # see above: Sized is in __mro__ + c.Set.register(O) + self.assertEqual(g(o), "set") # because c.Set is a subclass of + # c.Sized and c.Container + class P: + pass + p = P() + self.assertEqual(g(p), "base") + c.Iterable.register(P) + self.assertEqual(g(p), "iterable") + c.Container.register(P) + with self.assertRaises(RuntimeError) as re_one: + g(p) + self.assertIn( + str(re_one.exception), + (("Ambiguous dispatch: <class 'collections.abc.Container'> " + "or <class 'collections.abc.Iterable'>"), + ("Ambiguous dispatch: <class 'collections.abc.Iterable'> " + "or <class 'collections.abc.Container'>")), + ) + class Q(c.Sized): + def __len__(self): + return 0 + q = Q() + self.assertEqual(g(q), "sized") + c.Iterable.register(Q) + self.assertEqual(g(q), "sized") # because it's explicitly in __mro__ + c.Set.register(Q) + self.assertEqual(g(q), "set") # because c.Set is a subclass of + # c.Sized and c.Iterable + @functools.singledispatch + def h(arg): + return "base" + @h.register(c.Sized) + def _(arg): + return "sized" + @h.register(c.Container) + def _(arg): + return "container" + # Even though Sized and Container are explicit bases of MutableMapping, + # this ABC is implicitly registered on defaultdict which makes all of + # MutableMapping's bases implicit as well from defaultdict's + # perspective. + with self.assertRaises(RuntimeError) as re_two: + h(c.defaultdict(lambda: 0)) + self.assertIn( + str(re_two.exception), + (("Ambiguous dispatch: <class 'collections.abc.Container'> " + "or <class 'collections.abc.Sized'>"), + ("Ambiguous dispatch: <class 'collections.abc.Sized'> " + "or <class 'collections.abc.Container'>")), + ) + class R(c.defaultdict): + pass + c.MutableSequence.register(R) + @functools.singledispatch + def i(arg): + return "base" + @i.register(c.MutableMapping) + def _(arg): + return "mapping" + @i.register(c.MutableSequence) + def _(arg): + return "sequence" + r = R() + self.assertEqual(i(r), "sequence") + class S: + pass + class T(S, c.Sized): + def __len__(self): + return 0 + t = T() + self.assertEqual(h(t), "sized") + c.Container.register(T) + self.assertEqual(h(t), "sized") # because it's explicitly in the MRO + class U: + def __len__(self): + return 0 + u = U() + self.assertEqual(h(u), "sized") # implicit Sized subclass inferred + # from the existence of __len__() + c.Container.register(U) + # There is no preference for registered versus inferred ABCs. + with self.assertRaises(RuntimeError) as re_three: + h(u) + self.assertIn( + str(re_three.exception), + (("Ambiguous dispatch: <class 'collections.abc.Container'> " + "or <class 'collections.abc.Sized'>"), + ("Ambiguous dispatch: <class 'collections.abc.Sized'> " + "or <class 'collections.abc.Container'>")), + ) + class V(c.Sized, S): + def __len__(self): + return 0 + @functools.singledispatch + def j(arg): + return "base" + @j.register(S) + def _(arg): + return "s" + @j.register(c.Container) + def _(arg): + return "container" + v = V() + self.assertEqual(j(v), "s") + c.Container.register(V) + self.assertEqual(j(v), "container") # because it ends up right after + # Sized in the MRO + + def test_cache_invalidation(self): + from collections import UserDict + class TracingDict(UserDict): + def __init__(self, *args, **kwargs): + super(TracingDict, self).__init__(*args, **kwargs) + self.set_ops = [] + self.get_ops = [] + def __getitem__(self, key): + result = self.data[key] + self.get_ops.append(key) + return result + def __setitem__(self, key, value): + self.set_ops.append(key) + self.data[key] = value + def clear(self): + self.data.clear() + _orig_wkd = functools.WeakKeyDictionary + td = TracingDict() + functools.WeakKeyDictionary = lambda: td + c = collections + @functools.singledispatch + def g(arg): + return "base" + d = {} + l = [] + self.assertEqual(len(td), 0) + self.assertEqual(g(d), "base") + self.assertEqual(len(td), 1) + self.assertEqual(td.get_ops, []) + self.assertEqual(td.set_ops, [dict]) + self.assertEqual(td.data[dict], g.registry[object]) + self.assertEqual(g(l), "base") + self.assertEqual(len(td), 2) + self.assertEqual(td.get_ops, []) + self.assertEqual(td.set_ops, [dict, list]) + self.assertEqual(td.data[dict], g.registry[object]) + self.assertEqual(td.data[list], g.registry[object]) + self.assertEqual(td.data[dict], td.data[list]) + self.assertEqual(g(l), "base") + self.assertEqual(g(d), "base") + self.assertEqual(td.get_ops, [list, dict]) + self.assertEqual(td.set_ops, [dict, list]) + g.register(list, lambda arg: "list") + self.assertEqual(td.get_ops, [list, dict]) + self.assertEqual(len(td), 0) + self.assertEqual(g(d), "base") + self.assertEqual(len(td), 1) + self.assertEqual(td.get_ops, [list, dict]) + self.assertEqual(td.set_ops, [dict, list, dict]) + self.assertEqual(td.data[dict], + functools._find_impl(dict, g.registry)) + self.assertEqual(g(l), "list") + self.assertEqual(len(td), 2) + self.assertEqual(td.get_ops, [list, dict]) + self.assertEqual(td.set_ops, [dict, list, dict, list]) + self.assertEqual(td.data[list], + functools._find_impl(list, g.registry)) + class X: + pass + c.MutableMapping.register(X) # Will not invalidate the cache, + # not using ABCs yet. + self.assertEqual(g(d), "base") + self.assertEqual(g(l), "list") + self.assertEqual(td.get_ops, [list, dict, dict, list]) + self.assertEqual(td.set_ops, [dict, list, dict, list]) + g.register(c.Sized, lambda arg: "sized") + self.assertEqual(len(td), 0) + self.assertEqual(g(d), "sized") + self.assertEqual(len(td), 1) + self.assertEqual(td.get_ops, [list, dict, dict, list]) + self.assertEqual(td.set_ops, [dict, list, dict, list, dict]) + self.assertEqual(g(l), "list") + self.assertEqual(len(td), 2) + self.assertEqual(td.get_ops, [list, dict, dict, list]) + self.assertEqual(td.set_ops, [dict, list, dict, list, dict, list]) + self.assertEqual(g(l), "list") + self.assertEqual(g(d), "sized") + self.assertEqual(td.get_ops, [list, dict, dict, list, list, dict]) + self.assertEqual(td.set_ops, [dict, list, dict, list, dict, list]) + g.dispatch(list) + g.dispatch(dict) + self.assertEqual(td.get_ops, [list, dict, dict, list, list, dict, + list, dict]) + self.assertEqual(td.set_ops, [dict, list, dict, list, dict, list]) + c.MutableSet.register(X) # Will invalidate the cache. + self.assertEqual(len(td), 2) # Stale cache. + self.assertEqual(g(l), "list") + self.assertEqual(len(td), 1) + g.register(c.MutableMapping, lambda arg: "mutablemapping") + self.assertEqual(len(td), 0) + self.assertEqual(g(d), "mutablemapping") + self.assertEqual(len(td), 1) + self.assertEqual(g(l), "list") + self.assertEqual(len(td), 2) + g.register(dict, lambda arg: "dict") + self.assertEqual(g(d), "dict") + self.assertEqual(g(l), "list") + g._clear_cache() + self.assertEqual(len(td), 0) + functools.WeakKeyDictionary = _orig_wkd + + def test_main(verbose=None): test_classes = ( - TestPartial, - TestPartialSubclass, - TestPythonPartial, + TestPartialC, + TestPartialPy, + TestPartialCSubclass, TestUpdateWrapper, TestTotalOrdering, - TestCmpToKey, + TestCmpToKeyC, + TestCmpToKeyPy, TestWraps, TestReduce, TestLRU, + TestSingleDispatch, ) support.run_unittest(*test_classes) diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index a0c156f5f7..beac993e4d 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -82,6 +82,14 @@ class FutureTest(unittest.TestCase): else: self.fail("expected exception didn't occur") + def test_badfuture10(self): + try: + from test import badsyntax_future10 + except SyntaxError as msg: + self.assertEqual(get_error_location(msg), ("badsyntax_future10", '3')) + else: + self.fail("expected exception didn't occur") + def test_parserhack(self): # test that the parser.c::future_hack function works as expected # Note: although this test must pass, it's not testing the original diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index c59b72eacf..e8f52a508f 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1,6 +1,9 @@ +import _testcapi import unittest from test.support import (verbose, refcount_test, run_unittest, strip_python_stderr) +from test.script_helper import assert_python_ok, make_script, temp_dir + import sys import time import gc @@ -38,6 +41,7 @@ class GC_Detector(object): # gc collects it. self.wr = weakref.ref(C1055820(666), it_happened) +@_testcapi.with_tp_del class Uncollectable(object): """Create a reference cycle with multiple __del__ methods. @@ -50,7 +54,7 @@ class Uncollectable(object): self.partner = Uncollectable(partner=self) else: self.partner = partner - def __del__(self): + def __tp_del__(self): pass ### Tests @@ -139,11 +143,12 @@ class GCTests(unittest.TestCase): del a self.assertNotEqual(gc.collect(), 0) - def test_finalizer(self): + def test_legacy_finalizer(self): # A() is uncollectable if it is part of a cycle, make sure it shows up # in gc.garbage. + @_testcapi.with_tp_del class A: - def __del__(self): pass + def __tp_del__(self): pass class B: pass a = A() @@ -163,11 +168,12 @@ class GCTests(unittest.TestCase): self.fail("didn't find obj in garbage (finalizer)") gc.garbage.remove(obj) - def test_finalizer_newclass(self): + def test_legacy_finalizer_newclass(self): # A() is uncollectable if it is part of a cycle, make sure it shows up # in gc.garbage. + @_testcapi.with_tp_del class A(object): - def __del__(self): pass + def __tp_del__(self): pass class B(object): pass a = A() @@ -568,12 +574,14 @@ class GCTests(unittest.TestCase): import subprocess code = """if 1: import gc + import _testcapi + @_testcapi.with_tp_del class X: def __init__(self, name): self.name = name def __repr__(self): return "<X %%r>" %% self.name - def __del__(self): + def __tp_del__(self): pass x = X('first') @@ -610,6 +618,66 @@ class GCTests(unittest.TestCase): stderr = run_command(code % "gc.DEBUG_SAVEALL") self.assertNotIn(b"uncollectable objects at shutdown", stderr) + def test_gc_main_module_at_shutdown(self): + # Create a reference cycle through the __main__ module and check + # it gets collected at interpreter shutdown. + code = """if 1: + import weakref + class C: + def __del__(self): + print('__del__ called') + l = [C()] + l.append(l) + """ + rc, out, err = assert_python_ok('-c', code) + self.assertEqual(out.strip(), b'__del__ called') + + def test_gc_ordinary_module_at_shutdown(self): + # Same as above, but with a non-__main__ module. + with temp_dir() as script_dir: + module = """if 1: + import weakref + class C: + def __del__(self): + print('__del__ called') + l = [C()] + l.append(l) + """ + code = """if 1: + import sys + sys.path.insert(0, %r) + import gctest + """ % (script_dir,) + make_script(script_dir, 'gctest', module) + rc, out, err = assert_python_ok('-c', code) + self.assertEqual(out.strip(), b'__del__ called') + + def test_get_stats(self): + stats = gc.get_stats() + self.assertEqual(len(stats), 3) + for st in stats: + self.assertIsInstance(st, dict) + self.assertEqual(set(st), + {"collected", "collections", "uncollectable"}) + self.assertGreaterEqual(st["collected"], 0) + self.assertGreaterEqual(st["collections"], 0) + self.assertGreaterEqual(st["uncollectable"], 0) + # Check that collection counts are incremented correctly + if gc.isenabled(): + self.addCleanup(gc.enable) + gc.disable() + old = gc.get_stats() + gc.collect(0) + new = gc.get_stats() + self.assertEqual(new[0]["collections"], old[0]["collections"] + 1) + self.assertEqual(new[1]["collections"], old[1]["collections"]) + self.assertEqual(new[2]["collections"], old[2]["collections"]) + gc.collect(2) + new = gc.get_stats() + self.assertEqual(new[0]["collections"], old[0]["collections"] + 1) + self.assertEqual(new[1]["collections"], old[1]["collections"]) + self.assertEqual(new[2]["collections"], old[2]["collections"] + 1) + class GCCallbackTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 284c8d8fd9..624e3d3db0 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -5,6 +5,7 @@ import os import re +import pprint import subprocess import sys import sysconfig @@ -17,6 +18,7 @@ try: except ImportError: _thread = None +from test import support from test.support import run_unittest, findfile, python_is_optimized try: @@ -165,6 +167,7 @@ class DebuggerTests(unittest.TestCase): 'linux-gate.so', 'Do you need "set solib-search-path" or ' '"set sysroot"?', + 'warning: Source file is more recent than executable.', ) for line in errlines: if not line.startswith(ignore_patterns): @@ -307,6 +310,8 @@ class PrettyPrintTests(DebuggerTests): def test_sets(self): 'Verify the pretty-printing of sets' + if (gdb_major_version, gdb_minor_version) < (7, 3): + self.skipTest("pretty-printing of sets needs gdb 7.3 or later") self.assertGdbRepr(set()) self.assertGdbRepr(set(['a', 'b']), "{'a', 'b'}") self.assertGdbRepr(set([4, 5, 6]), "{4, 5, 6}") @@ -320,6 +325,8 @@ id(s)''') def test_frozensets(self): 'Verify the pretty-printing of frozensets' + if (gdb_major_version, gdb_minor_version) < (7, 3): + self.skipTest("pretty-printing of frozensets needs gdb 7.3 or later") self.assertGdbRepr(frozenset()) self.assertGdbRepr(frozenset(['a', 'b']), "frozenset({'a', 'b'})") self.assertGdbRepr(frozenset([4, 5, 6]), "frozenset({4, 5, 6})") @@ -837,6 +844,10 @@ class PyLocalsTests(DebuggerTests): r".*\na = 1\nb = 2\nc = 3\n.*") def test_main(): + if support.verbose: + print("GDB version:") + for line in os.fsdecode(gdb_version).splitlines(): + print(" " * 4 + line) run_unittest(PrettyPrintTests, PyListTests, StackNavigationTests, diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 958054aef5..4e921177a5 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -1,3 +1,55 @@ +import gc +import sys +import unittest +import weakref + +from test import support + + +class FinalizationTest(unittest.TestCase): + + def test_frame_resurrect(self): + # A generator frame can be resurrected by a generator's finalization. + def gen(): + nonlocal frame + try: + yield + finally: + frame = sys._getframe() + + g = gen() + wr = weakref.ref(g) + next(g) + del g + support.gc_collect() + self.assertIs(wr(), None) + self.assertTrue(frame) + del frame + support.gc_collect() + + def test_refcycle(self): + # A generator caught in a refcycle gets finalized anyway. + old_garbage = gc.garbage[:] + finalized = False + def gen(): + nonlocal finalized + try: + g = yield + yield 1 + finally: + finalized = True + + g = gen() + next(g) + g.send(g) + self.assertGreater(sys.getrefcount(g), 2) + self.assertFalse(finalized) + del g + support.gc_collect() + self.assertTrue(finalized) + self.assertEqual(gc.garbage, old_garbage) + + tutorial_tests = """ Let's try a simple generator: @@ -1729,9 +1781,7 @@ Our ill-behaved code should be invoked during GC: >>> g = f() >>> next(g) >>> del g ->>> sys.stderr.getvalue().startswith( -... "Exception RuntimeError: 'generator ignored GeneratorExit' in " -... ) +>>> "RuntimeError: generator ignored GeneratorExit" in sys.stderr.getvalue() True >>> sys.stderr = old @@ -1841,22 +1891,23 @@ to test. ... sys.stderr = io.StringIO() ... class Leaker: ... def __del__(self): -... raise RuntimeError +... def invoke(message): +... raise RuntimeError(message) +... invoke("test") ... ... l = Leaker() ... del l ... err = sys.stderr.getvalue().strip() -... err.startswith( -... "Exception RuntimeError: RuntimeError() in <" -... ) -... err.endswith("> ignored") -... len(err.splitlines()) +... "Exception ignored in" in err +... "RuntimeError: test" in err +... "Traceback" in err +... "in invoke" in err ... finally: ... sys.stderr = old True True -1 - +True +True These refleak tests should perhaps be in a testfile of their own, @@ -1881,6 +1932,7 @@ __test__ = {"tut": tutorial_tests, # so this works as expected in both ways of running regrtest. def test_main(verbose=None): from test import support, test_generators + support.run_unittest(__name__) support.run_doctest(test_generators, verbose) # This part isn't needed for regrtest, but for running the test directly. diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index fd8bc577ca..e967897d75 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -188,12 +188,93 @@ class GenericTest: support.unlink(support.TESTFN) safe_rmdir(support.TESTFN) + @staticmethod + def _create_file(filename): + with open(filename, 'wb') as f: + f.write(b'foo') + + def test_samefile(self): + try: + test_fn = support.TESTFN + "1" + self._create_file(test_fn) + self.assertTrue(self.pathmodule.samefile(test_fn, test_fn)) + self.assertRaises(TypeError, self.pathmodule.samefile) + finally: + os.remove(test_fn) + + @support.skip_unless_symlink + def test_samefile_on_symlink(self): + self._test_samefile_on_link_func(os.symlink) + + def test_samefile_on_link(self): + self._test_samefile_on_link_func(os.link) + + def _test_samefile_on_link_func(self, func): + try: + test_fn1 = support.TESTFN + "1" + test_fn2 = support.TESTFN + "2" + self._create_file(test_fn1) + + func(test_fn1, test_fn2) + self.assertTrue(self.pathmodule.samefile(test_fn1, test_fn2)) + os.remove(test_fn2) + + self._create_file(test_fn2) + self.assertFalse(self.pathmodule.samefile(test_fn1, test_fn2)) + finally: + os.remove(test_fn1) + os.remove(test_fn2) + + def test_samestat(self): + try: + test_fn = support.TESTFN + "1" + self._create_file(test_fn) + test_fns = [test_fn]*2 + stats = map(os.stat, test_fns) + self.assertTrue(self.pathmodule.samestat(*stats)) + finally: + os.remove(test_fn) + + @support.skip_unless_symlink + def test_samestat_on_symlink(self): + self._test_samestat_on_link_func(os.symlink) + + def test_samestat_on_link(self): + self._test_samestat_on_link_func(os.link) + + def _test_samestat_on_link_func(self, func): + try: + test_fn1 = support.TESTFN + "1" + test_fn2 = support.TESTFN + "2" + self._create_file(test_fn1) + test_fns = (test_fn1, test_fn2) + func(*test_fns) + stats = map(os.stat, test_fns) + self.assertTrue(self.pathmodule.samestat(*stats)) + os.remove(test_fn2) + + self._create_file(test_fn2) + stats = map(os.stat, test_fns) + self.assertFalse(self.pathmodule.samestat(*stats)) + + self.assertRaises(TypeError, self.pathmodule.samestat) + finally: + os.remove(test_fn1) + os.remove(test_fn2) + + def test_sameopenfile(self): + fname = support.TESTFN + "1" + with open(fname, "wb") as a, open(fname, "wb") as b: + self.assertTrue(self.pathmodule.sameopenfile( + a.fileno(), b.fileno())) + class TestGenericTest(GenericTest, unittest.TestCase): # Issue 16852: GenericTest can't inherit from unittest.TestCase # for test discovery purposes; CommonTest inherits from GenericTest # and is only meant to be inherited by others. pathmodule = genericpath + # Following TestCase is not supposed to be run from test_genericpath. # It is inherited by other test modules (macpath, ntpath, posixpath). @@ -322,7 +403,6 @@ class CommonTest(GenericTest): else: self.skipTest("need support.TESTFN_NONASCII") - # Test non-ASCII, non-UTF8 bytes in the path. with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) with support.temp_cwd(name): diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_getargs2.py index 48ca94ee38..beea76a920 100644 --- a/Lib/test/test_getargs2.py +++ b/Lib/test/test_getargs2.py @@ -1,38 +1,40 @@ import unittest from test import support from _testcapi import getargs_keywords, getargs_keyword_only - -""" -> How about the following counterproposal. This also changes some of -> the other format codes to be a little more regular. -> -> Code C type Range check -> -> b unsigned char 0..UCHAR_MAX -> h signed short SHRT_MIN..SHRT_MAX -> B unsigned char none ** -> H unsigned short none ** -> k * unsigned long none -> I * unsigned int 0..UINT_MAX - - -> i int INT_MIN..INT_MAX -> l long LONG_MIN..LONG_MAX - -> K * unsigned long long none -> L long long LLONG_MIN..LLONG_MAX - -> Notes: -> -> * New format codes. -> -> ** Changed from previous "range-and-a-half" to "none"; the -> range-and-a-half checking wasn't particularly useful. - -Plus a C API or two, e.g. PyInt_AsLongMask() -> -unsigned long and PyInt_AsLongLongMask() -> unsigned -long long (if that exists). -""" +try: + from _testcapi import getargs_L, getargs_K +except ImportError: + getargs_L = None # PY_LONG_LONG not available + +# > How about the following counterproposal. This also changes some of +# > the other format codes to be a little more regular. +# > +# > Code C type Range check +# > +# > b unsigned char 0..UCHAR_MAX +# > h signed short SHRT_MIN..SHRT_MAX +# > B unsigned char none ** +# > H unsigned short none ** +# > k * unsigned long none +# > I * unsigned int 0..UINT_MAX +# +# +# > i int INT_MIN..INT_MAX +# > l long LONG_MIN..LONG_MAX +# +# > K * unsigned long long none +# > L long long LLONG_MIN..LLONG_MAX +# +# > Notes: +# > +# > * New format codes. +# > +# > ** Changed from previous "range-and-a-half" to "none"; the +# > range-and-a-half checking wasn't particularly useful. +# +# Plus a C API or two, e.g. PyInt_AsLongMask() -> +# unsigned long and PyInt_AsLongLongMask() -> unsigned +# long long (if that exists). LARGE = 0x7FFFFFFF VERY_LARGE = 0xFF0000121212121212121242 @@ -184,6 +186,7 @@ class Signed_TestCase(unittest.TestCase): self.assertRaises(OverflowError, getargs_n, VERY_LARGE) +@unittest.skipIf(getargs_L is None, 'PY_LONG_LONG is not available') class LongLong_TestCase(unittest.TestCase): def test_L(self): from _testcapi import getargs_L @@ -536,24 +539,5 @@ class Unicode_TestCase(unittest.TestCase): self.assertIsNone(getargs_Z_hash(None)) -def test_main(): - tests = [ - Signed_TestCase, - Unsigned_TestCase, - Boolean_TestCase, - Tuple_TestCase, - Keywords_TestCase, - KeywordOnly_TestCase, - Bytes_TestCase, - Unicode_TestCase, - ] - try: - from _testcapi import getargs_L, getargs_K - except ImportError: - pass # PY_LONG_LONG not available - else: - tests.append(LongLong_TestCase) - support.run_unittest(*tests) - if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_getpass.py b/Lib/test/test_getpass.py new file mode 100644 index 0000000000..1731bd44a6 --- /dev/null +++ b/Lib/test/test_getpass.py @@ -0,0 +1,155 @@ +import getpass +import os +import unittest +from io import BytesIO, StringIO +from unittest import mock +from test import support + +try: + import termios +except ImportError: + termios = None +try: + import pwd +except ImportError: + pwd = None + +@mock.patch('os.environ') +class GetpassGetuserTest(unittest.TestCase): + + def test_username_takes_username_from_env(self, environ): + expected_name = 'some_name' + environ.get.return_value = expected_name + self.assertEqual(expected_name, getpass.getuser()) + + def test_username_priorities_of_env_values(self, environ): + environ.get.return_value = None + try: + getpass.getuser() + except ImportError: # in case there's no pwd module + pass + self.assertEqual( + environ.get.call_args_list, + [mock.call(x) for x in ('LOGNAME', 'USER', 'LNAME', 'USERNAME')]) + + def test_username_falls_back_to_pwd(self, environ): + expected_name = 'some_name' + environ.get.return_value = None + if pwd: + with mock.patch('os.getuid') as uid, \ + mock.patch('pwd.getpwuid') as getpw: + uid.return_value = 42 + getpw.return_value = [expected_name] + self.assertEqual(expected_name, + getpass.getuser()) + getpw.assert_called_once_with(42) + else: + self.assertRaises(ImportError, getpass.getuser) + + +class GetpassRawinputTest(unittest.TestCase): + + def test_flushes_stream_after_prompt(self): + # see issue 1703 + stream = mock.Mock(spec=StringIO) + input = StringIO('input_string') + getpass._raw_input('some_prompt', stream, input=input) + stream.flush.assert_called_once_with() + + def test_uses_stderr_as_default(self): + input = StringIO('input_string') + prompt = 'some_prompt' + with mock.patch('sys.stderr') as stderr: + getpass._raw_input(prompt, input=input) + stderr.write.assert_called_once_with(prompt) + + @mock.patch('sys.stdin') + def test_uses_stdin_as_default_input(self, mock_input): + mock_input.readline.return_value = 'input_string' + getpass._raw_input(stream=StringIO()) + mock_input.readline.assert_called_once_with() + + def test_raises_on_empty_input(self): + input = StringIO('') + self.assertRaises(EOFError, getpass._raw_input, input=input) + + def test_trims_trailing_newline(self): + input = StringIO('test\n') + self.assertEqual('test', getpass._raw_input(input=input)) + + +# Some of these tests are a bit white-box. The functional requirement is that +# the password input be taken directly from the tty, and that it not be echoed +# on the screen, unless we are falling back to stderr/stdin. + +# Some of these might run on platforms without termios, but play it safe. +@unittest.skipUnless(termios, 'tests require system with termios') +class UnixGetpassTest(unittest.TestCase): + + def test_uses_tty_directly(self): + with mock.patch('os.open') as open, \ + mock.patch('io.FileIO') as fileio, \ + mock.patch('io.TextIOWrapper') as textio: + # By setting open's return value to None the implementation will + # skip code we don't care about in this test. We can mock this out + # fully if an alternate implementation works differently. + open.return_value = None + getpass.unix_getpass() + open.assert_called_once_with('/dev/tty', + os.O_RDWR | os.O_NOCTTY) + fileio.assert_called_once_with(open.return_value, 'w+') + textio.assert_called_once_with(fileio.return_value) + + def test_resets_termios(self): + with mock.patch('os.open') as open, \ + mock.patch('io.FileIO'), \ + mock.patch('io.TextIOWrapper'), \ + mock.patch('termios.tcgetattr') as tcgetattr, \ + mock.patch('termios.tcsetattr') as tcsetattr: + open.return_value = 3 + fake_attrs = [255, 255, 255, 255, 255] + tcgetattr.return_value = list(fake_attrs) + getpass.unix_getpass() + tcsetattr.assert_called_with(3, mock.ANY, fake_attrs) + + def test_falls_back_to_fallback_if_termios_raises(self): + with mock.patch('os.open') as open, \ + mock.patch('io.FileIO') as fileio, \ + mock.patch('io.TextIOWrapper') as textio, \ + mock.patch('termios.tcgetattr'), \ + mock.patch('termios.tcsetattr') as tcsetattr, \ + mock.patch('getpass.fallback_getpass') as fallback: + open.return_value = 3 + fileio.return_value = BytesIO() + tcsetattr.side_effect = termios.error + getpass.unix_getpass() + fallback.assert_called_once_with('Password: ', + textio.return_value) + + def test_flushes_stream_after_input(self): + # issue 7208 + with mock.patch('os.open') as open, \ + mock.patch('io.FileIO'), \ + mock.patch('io.TextIOWrapper'), \ + mock.patch('termios.tcgetattr'), \ + mock.patch('termios.tcsetattr'): + open.return_value = 3 + mock_stream = mock.Mock(spec=StringIO) + getpass.unix_getpass(stream=mock_stream) + mock_stream.flush.assert_called_with() + + def test_falls_back_to_stdin(self): + with mock.patch('os.open') as os_open, \ + mock.patch('sys.stdin', spec=StringIO) as stdin: + os_open.side_effect = IOError + stdin.fileno.side_effect = AttributeError + with support.captured_stderr() as stderr: + with self.assertWarns(getpass.GetPassWarning): + getpass.unix_getpass() + stdin.readline.assert_called_once_with() + self.assertIn('Warning', stderr.getvalue()) + self.assertIn('Password:', stderr.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index 5eac9217b2..37fe8538a7 100755..100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -389,6 +389,20 @@ class TestGzip(BaseTest): datac = gzip.compress(data) self.assertEqual(gzip.decompress(datac), data) + def test_read_truncated(self): + data = data1*50 + # Drop the CRC (4 bytes) and file size (4 bytes). + truncated = gzip.compress(data)[:-8] + with gzip.GzipFile(fileobj=io.BytesIO(truncated)) as f: + self.assertRaises(EOFError, f.read) + with gzip.GzipFile(fileobj=io.BytesIO(truncated)) as f: + self.assertEqual(f.read(len(data)), data) + self.assertRaises(EOFError, f.read, 1) + # Incomplete 10-byte header. + for i in range(2, 10): + with gzip.GzipFile(fileobj=io.BytesIO(truncated[:i])) as f: + self.assertRaises(EOFError, f.read, 1) + def test_read_with_extra(self): # Gzip data with an extra field gzdata = (b'\x1f\x8b\x08\x04\xb2\x17cQ\x02\xff' diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index f3385f6d6f..3eadee1cc7 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -36,7 +36,10 @@ def hexstr(s): class HashLibTestCase(unittest.TestCase): supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1', 'sha224', 'SHA224', 'sha256', 'SHA256', - 'sha384', 'SHA384', 'sha512', 'SHA512' ) + 'sha384', 'SHA384', 'sha512', 'SHA512', + 'sha3_224', 'sha3_256', 'sha3_384', + 'sha3_512', 'SHA3_224', 'SHA3_256', + 'SHA3_384', 'SHA3_512' ) # Issue #14693: fallback modules are always compiled under POSIX _warn_on_extension_import = os.name == 'posix' or COMPILED_WITH_PYDEBUG @@ -72,8 +75,8 @@ class HashLibTestCase(unittest.TestCase): if _hashlib: # These two algorithms should always be present when this module # is compiled. If not, something was compiled wrong. - assert hasattr(_hashlib, 'openssl_md5') - assert hasattr(_hashlib, 'openssl_sha1') + self.assertTrue(hasattr(_hashlib, 'openssl_md5')) + self.assertTrue(hasattr(_hashlib, 'openssl_sha1')) for algorithm, constructors in self.constructors_to_test.items(): constructor = getattr(_hashlib, 'openssl_'+algorithm, None) if constructor: @@ -93,6 +96,12 @@ class HashLibTestCase(unittest.TestCase): if _sha512: self.constructors_to_test['sha384'].add(_sha512.sha384) self.constructors_to_test['sha512'].add(_sha512.sha512) + _sha3 = self._conditional_import_module('_sha3') + if _sha3: + self.constructors_to_test['sha3_224'].add(_sha3.sha3_224) + self.constructors_to_test['sha3_256'].add(_sha3.sha3_256) + self.constructors_to_test['sha3_384'].add(_sha3.sha3_384) + self.constructors_to_test['sha3_512'].add(_sha3.sha3_512) super(HashLibTestCase, self).__init__(*args, **kwargs) @@ -142,9 +151,16 @@ class HashLibTestCase(unittest.TestCase): def test_hexdigest(self): for cons in self.hash_constructors: h = cons() - assert isinstance(h.digest(), bytes), name + self.assertIsInstance(h.digest(), bytes) self.assertEqual(hexstr(h.digest()), h.hexdigest()) + def test_name_attribute(self): + for cons in self.hash_constructors: + h = cons() + self.assertIsInstance(h.name, str) + self.assertIn(h.name, self.supported_hash_names) + self.assertEqual(h.name, hashlib.new(h.name).name) + def test_large_update(self): aas = b'a' * 128 bees = b'b' * 127 @@ -205,6 +221,10 @@ class HashLibTestCase(unittest.TestCase): self.check_no_unicode('sha256') self.check_no_unicode('sha384') self.check_no_unicode('sha512') + self.check_no_unicode('sha3_224') + self.check_no_unicode('sha3_256') + self.check_no_unicode('sha3_384') + self.check_no_unicode('sha3_512') def check_blocksize_name(self, name, block_size=0, digest_size=0): constructors = self.constructors_to_test[name] @@ -213,8 +233,9 @@ class HashLibTestCase(unittest.TestCase): self.assertEqual(m.block_size, block_size) self.assertEqual(m.digest_size, digest_size) self.assertEqual(len(m.digest()), digest_size) - self.assertEqual(m.name.lower(), name.lower()) - self.assertIn(name.split("_")[0], repr(m).lower()) + self.assertEqual(m.name, name) + # split for sha3_512 / _sha3.sha3 object + self.assertIn(name.split("_")[0], repr(m)) def test_blocksize_name(self): self.check_blocksize_name('md5', 64, 16) @@ -223,6 +244,10 @@ class HashLibTestCase(unittest.TestCase): self.check_blocksize_name('sha256', 64, 32) self.check_blocksize_name('sha384', 128, 48) self.check_blocksize_name('sha512', 128, 64) + self.check_blocksize_name('sha3_224', NotImplemented, 28) + self.check_blocksize_name('sha3_256', NotImplemented, 32) + self.check_blocksize_name('sha3_384', NotImplemented, 48) + self.check_blocksize_name('sha3_512', NotImplemented, 64) def test_case_md5_0(self): self.check('md5', b'', 'd41d8cd98f00b204e9800998ecf8427e') @@ -358,6 +383,108 @@ class HashLibTestCase(unittest.TestCase): "e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973eb"+ "de0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b") + # SHA-3 family + def test_case_sha3_224_0(self): + self.check('sha3_224', b"", + "F71837502BA8E10837BDD8D365ADB85591895602FC552B48B7390ABD") + + def test_case_sha3_224_1(self): + self.check('sha3_224', bytes.fromhex("CC"), + "A9CAB59EB40A10B246290F2D6086E32E3689FAF1D26B470C899F2802") + + def test_case_sha3_224_2(self): + self.check('sha3_224', bytes.fromhex("41FB"), + "615BA367AFDC35AAC397BC7EB5D58D106A734B24986D5D978FEFD62C") + + def test_case_sha3_224_3(self): + self.check('sha3_224', bytes.fromhex( + "433C5303131624C0021D868A30825475E8D0BD3052A022180398F4CA4423B9"+ + "8214B6BEAAC21C8807A2C33F8C93BD42B092CC1B06CEDF3224D5ED1EC29784"+ + "444F22E08A55AA58542B524B02CD3D5D5F6907AFE71C5D7462224A3F9D9E53"+ + "E7E0846DCBB4CE"), + "62B10F1B6236EBC2DA72957742A8D4E48E213B5F8934604BFD4D2C3A") + + @bigmemtest(size=_4G + 5, memuse=1) + def test_case_sha3_224_huge(self, size): + if size == _4G + 5: + try: + self.check('sha3_224', b'A'*size, + '58ef60057c9dddb6a87477e9ace5a26f0d9db01881cf9b10a9f8c224') + except OverflowError: + pass # 32-bit arch + + + def test_case_sha3_256_0(self): + self.check('sha3_256', b"", + "C5D2460186F7233C927E7DB2DCC703C0E500B653CA82273B7BFAD8045D85A470") + + def test_case_sha3_256_1(self): + self.check('sha3_256', bytes.fromhex("CC"), + "EEAD6DBFC7340A56CAEDC044696A168870549A6A7F6F56961E84A54BD9970B8A") + + def test_case_sha3_256_2(self): + self.check('sha3_256', bytes.fromhex("41FB"), + "A8EACEDA4D47B3281A795AD9E1EA2122B407BAF9AABCB9E18B5717B7873537D2") + + def test_case_sha3_256_3(self): + self.check('sha3_256', bytes.fromhex( + "433C5303131624C0021D868A30825475E8D0BD3052A022180398F4CA4423B9"+ + "8214B6BEAAC21C8807A2C33F8C93BD42B092CC1B06CEDF3224D5ED1EC29784"+ + "444F22E08A55AA58542B524B02CD3D5D5F6907AFE71C5D7462224A3F9D9E53"+ + "E7E0846DCBB4CE"), + "CE87A5173BFFD92399221658F801D45C294D9006EE9F3F9D419C8D427748DC41") + + + def test_case_sha3_384_0(self): + self.check('sha3_384', b"", + "2C23146A63A29ACF99E73B88F8C24EAA7DC60AA771780CCC006AFBFA8FE2479B"+ + "2DD2B21362337441AC12B515911957FF") + + def test_case_sha3_384_1(self): + self.check('sha3_384', bytes.fromhex("CC"), + "1B84E62A46E5A201861754AF5DC95C4A1A69CAF4A796AE405680161E29572641"+ + "F5FA1E8641D7958336EE7B11C58F73E9") + + def test_case_sha3_384_2(self): + self.check('sha3_384', bytes.fromhex("41FB"), + "495CCE2714CD72C8C53C3363D22C58B55960FE26BE0BF3BBC7A3316DD563AD1D"+ + "B8410E75EEFEA655E39D4670EC0B1792") + + def test_case_sha3_384_3(self): + self.check('sha3_384', bytes.fromhex( + "433C5303131624C0021D868A30825475E8D0BD3052A022180398F4CA4423B9"+ + "8214B6BEAAC21C8807A2C33F8C93BD42B092CC1B06CEDF3224D5ED1EC29784"+ + "444F22E08A55AA58542B524B02CD3D5D5F6907AFE71C5D7462224A3F9D9E53"+ + "E7E0846DCBB4CE"), + "135114508DD63E279E709C26F7817C0482766CDE49132E3EDF2EEDD8996F4E35"+ + "96D184100B384868249F1D8B8FDAA2C9") + + + def test_case_sha3_512_0(self): + self.check('sha3_512', b"", + "0EAB42DE4C3CEB9235FC91ACFFE746B29C29A8C366B7C60E4E67C466F36A4304"+ + "C00FA9CAF9D87976BA469BCBE06713B435F091EF2769FB160CDAB33D3670680E") + + def test_case_sha3_512_1(self): + self.check('sha3_512', bytes.fromhex("CC"), + "8630C13CBD066EA74BBE7FE468FEC1DEE10EDC1254FB4C1B7C5FD69B646E4416"+ + "0B8CE01D05A0908CA790DFB080F4B513BC3B6225ECE7A810371441A5AC666EB9") + + def test_case_sha3_512_2(self): + self.check('sha3_512', bytes.fromhex("41FB"), + "551DA6236F8B96FCE9F97F1190E901324F0B45E06DBBB5CDB8355D6ED1DC34B3"+ + "F0EAE7DCB68622FF232FA3CECE0D4616CDEB3931F93803662A28DF1CD535B731") + + def test_case_sha3_512_3(self): + self.check('sha3_512', bytes.fromhex( + "433C5303131624C0021D868A30825475E8D0BD3052A022180398F4CA4423B9"+ + "8214B6BEAAC21C8807A2C33F8C93BD42B092CC1B06CEDF3224D5ED1EC29784"+ + "444F22E08A55AA58542B524B02CD3D5D5F6907AFE71C5D7462224A3F9D9E53"+ + "E7E0846DCBB4CE"), + "527D28E341E6B14F4684ADB4B824C496C6482E51149565D3D17226828884306B"+ + "51D6148A72622C2B75F5D3510B799D8BDC03EAEDE453676A6EC8FE03A1AD0EAB") + + def test_gil(self): # Check things work fine with an input larger than the size required # for multithreaded operation (which is hardwired to 2048). @@ -406,8 +533,8 @@ class HashLibTestCase(unittest.TestCase): events = [] for threadnum in range(num_threads): chunk_size = len(data) // (10**threadnum) - assert chunk_size > 0 - assert chunk_size % len(smallest_data) == 0 + self.assertGreater(chunk_size, 0) + self.assertEqual(chunk_size % len(smallest_data), 0) event = threading.Event() events.append(event) threading.Thread(target=hash_in_chunks, @@ -418,8 +545,88 @@ class HashLibTestCase(unittest.TestCase): self.assertEqual(expected_hash, hasher.hexdigest()) -def test_main(): - support.run_unittest(HashLibTestCase) + pbkdf2_test_vectors = [ + (b'password', b'salt', 1, None), + (b'password', b'salt', 2, None), + (b'password', b'salt', 4096, None), + # too slow, it takes over a minute on a fast CPU. + #(b'password', b'salt', 16777216, None), + (b'passwordPASSWORDpassword', b'saltSALTsaltSALTsaltSALTsaltSALTsalt', + 4096, -1), + (b'pass\0word', b'sa\0lt', 4096, 16), + ] + + pbkdf2_results = { + "sha1": [ + # offical test vectors from RFC 6070 + (bytes.fromhex('0c60c80f961f0e71f3a9b524af6012062fe037a6'), None), + (bytes.fromhex('ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957'), None), + (bytes.fromhex('4b007901b765489abead49d926f721d065a429c1'), None), + #(bytes.fromhex('eefe3d61cd4da4e4e9945b3d6ba2158c2634e984'), None), + (bytes.fromhex('3d2eec4fe41c849b80c8d83662c0e44a8b291a964c' + 'f2f07038'), 25), + (bytes.fromhex('56fa6aa75548099dcc37d7f03425e0c3'), None),], + "sha256": [ + (bytes.fromhex('120fb6cffcf8b32c43e7225256c4f837' + 'a86548c92ccc35480805987cb70be17b'), None), + (bytes.fromhex('ae4d0c95af6b46d32d0adff928f06dd0' + '2a303f8ef3c251dfd6e2d85a95474c43'), None), + (bytes.fromhex('c5e478d59288c841aa530db6845c4c8d' + '962893a001ce4e11a4963873aa98134a'), None), + #(bytes.fromhex('cf81c66fe8cfc04d1f31ecb65dab4089' + # 'f7f179e89b3b0bcb17ad10e3ac6eba46'), None), + (bytes.fromhex('348c89dbcbd32b2f32d814b8116e84cf2b17' + '347ebc1800181c4e2a1fb8dd53e1c635518c7dac47e9'), 40), + (bytes.fromhex('89b69d0516f829893c696226650a8687'), None),], + "sha512": [ + (bytes.fromhex('867f70cf1ade02cff3752599a3a53dc4af34c7a669815ae5' + 'd513554e1c8cf252c02d470a285a0501bad999bfe943c08f' + '050235d7d68b1da55e63f73b60a57fce'), None), + (bytes.fromhex('e1d9c16aa681708a45f5c7c4e215ceb66e011a2e9f004071' + '3f18aefdb866d53cf76cab2868a39b9f7840edce4fef5a82' + 'be67335c77a6068e04112754f27ccf4e'), None), + (bytes.fromhex('d197b1b33db0143e018b12f3d1d1479e6cdebdcc97c5c0f8' + '7f6902e072f457b5143f30602641b3d55cd335988cb36b84' + '376060ecd532e039b742a239434af2d5'), None), + (bytes.fromhex('8c0511f4c6e597c6ac6315d8f0362e225f3c501495ba23b8' + '68c005174dc4ee71115b59f9e60cd9532fa33e0f75aefe30' + '225c583a186cd82bd4daea9724a3d3b8'), 64), + (bytes.fromhex('9d9e9c4cd21fe4be24d5b8244c759665'), None),], + } + + @unittest.skipUnless(hasattr(hashlib, 'pbkdf2_hmac'), + 'pbkdf2_hmac required for this test.') + def test_pbkdf2_hmac(self): + pbkdf2 = hashlib.pbkdf2_hmac + + for digest_name, results in self.pbkdf2_results.items(): + for i, vector in enumerate(self.pbkdf2_test_vectors): + password, salt, rounds, dklen = vector + expected, overwrite_dklen = results[i] + if overwrite_dklen: + dklen = overwrite_dklen + out = pbkdf2(digest_name, password, salt, rounds, dklen) + self.assertEqual(out, expected, + (digest_name, password, salt, rounds, dklen)) + out = pbkdf2(digest_name, memoryview(password), + memoryview(salt), rounds, dklen) + out = pbkdf2(digest_name, bytearray(password), + bytearray(salt), rounds, dklen) + self.assertEqual(out, expected) + if dklen is None: + out = pbkdf2(digest_name, password, salt, rounds) + self.assertEqual(out, expected, + (digest_name, password, salt, rounds)) + + self.assertRaises(TypeError, pbkdf2, b'sha1', b'pass', b'salt', 1) + self.assertRaises(TypeError, pbkdf2, 'sha1', 'pass', 'salt', 1) + self.assertRaises(ValueError, pbkdf2, 'sha1', b'pass', b'salt', 0) + self.assertRaises(ValueError, pbkdf2, 'sha1', b'pass', b'salt', -1) + self.assertRaises(ValueError, pbkdf2, 'sha1', b'pass', b'salt', 1, 0) + self.assertRaises(ValueError, pbkdf2, 'sha1', b'pass', b'salt', 1, -1) + with self.assertRaisesRegex(ValueError, 'unsupported hash type'): + pbkdf2('unknown', b'pass', b'salt', 1) + if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 4ca7cec44c..efd63ad7be 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -253,6 +253,20 @@ class ConstructorTestCase(unittest.TestCase): except: self.fail("Constructor call with text argument raised exception.") + def test_with_bytearray(self): + try: + h = hmac.HMAC(bytearray(b"key"), bytearray(b"hash this!")) + self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864') + except: + self.fail("Constructor call with bytearray arguments raised exception.") + + def test_with_memoryview_msg(self): + try: + h = hmac.HMAC(b"key", memoryview(b"hash this!")) + self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864') + except: + self.fail("Constructor call with memoryview msg raised exception.") + def test_withmodule(self): # Constructor call with text and digest module. try: diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 769ab13fca..778e919fb9 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -27,8 +27,10 @@ class FakeSocket: self.text = text self.fileclass = fileclass self.data = b'' + self.sendall_calls = 0 def sendall(self, data): + self.sendall_calls += 1 self.data += data def makefile(self, mode, bufsize=None): @@ -45,7 +47,7 @@ class EPipeSocket(FakeSocket): def sendall(self, data): if self.pipe_trigger in data: - raise socket.error(errno.EPIPE, "gotcha") + raise OSError(errno.EPIPE, "gotcha") self.data += data def close(self): @@ -588,7 +590,7 @@ class BasicTest(TestCase): b"Content-Length") conn = client.HTTPConnection("example.com") conn.sock = sock - self.assertRaises(socket.error, + self.assertRaises(OSError, lambda: conn.request("PUT", "/url", "body")) resp = conn.getresponse() self.assertEqual(401, resp.status) @@ -634,6 +636,28 @@ class BasicTest(TestCase): resp.close() self.assertTrue(resp.closed) + def test_delayed_ack_opt(self): + # Test that Nagle/delayed_ack optimistaion works correctly. + + # For small payloads, it should coalesce the body with + # headers, resulting in a single sendall() call + conn = client.HTTPConnection('example.com') + sock = FakeSocket(None) + conn.sock = sock + body = b'x' * (conn.mss - 1) + conn.request('POST', '/', body) + self.assertEqual(sock.sendall_calls, 1) + + # For large payloads, it should send the headers and + # then the body, resulting in more than one sendall() + # call + conn = client.HTTPConnection('example.com') + sock = FakeSocket(None) + conn.sock = sock + body = b'x' * conn.mss + conn.request('POST', '/', body) + self.assertGreater(sock.sendall_calls, 1) + class OfflineTest(TestCase): def test_responses(self): self.assertEqual(client.responses[client.NOT_FOUND], "Not Found") @@ -724,7 +748,7 @@ class HTTPSTest(TestCase): def make_server(self, certfile): from test.ssl_servers import make_https_server - return make_https_server(self, certfile) + return make_https_server(self, certfile=certfile) def test_attributes(self): # simple test to check it's storing the timeout diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index b8bbcb6734..4083afcbf2 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -92,6 +92,13 @@ class BaseHTTPServerTestCase(BaseTestCase): def do_KEYERROR(self): self.send_error(999) + def do_NOTFOUND(self): + self.send_error(404) + + def do_EXPLAINERROR(self): + self.send_error(999, "Short Message", + "This is a long \n explaination") + def do_CUSTOM(self): self.send_response(999) self.send_header('Content-Type', 'text/html') @@ -203,6 +210,12 @@ class BaseHTTPServerTestCase(BaseTestCase): res = self.con.getresponse() self.assertEqual(res.status, 999) + def test_return_explain_error(self): + self.con.request('EXPLAINERROR', '/') + res = self.con.getresponse() + self.assertEqual(res.status, 999) + self.assertTrue(int(res.getheader('Content-Length'))) + def test_latin1_header(self): self.con.request('LATINONEHEADER', '/', headers={ 'X-Special-Incoming': 'Ärger mit Unicode' @@ -211,6 +224,14 @@ class BaseHTTPServerTestCase(BaseTestCase): self.assertEqual(res.getheader('X-Special'), 'Dängerous Mind') self.assertEqual(res.read(), 'Ärger mit Unicode'.encode('utf-8')) + def test_error_content_length(self): + # Issue #16088: standard error responses should have a content-length + self.con.request('NOTFOUND', '/') + res = self.con.getresponse() + self.assertEqual(res.status, 404) + data = res.read() + self.assertEqual(int(res.getheader('Content-Length')), len(data)) + class SimpleHTTPServerTestCase(BaseTestCase): class request_handler(NoLogRequestHandler, SimpleHTTPRequestHandler): diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 7db3f7dddc..c37ea1dccf 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -125,7 +125,7 @@ class SimpleIMAPHandler(socketserver.StreamRequestHandler): # Naked sockets return empty strings.. return line += part - except IOError: + except OSError: # ..but SSLSockets raise exceptions. return if line.endswith(b'\r\n'): diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index b56efe3bea..cf27a712d5 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -1,14 +1,29 @@ -import imp +try: + import _thread +except ImportError: + _thread = None import importlib import os import os.path import shutil import sys from test import support -from test.test_importlib import util import unittest import warnings +with warnings.catch_warnings(): + warnings.simplefilter('ignore', PendingDeprecationWarning) + import imp + +def requires_load_dynamic(meth): + """Decorator to skip a test if not running under CPython or lacking + imp.load_dynamic().""" + meth = support.cpython_only(meth) + return unittest.skipIf(not hasattr(imp, 'load_dynamic'), + 'imp.load_dynamic() required')(meth) + + +@unittest.skipIf(_thread is None, '_thread module is required') class LockTests(unittest.TestCase): """Very basic test of import lock functions.""" @@ -208,9 +223,7 @@ class ImportTests(unittest.TestCase): self.assertIs(orig_path, new_os.path) self.assertIsNot(orig_getenv, new_os.getenv) - @support.cpython_only - @unittest.skipIf(not hasattr(imp, 'load_dynamic'), - 'imp.load_dynamic() required') + @requires_load_dynamic def test_issue15828_load_extensions(self): # Issue 15828 picked up that the adapter between the old imp API # and importlib couldn't handle C extensions @@ -222,6 +235,22 @@ class ImportTests(unittest.TestCase): mod = imp.load_module(example, *x) self.assertEqual(mod.__name__, example) + @requires_load_dynamic + def test_issue16421_multiple_modules_in_one_dll(self): + # Issue 16421: loading several modules from the same compiled file fails + m = '_testimportmultiple' + fileobj, pathname, description = imp.find_module(m) + fileobj.close() + mod0 = imp.load_dynamic(m, pathname) + mod1 = imp.load_dynamic('_testimportmultiple_foo', pathname) + mod2 = imp.load_dynamic('_testimportmultiple_bar', pathname) + self.assertEqual(mod0.__name__, m) + self.assertEqual(mod1.__name__, '_testimportmultiple_foo') + self.assertEqual(mod2.__name__, '_testimportmultiple_bar') + with self.assertRaises(ImportError): + imp.load_dynamic('nonexistent', pathname) + + @requires_load_dynamic def test_load_dynamic_ImportError_path(self): # Issue #1559549 added `name` and `path` attributes to ImportError # in order to provide better detail. Issue #10854 implemented those @@ -233,14 +262,12 @@ class ImportTests(unittest.TestCase): self.assertIn(path, err.exception.path) self.assertEqual(name, err.exception.name) - @support.cpython_only - @unittest.skipIf(not hasattr(imp, 'load_dynamic'), - 'imp.load_dynamic() required') + @requires_load_dynamic def test_load_module_extension_file_is_None(self): # When loading an extension module and the file is None, open one # on the behalf of imp.load_dynamic(). # Issue #15902 - name = '_heapq' + name = '_testimportmultiple' found = imp.find_module(name) if found[0] is not None: found[0].close() @@ -248,6 +275,15 @@ class ImportTests(unittest.TestCase): return imp.load_module(name, None, *found[1:]) + @unittest.skipIf(sys.dont_write_bytecode, + "test meaningful only when writing bytecode") + def test_bug7732(self): + with support.temp_cwd(): + source = support.TESTFN + '.py' + os.mkdir(source) + self.assertRaisesRegex(ImportError, '^No module', + imp.find_module, support.TESTFN, ["."]) + def test_multiple_calls_to_get_data(self): # Issue #18755: make sure multiple calls to get_data() can succeed. loader = imp._LoadSourceCompatibility('imp', imp.__file__, @@ -293,22 +329,6 @@ class ReloadTests(unittest.TestCase): with self.assertRaisesRegex(ImportError, 'html'): imp.reload(parser) - def test_module_replaced(self): - # see #18698 - def code(): - module = type(sys)('top_level') - module.spam = 3 - sys.modules['top_level'] = module - mock = util.mock_modules('top_level', - module_code={'top_level': code}) - with mock: - with util.import_state(meta_path=[mock]): - module = importlib.import_module('top_level') - reloaded = imp.reload(module) - actual = sys.modules['top_level'] - self.assertEqual(actual.spam, 3) - self.assertEqual(reloaded.spam, 3) - class PEP3147Tests(unittest.TestCase): """Tests of PEP 3147.""" @@ -461,20 +481,5 @@ class NullImporterTests(unittest.TestCase): os.rmdir(name) -def test_main(): - tests = [ - ImportTests, - PEP3147Tests, - ReloadTests, - NullImporterTests, - ] - try: - import _thread - except ImportError: - pass - else: - tests.append(LockTests) - support.run_unittest(*tests) - if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py index df08e6a86f..e88141c4e8 100644 --- a/Lib/test/test_import.py +++ b/Lib/test/test_import.py @@ -1,8 +1,8 @@ # We import importlib *ASAP* in order to test #15386 import importlib +import importlib.util from importlib._bootstrap import _get_sourcefile import builtins -import imp from test.test_importlib.import_ import util as importlib_util import marshal import os @@ -22,7 +22,7 @@ import test.support from test.support import ( EnvironmentVarGuard, TESTFN, check_warnings, forget, is_jython, make_legacy_pyc, rmtree, run_unittest, swap_attr, swap_item, temp_umask, - unlink, unload, create_empty_file, cpython_only) + unlink, unload, create_empty_file, cpython_only, TESTFN_UNENCODABLE) from test import script_helper @@ -70,8 +70,6 @@ class ImportTests(unittest.TestCase): def tearDown(self): unload(TESTFN) - setUp = tearDown - def test_case_sensitivity(self): # Brief digression to test that import is case-sensitive: if we got # this far, we know for sure that "random" exists. @@ -129,16 +127,6 @@ class ImportTests(unittest.TestCase): finally: del sys.path[0] - @skip_if_dont_write_bytecode - def test_bug7732(self): - source = TESTFN + '.py' - os.mkdir(source) - try: - self.assertRaisesRegex(ImportError, '^No module', - imp.find_module, TESTFN, ["."]) - finally: - os.rmdir(source) - def test_module_with_large_stack(self, module='longlist'): # Regression test for http://bugs.python.org/issue561858. filename = module + '.py' @@ -161,16 +149,24 @@ class ImportTests(unittest.TestCase): sys.path.append('') importlib.invalidate_caches() + namespace = {} try: make_legacy_pyc(filename) # This used to crash. - exec('import ' + module) + exec('import ' + module, None, namespace) finally: # Cleanup. del sys.path[-1] unlink(filename + 'c') unlink(filename + 'o') + # Remove references to the module (unload the module) + namespace.clear() + try: + del sys.modules[module] + except KeyError: + pass + def test_failing_import_sticks(self): source = TESTFN + ".py" with open(source, "w") as f: @@ -225,7 +221,7 @@ class ImportTests(unittest.TestCase): with open(source, "w") as f: f.write("a = 10\nb=20//0\n") - self.assertRaises(ZeroDivisionError, imp.reload, mod) + self.assertRaises(ZeroDivisionError, importlib.reload, mod) # But we still expect the module to be in sys.modules. mod = sys.modules.get(TESTFN) self.assertIsNot(mod, None, "expected module to be in sys.modules") @@ -280,7 +276,7 @@ class ImportTests(unittest.TestCase): import sys class C: def __del__(self): - import imp + import importlib sys.argv.insert(0, C()) """)) script_helper.assert_python_ok(testfn) @@ -291,7 +287,7 @@ class ImportTests(unittest.TestCase): sys.path.insert(0, os.curdir) try: source = TESTFN + ".py" - compiled = imp.cache_from_source(source) + compiled = importlib.util.cache_from_source(source) with open(source, 'w') as f: pass try: @@ -322,6 +318,14 @@ class ImportTests(unittest.TestCase): stdout, stderr = popen.communicate() self.assertIn(b"ImportError", stdout) + def test_from_import_message_for_nonexistent_module(self): + with self.assertRaisesRegex(ImportError, "^No module named 'bogus'"): + from bogus import foo + + def test_from_import_message_for_existing_module(self): + with self.assertRaisesRegex(ImportError, "^cannot import name 'bogus'"): + from re import bogus + @skip_if_dont_write_bytecode class FilePermissionTests(unittest.TestCase): @@ -332,7 +336,7 @@ class FilePermissionTests(unittest.TestCase): def test_creation_mode(self): mask = 0o022 with temp_umask(mask), _ready_to_import() as (name, path): - cached_path = imp.cache_from_source(path) + cached_path = importlib.util.cache_from_source(path) module = __import__(name) if not os.path.exists(cached_path): self.fail("__import__ did not result in creation of " @@ -350,7 +354,7 @@ class FilePermissionTests(unittest.TestCase): # permissions of .pyc should match those of .py, regardless of mask mode = 0o600 with temp_umask(0o022), _ready_to_import() as (name, path): - cached_path = imp.cache_from_source(path) + cached_path = importlib.util.cache_from_source(path) os.chmod(path, mode) __import__(name) if not os.path.exists(cached_path): @@ -365,7 +369,7 @@ class FilePermissionTests(unittest.TestCase): def test_cached_readonly(self): mode = 0o400 with temp_umask(0o022), _ready_to_import() as (name, path): - cached_path = imp.cache_from_source(path) + cached_path = importlib.util.cache_from_source(path) os.chmod(path, mode) __import__(name) if not os.path.exists(cached_path): @@ -405,7 +409,7 @@ class FilePermissionTests(unittest.TestCase): bytecode_only = path + "c" else: bytecode_only = path + "o" - os.rename(imp.cache_from_source(path), bytecode_only) + os.rename(importlib.util.cache_from_source(path), bytecode_only) m = __import__(name) self.assertEqual(m.x, 'rewritten') @@ -427,7 +431,7 @@ func_filename = func.__code__.co_filename """ dir_name = os.path.abspath(TESTFN) file_name = os.path.join(dir_name, module_name) + os.extsep + "py" - compiled_name = imp.cache_from_source(file_name) + compiled_name = importlib.util.cache_from_source(file_name) def setUp(self): self.sys_path = sys.path[:] @@ -488,7 +492,7 @@ func_filename = func.__code__.co_filename header = f.read(12) code = marshal.load(f) constants = list(code.co_consts) - foreign_code = test_main.__code__ + foreign_code = importlib.import_module.__code__ pos = constants.index(1) constants[pos] = foreign_code code = type(code)(code.co_argcount, code.co_kwonlyargcount, @@ -630,7 +634,7 @@ class OverridingImportBuiltinTests(unittest.TestCase): class PycacheTests(unittest.TestCase): # Test the various PEP 3147 related behaviors. - tag = imp.get_tag() + tag = sys.implementation.cache_tag def _clean(self): forget(TESTFN) @@ -678,10 +682,11 @@ class PycacheTests(unittest.TestCase): # With PEP 3147 cache layout, removing the source but leaving the pyc # file does not satisfy the import. __import__(TESTFN) - pyc_file = imp.cache_from_source(self.source) + pyc_file = importlib.util.cache_from_source(self.source) self.assertTrue(os.path.exists(pyc_file)) os.remove(self.source) forget(TESTFN) + importlib.invalidate_caches() self.assertRaises(ImportError, __import__, TESTFN) @skip_if_dont_write_bytecode @@ -703,7 +708,7 @@ class PycacheTests(unittest.TestCase): def test___cached__(self): # Modules now also have an __cached__ that points to the pyc file. m = __import__(TESTFN) - pyc_file = imp.cache_from_source(TESTFN + '.py') + pyc_file = importlib.util.cache_from_source(TESTFN + '.py') self.assertEqual(m.__cached__, os.path.join(os.curdir, pyc_file)) @skip_if_dont_write_bytecode @@ -738,10 +743,10 @@ class PycacheTests(unittest.TestCase): pass importlib.invalidate_caches() m = __import__('pep3147.foo') - init_pyc = imp.cache_from_source( + init_pyc = importlib.util.cache_from_source( os.path.join('pep3147', '__init__.py')) self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc)) - foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py')) + foo_pyc = importlib.util.cache_from_source(os.path.join('pep3147', 'foo.py')) self.assertEqual(sys.modules['pep3147.foo'].__cached__, os.path.join(os.curdir, foo_pyc)) @@ -765,10 +770,10 @@ class PycacheTests(unittest.TestCase): unload('pep3147') importlib.invalidate_caches() m = __import__('pep3147.foo') - init_pyc = imp.cache_from_source( + init_pyc = importlib.util.cache_from_source( os.path.join('pep3147', '__init__.py')) self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc)) - foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py')) + foo_pyc = importlib.util.cache_from_source(os.path.join('pep3147', 'foo.py')) self.assertEqual(sys.modules['pep3147.foo'].__cached__, os.path.join(os.curdir, foo_pyc)) @@ -852,7 +857,6 @@ class ImportlibBootstrapTests(unittest.TestCase): from importlib import machinery mod = sys.modules['_frozen_importlib'] self.assertIs(machinery.FileFinder, mod.FileFinder) - self.assertIs(imp.new_module, mod.new_module) @cpython_only @@ -1048,17 +1052,16 @@ class ImportTracebackTests(unittest.TestCase): finally: importlib.SourceLoader.load_module = old_load_module - -def test_main(verbose=None): - run_unittest(ImportTests, PycacheTests, FilePermissionTests, - PycRewritingTests, PathsTests, RelativeImportTests, - OverridingImportBuiltinTests, - ImportlibBootstrapTests, GetSourcefileTests, - TestSymbolicallyLinkedPackage, - ImportTracebackTests) + @unittest.skipUnless(TESTFN_UNENCODABLE, 'need TESTFN_UNENCODABLE') + def test_unencodable_filename(self): + # Issue #11619: The Python parser and the import machinery must not + # encode filenames, especially on Windows + pyname = script_helper.make_script('', TESTFN_UNENCODABLE, 'pass') + name = pyname[:-3] + script_helper.assert_python_ok("-c", "mod = __import__(%a)" % name, + __isolated=False) if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. - from test.test_import import test_main - test_main() + unittest.main() diff --git a/Lib/test/test_importhooks.py b/Lib/test/test_importhooks.py deleted file mode 100644 index 2a22d1a186..0000000000 --- a/Lib/test/test_importhooks.py +++ /dev/null @@ -1,250 +0,0 @@ -import sys -import imp -import os -import unittest -from test import support - - -test_src = """\ -def get_name(): - return __name__ -def get_file(): - return __file__ -""" - -absimp = "import sub\n" -relimp = "from . import sub\n" -deeprelimp = "from .... import sub\n" -futimp = "from __future__ import absolute_import\n" - -reload_src = test_src+"""\ -reloaded = True -""" - -test_co = compile(test_src, "<???>", "exec") -reload_co = compile(reload_src, "<???>", "exec") - -test2_oldabs_co = compile(absimp + test_src, "<???>", "exec") -test2_newabs_co = compile(futimp + absimp + test_src, "<???>", "exec") -test2_newrel_co = compile(relimp + test_src, "<???>", "exec") -test2_deeprel_co = compile(deeprelimp + test_src, "<???>", "exec") -test2_futrel_co = compile(futimp + relimp + test_src, "<???>", "exec") - -test_path = "!!!_test_!!!" - - -class TestImporter: - - modules = { - "hooktestmodule": (False, test_co), - "hooktestpackage": (True, test_co), - "hooktestpackage.sub": (True, test_co), - "hooktestpackage.sub.subber": (True, test_co), - "hooktestpackage.oldabs": (False, test2_oldabs_co), - "hooktestpackage.newabs": (False, test2_newabs_co), - "hooktestpackage.newrel": (False, test2_newrel_co), - "hooktestpackage.sub.subber.subest": (True, test2_deeprel_co), - "hooktestpackage.futrel": (False, test2_futrel_co), - "sub": (False, test_co), - "reloadmodule": (False, test_co), - } - - def __init__(self, path=test_path): - if path != test_path: - # if our class is on sys.path_hooks, we must raise - # ImportError for any path item that we can't handle. - raise ImportError - self.path = path - - def _get__path__(self): - raise NotImplementedError - - def find_module(self, fullname, path=None): - if fullname in self.modules: - return self - else: - return None - - def load_module(self, fullname): - ispkg, code = self.modules[fullname] - mod = sys.modules.setdefault(fullname,imp.new_module(fullname)) - mod.__file__ = "<%s>" % self.__class__.__name__ - mod.__loader__ = self - if ispkg: - mod.__path__ = self._get__path__() - exec(code, mod.__dict__) - return mod - - -class MetaImporter(TestImporter): - def _get__path__(self): - return [] - -class PathImporter(TestImporter): - def _get__path__(self): - return [self.path] - - -class ImportBlocker: - """Place an ImportBlocker instance on sys.meta_path and you - can be sure the modules you specified can't be imported, even - if it's a builtin.""" - def __init__(self, *namestoblock): - self.namestoblock = dict.fromkeys(namestoblock) - def find_module(self, fullname, path=None): - if fullname in self.namestoblock: - return self - return None - def load_module(self, fullname): - raise ImportError("I dare you") - - -class ImpWrapper: - - def __init__(self, path=None): - if path is not None and not os.path.isdir(path): - raise ImportError - self.path = path - - def find_module(self, fullname, path=None): - subname = fullname.split(".")[-1] - if subname != fullname and self.path is None: - return None - if self.path is None: - path = None - else: - path = [self.path] - try: - file, filename, stuff = imp.find_module(subname, path) - except ImportError: - return None - return ImpLoader(file, filename, stuff) - - -class ImpLoader: - - def __init__(self, file, filename, stuff): - self.file = file - self.filename = filename - self.stuff = stuff - - def load_module(self, fullname): - mod = imp.load_module(fullname, self.file, self.filename, self.stuff) - if self.file: - self.file.close() - mod.__loader__ = self # for introspection - return mod - - -class ImportHooksBaseTestCase(unittest.TestCase): - - def setUp(self): - self.path = sys.path[:] - self.meta_path = sys.meta_path[:] - self.path_hooks = sys.path_hooks[:] - sys.path_importer_cache.clear() - self.modules_before = support.modules_setup() - - def tearDown(self): - sys.path[:] = self.path - sys.meta_path[:] = self.meta_path - sys.path_hooks[:] = self.path_hooks - sys.path_importer_cache.clear() - support.modules_cleanup(*self.modules_before) - - -class ImportHooksTestCase(ImportHooksBaseTestCase): - - def doTestImports(self, importer=None): - import hooktestmodule - import hooktestpackage - import hooktestpackage.sub - import hooktestpackage.sub.subber - self.assertEqual(hooktestmodule.get_name(), - "hooktestmodule") - self.assertEqual(hooktestpackage.get_name(), - "hooktestpackage") - self.assertEqual(hooktestpackage.sub.get_name(), - "hooktestpackage.sub") - self.assertEqual(hooktestpackage.sub.subber.get_name(), - "hooktestpackage.sub.subber") - if importer: - self.assertEqual(hooktestmodule.__loader__, importer) - self.assertEqual(hooktestpackage.__loader__, importer) - self.assertEqual(hooktestpackage.sub.__loader__, importer) - self.assertEqual(hooktestpackage.sub.subber.__loader__, importer) - - TestImporter.modules['reloadmodule'] = (False, test_co) - import reloadmodule - self.assertFalse(hasattr(reloadmodule,'reloaded')) - - import hooktestpackage.newrel - self.assertEqual(hooktestpackage.newrel.get_name(), - "hooktestpackage.newrel") - self.assertEqual(hooktestpackage.newrel.sub, - hooktestpackage.sub) - - import hooktestpackage.sub.subber.subest as subest - self.assertEqual(subest.get_name(), - "hooktestpackage.sub.subber.subest") - self.assertEqual(subest.sub, - hooktestpackage.sub) - - import hooktestpackage.futrel - self.assertEqual(hooktestpackage.futrel.get_name(), - "hooktestpackage.futrel") - self.assertEqual(hooktestpackage.futrel.sub, - hooktestpackage.sub) - - import sub - self.assertEqual(sub.get_name(), "sub") - - import hooktestpackage.oldabs - self.assertEqual(hooktestpackage.oldabs.get_name(), - "hooktestpackage.oldabs") - self.assertEqual(hooktestpackage.oldabs.sub, sub) - - import hooktestpackage.newabs - self.assertEqual(hooktestpackage.newabs.get_name(), - "hooktestpackage.newabs") - self.assertEqual(hooktestpackage.newabs.sub, sub) - - def testMetaPath(self): - i = MetaImporter() - sys.meta_path.append(i) - self.doTestImports(i) - - def testPathHook(self): - sys.path_hooks.insert(0, PathImporter) - sys.path.append(test_path) - self.doTestImports() - - def testBlocker(self): - mname = "exceptions" # an arbitrary harmless builtin module - support.unload(mname) - sys.meta_path.append(ImportBlocker(mname)) - self.assertRaises(ImportError, __import__, mname) - - def testImpWrapper(self): - i = ImpWrapper() - sys.meta_path.append(i) - sys.path_hooks.insert(0, ImpWrapper) - mnames = ( - "colorsys", "urllib.parse", "distutils.core", "sys", - ) - for mname in mnames: - parent = mname.split(".")[0] - for n in list(sys.modules): - if n.startswith(parent): - del sys.modules[n] - for mname in mnames: - m = __import__(mname, globals(), locals(), ["__dummy__"]) - # to make sure we actually handled the import - self.assertTrue(hasattr(m, "__loader__")) - - -def test_main(): - support.run_unittest(ImportHooksTestCase) - -if __name__ == "__main__": - test_main() diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py index 76c53e4fd6..2b536e28ce 100644 --- a/Lib/test/test_importlib/extension/test_case_sensitivity.py +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -1,8 +1,9 @@ -import imp import sys from test import support import unittest + from importlib import _bootstrap +from importlib import machinery from .. import util from . import util as ext_util @@ -14,9 +15,9 @@ class ExtensionModuleCaseSensitivityTest(unittest.TestCase): good_name = ext_util.NAME bad_name = good_name.upper() assert good_name != bad_name - finder = _bootstrap.FileFinder(ext_util.PATH, - (_bootstrap.ExtensionFileLoader, - _bootstrap.EXTENSION_SUFFIXES)) + finder = machinery.FileFinder(ext_util.PATH, + (machinery.ExtensionFileLoader, + machinery.EXTENSION_SUFFIXES)) return finder.find_module(bad_name) def test_case_sensitive(self): diff --git a/Lib/test/test_importlib/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py index 1d969a1b28..c4b4f4ea1d 100644 --- a/Lib/test/test_importlib/extension/test_path_hook.py +++ b/Lib/test/test_importlib/extension/test_path_hook.py @@ -2,7 +2,6 @@ from importlib import machinery from . import util import collections -import imp import sys import unittest diff --git a/Lib/test/test_importlib/extension/util.py b/Lib/test/test_importlib/extension/util.py index a266dd98c8..8d089f0d7c 100644 --- a/Lib/test/test_importlib/extension/util.py +++ b/Lib/test/test_importlib/extension/util.py @@ -1,4 +1,3 @@ -import imp from importlib import machinery import os import sys diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index 4b8ec15554..3e801388d4 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -1,9 +1,11 @@ -from importlib import machinery -import imp -import unittest from .. import abc from .. import util + +from importlib import machinery +import unittest from test.support import captured_stdout +import types + class LoaderTests(abc.LoaderTests): @@ -24,7 +26,7 @@ class LoaderTests(abc.LoaderTests): module = machinery.FrozenImporter.load_module('__phello__') check = {'__name__': '__phello__', '__package__': '__phello__', - '__path__': ['__phello__'], + '__path__': [], '__loader__': machinery.FrozenImporter, } for attr, value in check.items(): @@ -85,7 +87,7 @@ class InspectLoaderTests(unittest.TestCase): name = '__hello__' with captured_stdout() as stdout: code = machinery.FrozenImporter.get_code(name) - mod = imp.new_module(name) + mod = types.ModuleType(name) exec(code, mod.__dict__) self.assertTrue(hasattr(mod, 'initialized')) self.assertEqual(stdout.getvalue(), 'Hello world!\n') diff --git a/Lib/test/test_importlib/import_/test___loader__.py b/Lib/test/test_importlib/import_/test___loader__.py new file mode 100644 index 0000000000..535daa0292 --- /dev/null +++ b/Lib/test/test_importlib/import_/test___loader__.py @@ -0,0 +1,44 @@ +import sys +import types +import unittest + +from .. import util +from . import util as import_util + + +class LoaderMock: + + def find_module(self, fullname, path=None): + return self + + def load_module(self, fullname): + sys.modules[fullname] = self.module + return self.module + + +class LoaderAttributeTests(unittest.TestCase): + + def test___loader___missing(self): + module = types.ModuleType('blah') + try: + del module.__loader__ + except AttributeError: + pass + loader = LoaderMock() + loader.module = module + with util.uncache('blah'), util.import_state(meta_path=[loader]): + module = import_util.import_('blah') + self.assertEqual(loader, module.__loader__) + + def test___loader___is_None(self): + module = types.ModuleType('blah') + module.__loader__ = None + loader = LoaderMock() + loader.module = module + with util.uncache('blah'), util.import_state(meta_path=[loader]): + returned_module = import_util.import_('blah') + self.assertEqual(loader, module.__loader__) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py index 3d4cd94889..bf74937e00 100644 --- a/Lib/test/test_importlib/import_/test_api.py +++ b/Lib/test/test_importlib/import_/test_api.py @@ -1,7 +1,7 @@ from .. import util as importlib_test_util from . import util -import imp import sys +import types import unittest @@ -37,7 +37,7 @@ class APITest(unittest.TestCase): def test_nonexistent_fromlist_entry(self): # If something in fromlist doesn't exist, that's okay. # issue15715 - mod = imp.new_module('fine') + mod = types.ModuleType('fine') mod.__path__ = ['XXX'] with importlib_test_util.import_state(meta_path=[BadLoaderFinder]): with importlib_test_util.uncache('fine'): @@ -48,7 +48,7 @@ class APITest(unittest.TestCase): # If something in fromlist triggers an exception not related to not # existing, let that exception propagate. # issue15316 - mod = imp.new_module('fine') + mod = types.ModuleType('fine') mod.__path__ = ['XXX'] with importlib_test_util.import_state(meta_path=[BadLoaderFinder]): with importlib_test_util.uncache('fine'): diff --git a/Lib/test/test_importlib/import_/test_fromlist.py b/Lib/test/test_importlib/import_/test_fromlist.py index c16c33710f..e44056674a 100644 --- a/Lib/test/test_importlib/import_/test_fromlist.py +++ b/Lib/test/test_importlib/import_/test_fromlist.py @@ -1,7 +1,6 @@ """Test that the semantics relating to the 'fromlist' argument are correct.""" from .. import util from . import util as import_util -import imp import unittest class ReturnValue(unittest.TestCase): diff --git a/Lib/test/test_importlib/source/test_abc_loader.py b/Lib/test/test_importlib/source/test_abc_loader.py deleted file mode 100644 index 0d912b6469..0000000000 --- a/Lib/test/test_importlib/source/test_abc_loader.py +++ /dev/null @@ -1,906 +0,0 @@ -import importlib -from importlib import abc - -from .. import abc as testing_abc -from .. import util -from . import util as source_util - -import imp -import inspect -import io -import marshal -import os -import sys -import types -import unittest -import warnings - - -class SourceOnlyLoaderMock(abc.SourceLoader): - - # Globals that should be defined for all modules. - source = (b"_ = '::'.join([__name__, __file__, __cached__, __package__, " - b"repr(__loader__)])") - - def __init__(self, path): - self.path = path - - def get_data(self, path): - assert self.path == path - return self.source - - def get_filename(self, fullname): - return self.path - - def module_repr(self, module): - return '<module>' - - -class SourceLoaderMock(SourceOnlyLoaderMock): - - source_mtime = 1 - - def __init__(self, path, magic=imp.get_magic()): - super().__init__(path) - self.bytecode_path = imp.cache_from_source(self.path) - self.source_size = len(self.source) - data = bytearray(magic) - data.extend(importlib._w_long(self.source_mtime)) - data.extend(importlib._w_long(self.source_size)) - code_object = compile(self.source, self.path, 'exec', - dont_inherit=True) - data.extend(marshal.dumps(code_object)) - self.bytecode = bytes(data) - self.written = {} - - def get_data(self, path): - if path == self.path: - return super().get_data(path) - elif path == self.bytecode_path: - return self.bytecode - else: - raise IOError - - def path_stats(self, path): - assert path == self.path - return {'mtime': self.source_mtime, 'size': self.source_size} - - def set_data(self, path, data): - self.written[path] = bytes(data) - return path == self.bytecode_path - - -class PyLoaderMock(abc.PyLoader): - - # Globals that should be defined for all modules. - source = (b"_ = '::'.join([__name__, __file__, __package__, " - b"repr(__loader__)])") - - def __init__(self, data): - """Take a dict of 'module_name: path' pairings. - - Paths should have no file extension, allowing packages to be denoted by - ending in '__init__'. - - """ - self.module_paths = data - self.path_to_module = {val:key for key,val in data.items()} - - def get_data(self, path): - if path not in self.path_to_module: - raise IOError - return self.source - - def is_package(self, name): - filename = os.path.basename(self.get_filename(name)) - return os.path.splitext(filename)[0] == '__init__' - - def source_path(self, name): - try: - return self.module_paths[name] - except KeyError: - raise ImportError - - def get_filename(self, name): - """Silence deprecation warning.""" - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - path = super().get_filename(name) - assert len(w) == 1 - assert issubclass(w[0].category, DeprecationWarning) - return path - - def module_repr(self): - return '<module>' - - -class PyLoaderCompatMock(PyLoaderMock): - - """Mock that matches what is suggested to have a loader that is compatible - from Python 3.1 onwards.""" - - def get_filename(self, fullname): - try: - return self.module_paths[fullname] - except KeyError: - raise ImportError - - def source_path(self, fullname): - try: - return self.get_filename(fullname) - except ImportError: - return None - - -class PyPycLoaderMock(abc.PyPycLoader, PyLoaderMock): - - default_mtime = 1 - - def __init__(self, source, bc={}): - """Initialize mock. - - 'bc' is a dict keyed on a module's name. The value is dict with - possible keys of 'path', 'mtime', 'magic', and 'bc'. Except for 'path', - each of those keys control if any part of created bytecode is to - deviate from default values. - - """ - super().__init__(source) - self.module_bytecode = {} - self.path_to_bytecode = {} - self.bytecode_to_path = {} - for name, data in bc.items(): - self.path_to_bytecode[data['path']] = name - self.bytecode_to_path[name] = data['path'] - magic = data.get('magic', imp.get_magic()) - mtime = importlib._w_long(data.get('mtime', self.default_mtime)) - source_size = importlib._w_long(len(self.source) & 0xFFFFFFFF) - if 'bc' in data: - bc = data['bc'] - else: - bc = self.compile_bc(name) - self.module_bytecode[name] = magic + mtime + source_size + bc - - def compile_bc(self, name): - source_path = self.module_paths.get(name, '<test>') or '<test>' - code = compile(self.source, source_path, 'exec') - return marshal.dumps(code) - - def source_mtime(self, name): - if name in self.module_paths: - return self.default_mtime - elif name in self.module_bytecode: - return None - else: - raise ImportError - - def bytecode_path(self, name): - try: - return self.bytecode_to_path[name] - except KeyError: - if name in self.module_paths: - return None - else: - raise ImportError - - def write_bytecode(self, name, bytecode): - self.module_bytecode[name] = bytecode - return True - - def get_data(self, path): - if path in self.path_to_module: - return super().get_data(path) - elif path in self.path_to_bytecode: - name = self.path_to_bytecode[path] - return self.module_bytecode[name] - else: - raise IOError - - def is_package(self, name): - try: - return super().is_package(name) - except TypeError: - return '__init__' in self.bytecode_to_path[name] - - def get_code(self, name): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - code_object = super().get_code(name) - assert len(w) == 1 - assert issubclass(w[0].category, DeprecationWarning) - return code_object - -class PyLoaderTests(testing_abc.LoaderTests): - - """Tests for importlib.abc.PyLoader.""" - - mocker = PyLoaderMock - - def eq_attrs(self, ob, **kwargs): - for attr, val in kwargs.items(): - found = getattr(ob, attr) - self.assertEqual(found, val, - "{} attribute: {} != {}".format(attr, found, val)) - - def test_module(self): - name = '<module>' - path = os.path.join('', 'path', 'to', 'module') - mock = self.mocker({name: path}) - with util.uncache(name): - module = mock.load_module(name) - self.assertIn(name, sys.modules) - self.eq_attrs(module, __name__=name, __file__=path, __package__='', - __loader__=mock) - self.assertTrue(not hasattr(module, '__path__')) - return mock, name - - def test_package(self): - name = '<pkg>' - path = os.path.join('path', 'to', name, '__init__') - mock = self.mocker({name: path}) - with util.uncache(name): - module = mock.load_module(name) - self.assertIn(name, sys.modules) - self.eq_attrs(module, __name__=name, __file__=path, - __path__=[os.path.dirname(path)], __package__=name, - __loader__=mock) - return mock, name - - def test_lacking_parent(self): - name = 'pkg.mod' - path = os.path.join('path', 'to', 'pkg', 'mod') - mock = self.mocker({name: path}) - with util.uncache(name): - module = mock.load_module(name) - self.assertIn(name, sys.modules) - self.eq_attrs(module, __name__=name, __file__=path, __package__='pkg', - __loader__=mock) - self.assertFalse(hasattr(module, '__path__')) - return mock, name - - def test_module_reuse(self): - name = 'mod' - path = os.path.join('path', 'to', 'mod') - module = imp.new_module(name) - mock = self.mocker({name: path}) - with util.uncache(name): - sys.modules[name] = module - loaded_module = mock.load_module(name) - self.assertIs(loaded_module, module) - self.assertIs(sys.modules[name], module) - return mock, name - - def test_state_after_failure(self): - name = "mod" - module = imp.new_module(name) - module.blah = None - mock = self.mocker({name: os.path.join('path', 'to', 'mod')}) - mock.source = b"1/0" - with util.uncache(name): - sys.modules[name] = module - with self.assertRaises(ZeroDivisionError): - mock.load_module(name) - self.assertIs(sys.modules[name], module) - self.assertTrue(hasattr(module, 'blah')) - return mock - - def test_unloadable(self): - name = "mod" - mock = self.mocker({name: os.path.join('path', 'to', 'mod')}) - mock.source = b"1/0" - with util.uncache(name): - with self.assertRaises(ZeroDivisionError): - mock.load_module(name) - self.assertNotIn(name, sys.modules) - return mock - - -class PyLoaderCompatTests(PyLoaderTests): - - """Test that the suggested code to make a loader that is compatible from - Python 3.1 forward works.""" - - mocker = PyLoaderCompatMock - - -class PyLoaderInterfaceTests(unittest.TestCase): - - """Tests for importlib.abc.PyLoader to make sure that when source_path() - doesn't return a path everything works as expected.""" - - def test_no_source_path(self): - # No source path should lead to ImportError. - name = 'mod' - mock = PyLoaderMock({}) - with util.uncache(name), self.assertRaises(ImportError): - mock.load_module(name) - - def test_source_path_is_None(self): - name = 'mod' - mock = PyLoaderMock({name: None}) - with util.uncache(name), self.assertRaises(ImportError): - mock.load_module(name) - - def test_get_filename_with_source_path(self): - # get_filename() should return what source_path() returns. - name = 'mod' - path = os.path.join('path', 'to', 'source') - mock = PyLoaderMock({name: path}) - with util.uncache(name): - self.assertEqual(mock.get_filename(name), path) - - def test_get_filename_no_source_path(self): - # get_filename() should raise ImportError if source_path returns None. - name = 'mod' - mock = PyLoaderMock({name: None}) - with util.uncache(name), self.assertRaises(ImportError): - mock.get_filename(name) - - -class PyPycLoaderTests(PyLoaderTests): - - """Tests for importlib.abc.PyPycLoader.""" - - mocker = PyPycLoaderMock - - @source_util.writes_bytecode_files - def verify_bytecode(self, mock, name): - assert name in mock.module_paths - self.assertIn(name, mock.module_bytecode) - magic = mock.module_bytecode[name][:4] - self.assertEqual(magic, imp.get_magic()) - mtime = importlib._r_long(mock.module_bytecode[name][4:8]) - self.assertEqual(mtime, 1) - source_size = mock.module_bytecode[name][8:12] - self.assertEqual(len(mock.source) & 0xFFFFFFFF, - importlib._r_long(source_size)) - bc = mock.module_bytecode[name][12:] - self.assertEqual(bc, mock.compile_bc(name)) - - def test_module(self): - mock, name = super().test_module() - self.verify_bytecode(mock, name) - - def test_package(self): - mock, name = super().test_package() - self.verify_bytecode(mock, name) - - def test_lacking_parent(self): - mock, name = super().test_lacking_parent() - self.verify_bytecode(mock, name) - - def test_module_reuse(self): - mock, name = super().test_module_reuse() - self.verify_bytecode(mock, name) - - def test_state_after_failure(self): - super().test_state_after_failure() - - def test_unloadable(self): - super().test_unloadable() - - -class PyPycLoaderInterfaceTests(unittest.TestCase): - - """Test for the interface of importlib.abc.PyPycLoader.""" - - def get_filename_check(self, src_path, bc_path, expect): - name = 'mod' - mock = PyPycLoaderMock({name: src_path}, {name: {'path': bc_path}}) - with util.uncache(name): - assert mock.source_path(name) == src_path - assert mock.bytecode_path(name) == bc_path - self.assertEqual(mock.get_filename(name), expect) - - def test_filename_with_source_bc(self): - # When source and bytecode paths present, return the source path. - self.get_filename_check('source_path', 'bc_path', 'source_path') - - def test_filename_with_source_no_bc(self): - # With source but no bc, return source path. - self.get_filename_check('source_path', None, 'source_path') - - def test_filename_with_no_source_bc(self): - # With not source but bc, return the bc path. - self.get_filename_check(None, 'bc_path', 'bc_path') - - def test_filename_with_no_source_or_bc(self): - # With no source or bc, raise ImportError. - name = 'mod' - mock = PyPycLoaderMock({name: None}, {name: {'path': None}}) - with util.uncache(name), self.assertRaises(ImportError): - mock.get_filename(name) - - -class SkipWritingBytecodeTests(unittest.TestCase): - - """Test that bytecode is properly handled based on - sys.dont_write_bytecode.""" - - @source_util.writes_bytecode_files - def run_test(self, dont_write_bytecode): - name = 'mod' - mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')}) - sys.dont_write_bytecode = dont_write_bytecode - with util.uncache(name): - mock.load_module(name) - self.assertIsNot(name in mock.module_bytecode, dont_write_bytecode) - - def test_no_bytecode_written(self): - self.run_test(True) - - def test_bytecode_written(self): - self.run_test(False) - - -class RegeneratedBytecodeTests(unittest.TestCase): - - """Test that bytecode is regenerated as expected.""" - - @source_util.writes_bytecode_files - def test_different_magic(self): - # A different magic number should lead to new bytecode. - name = 'mod' - bad_magic = b'\x00\x00\x00\x00' - assert bad_magic != imp.get_magic() - mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')}, - {name: {'path': os.path.join('path', 'to', - 'mod.bytecode'), - 'magic': bad_magic}}) - with util.uncache(name): - mock.load_module(name) - self.assertIn(name, mock.module_bytecode) - magic = mock.module_bytecode[name][:4] - self.assertEqual(magic, imp.get_magic()) - - @source_util.writes_bytecode_files - def test_old_mtime(self): - # Bytecode with an older mtime should be regenerated. - name = 'mod' - old_mtime = PyPycLoaderMock.default_mtime - 1 - mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')}, - {name: {'path': 'path/to/mod.bytecode', 'mtime': old_mtime}}) - with util.uncache(name): - mock.load_module(name) - self.assertIn(name, mock.module_bytecode) - mtime = importlib._r_long(mock.module_bytecode[name][4:8]) - self.assertEqual(mtime, PyPycLoaderMock.default_mtime) - - -class BadBytecodeFailureTests(unittest.TestCase): - - """Test import failures when there is no source and parts of the bytecode - is bad.""" - - def test_bad_magic(self): - # A bad magic number should lead to an ImportError. - name = 'mod' - bad_magic = b'\x00\x00\x00\x00' - bc = {name: - {'path': os.path.join('path', 'to', 'mod'), - 'magic': bad_magic}} - mock = PyPycLoaderMock({name: None}, bc) - with util.uncache(name), self.assertRaises(ImportError) as cm: - mock.load_module(name) - self.assertEqual(cm.exception.name, name) - - def test_no_bytecode(self): - # Missing code object bytecode should lead to an EOFError. - name = 'mod' - bc = {name: {'path': os.path.join('path', 'to', 'mod'), 'bc': b''}} - mock = PyPycLoaderMock({name: None}, bc) - with util.uncache(name), self.assertRaises(EOFError): - mock.load_module(name) - - def test_bad_bytecode(self): - # Malformed code object bytecode should lead to a ValueError. - name = 'mod' - bc = {name: {'path': os.path.join('path', 'to', 'mod'), 'bc': b'1234'}} - mock = PyPycLoaderMock({name: None}, bc) - with util.uncache(name), self.assertRaises(ValueError): - mock.load_module(name) - - -def raise_ImportError(*args, **kwargs): - raise ImportError - -class MissingPathsTests(unittest.TestCase): - - """Test what happens when a source or bytecode path does not exist (either - from *_path returning None or raising ImportError).""" - - def test_source_path_None(self): - # Bytecode should be used when source_path returns None, along with - # __file__ being set to the bytecode path. - name = 'mod' - bytecode_path = 'path/to/mod' - mock = PyPycLoaderMock({name: None}, {name: {'path': bytecode_path}}) - with util.uncache(name): - module = mock.load_module(name) - self.assertEqual(module.__file__, bytecode_path) - - # Testing for bytecode_path returning None handled by all tests where no - # bytecode initially exists. - - def test_all_paths_None(self): - # If all *_path methods return None, raise ImportError. - name = 'mod' - mock = PyPycLoaderMock({name: None}) - with util.uncache(name), self.assertRaises(ImportError) as cm: - mock.load_module(name) - self.assertEqual(cm.exception.name, name) - - def test_source_path_ImportError(self): - # An ImportError from source_path should trigger an ImportError. - name = 'mod' - mock = PyPycLoaderMock({}, {name: {'path': os.path.join('path', 'to', - 'mod')}}) - with util.uncache(name), self.assertRaises(ImportError): - mock.load_module(name) - - def test_bytecode_path_ImportError(self): - # An ImportError from bytecode_path should trigger an ImportError. - name = 'mod' - mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')}) - bad_meth = types.MethodType(raise_ImportError, mock) - mock.bytecode_path = bad_meth - with util.uncache(name), self.assertRaises(ImportError) as cm: - mock.load_module(name) - - -class SourceLoaderTestHarness(unittest.TestCase): - - def setUp(self, *, is_package=True, **kwargs): - self.package = 'pkg' - if is_package: - self.path = os.path.join(self.package, '__init__.py') - self.name = self.package - else: - module_name = 'mod' - self.path = os.path.join(self.package, '.'.join(['mod', 'py'])) - self.name = '.'.join([self.package, module_name]) - self.cached = imp.cache_from_source(self.path) - self.loader = self.loader_mock(self.path, **kwargs) - - def verify_module(self, module): - self.assertEqual(module.__name__, self.name) - self.assertEqual(module.__file__, self.path) - self.assertEqual(module.__cached__, self.cached) - self.assertEqual(module.__package__, self.package) - self.assertEqual(module.__loader__, self.loader) - values = module._.split('::') - self.assertEqual(values[0], self.name) - self.assertEqual(values[1], self.path) - self.assertEqual(values[2], self.cached) - self.assertEqual(values[3], self.package) - self.assertEqual(values[4], repr(self.loader)) - - def verify_code(self, code_object): - module = imp.new_module(self.name) - module.__file__ = self.path - module.__cached__ = self.cached - module.__package__ = self.package - module.__loader__ = self.loader - module.__path__ = [] - exec(code_object, module.__dict__) - self.verify_module(module) - - -class SourceOnlyLoaderTests(SourceLoaderTestHarness): - - """Test importlib.abc.SourceLoader for source-only loading. - - Reload testing is subsumed by the tests for - importlib.util.module_for_loader. - - """ - - loader_mock = SourceOnlyLoaderMock - - def test_get_source(self): - # Verify the source code is returned as a string. - # If an IOError is raised by get_data then raise ImportError. - expected_source = self.loader.source.decode('utf-8') - self.assertEqual(self.loader.get_source(self.name), expected_source) - def raise_IOError(path): - raise IOError - self.loader.get_data = raise_IOError - with self.assertRaises(ImportError) as cm: - self.loader.get_source(self.name) - self.assertEqual(cm.exception.name, self.name) - - def test_is_package(self): - # Properly detect when loading a package. - self.setUp(is_package=False) - self.assertFalse(self.loader.is_package(self.name)) - self.setUp(is_package=True) - self.assertTrue(self.loader.is_package(self.name)) - self.assertFalse(self.loader.is_package(self.name + '.__init__')) - - def test_get_code(self): - # Verify the code object is created. - code_object = self.loader.get_code(self.name) - self.verify_code(code_object) - - def test_load_module(self): - # Loading a module should set __name__, __loader__, __package__, - # __path__ (for packages), __file__, and __cached__. - # The module should also be put into sys.modules. - with util.uncache(self.name): - module = self.loader.load_module(self.name) - self.verify_module(module) - self.assertEqual(module.__path__, [os.path.dirname(self.path)]) - self.assertIn(self.name, sys.modules) - - def test_package_settings(self): - # __package__ needs to be set, while __path__ is set on if the module - # is a package. - # Testing the values for a package are covered by test_load_module. - self.setUp(is_package=False) - with util.uncache(self.name): - module = self.loader.load_module(self.name) - self.verify_module(module) - self.assertTrue(not hasattr(module, '__path__')) - - def test_get_source_encoding(self): - # Source is considered encoded in UTF-8 by default unless otherwise - # specified by an encoding line. - source = "_ = 'ü'" - self.loader.source = source.encode('utf-8') - returned_source = self.loader.get_source(self.name) - self.assertEqual(returned_source, source) - source = "# coding: latin-1\n_ = ü" - self.loader.source = source.encode('latin-1') - returned_source = self.loader.get_source(self.name) - self.assertEqual(returned_source, source) - - -@unittest.skipIf(sys.dont_write_bytecode, "sys.dont_write_bytecode is true") -class SourceLoaderBytecodeTests(SourceLoaderTestHarness): - - """Test importlib.abc.SourceLoader's use of bytecode. - - Source-only testing handled by SourceOnlyLoaderTests. - - """ - - loader_mock = SourceLoaderMock - - def verify_code(self, code_object, *, bytecode_written=False): - super().verify_code(code_object) - if bytecode_written: - self.assertIn(self.cached, self.loader.written) - data = bytearray(imp.get_magic()) - data.extend(importlib._w_long(self.loader.source_mtime)) - data.extend(importlib._w_long(self.loader.source_size)) - data.extend(marshal.dumps(code_object)) - self.assertEqual(self.loader.written[self.cached], bytes(data)) - - def test_code_with_everything(self): - # When everything should work. - code_object = self.loader.get_code(self.name) - self.verify_code(code_object) - - def test_no_bytecode(self): - # If no bytecode exists then move on to the source. - self.loader.bytecode_path = "<does not exist>" - # Sanity check - with self.assertRaises(IOError): - bytecode_path = imp.cache_from_source(self.path) - self.loader.get_data(bytecode_path) - code_object = self.loader.get_code(self.name) - self.verify_code(code_object, bytecode_written=True) - - def test_code_bad_timestamp(self): - # Bytecode is only used when the timestamp matches the source EXACTLY. - for source_mtime in (0, 2): - assert source_mtime != self.loader.source_mtime - original = self.loader.source_mtime - self.loader.source_mtime = source_mtime - # If bytecode is used then EOFError would be raised by marshal. - self.loader.bytecode = self.loader.bytecode[8:] - code_object = self.loader.get_code(self.name) - self.verify_code(code_object, bytecode_written=True) - self.loader.source_mtime = original - - def test_code_bad_magic(self): - # Skip over bytecode with a bad magic number. - self.setUp(magic=b'0000') - # If bytecode is used then EOFError would be raised by marshal. - self.loader.bytecode = self.loader.bytecode[8:] - code_object = self.loader.get_code(self.name) - self.verify_code(code_object, bytecode_written=True) - - def test_dont_write_bytecode(self): - # Bytecode is not written if sys.dont_write_bytecode is true. - # Can assume it is false already thanks to the skipIf class decorator. - try: - sys.dont_write_bytecode = True - self.loader.bytecode_path = "<does not exist>" - code_object = self.loader.get_code(self.name) - self.assertNotIn(self.cached, self.loader.written) - finally: - sys.dont_write_bytecode = False - - def test_no_set_data(self): - # If set_data is not defined, one can still read bytecode. - self.setUp(magic=b'0000') - original_set_data = self.loader.__class__.set_data - try: - del self.loader.__class__.set_data - code_object = self.loader.get_code(self.name) - self.verify_code(code_object) - finally: - self.loader.__class__.set_data = original_set_data - - def test_set_data_raises_exceptions(self): - # Raising NotImplementedError or IOError is okay for set_data. - def raise_exception(exc): - def closure(*args, **kwargs): - raise exc - return closure - - self.setUp(magic=b'0000') - self.loader.set_data = raise_exception(NotImplementedError) - code_object = self.loader.get_code(self.name) - self.verify_code(code_object) - - -class SourceLoaderGetSourceTests(unittest.TestCase): - - """Tests for importlib.abc.SourceLoader.get_source().""" - - def test_default_encoding(self): - # Should have no problems with UTF-8 text. - name = 'mod' - mock = SourceOnlyLoaderMock('mod.file') - source = 'x = "ü"' - mock.source = source.encode('utf-8') - returned_source = mock.get_source(name) - self.assertEqual(returned_source, source) - - def test_decoded_source(self): - # Decoding should work. - name = 'mod' - mock = SourceOnlyLoaderMock("mod.file") - source = "# coding: Latin-1\nx='ü'" - assert source.encode('latin-1') != source.encode('utf-8') - mock.source = source.encode('latin-1') - returned_source = mock.get_source(name) - self.assertEqual(returned_source, source) - - def test_universal_newlines(self): - # PEP 302 says universal newlines should be used. - name = 'mod' - mock = SourceOnlyLoaderMock('mod.file') - source = "x = 42\r\ny = -13\r\n" - mock.source = source.encode('utf-8') - expect = io.IncrementalNewlineDecoder(None, True).decode(source) - self.assertEqual(mock.get_source(name), expect) - - -class AbstractMethodImplTests(unittest.TestCase): - - """Test the concrete abstractmethod implementations.""" - - class MetaPathFinder(abc.MetaPathFinder): - def find_module(self, fullname, path): - super().find_module(fullname, path) - - class PathEntryFinder(abc.PathEntryFinder): - def find_module(self, _): - super().find_module(_) - - def find_loader(self, _): - super().find_loader(_) - - class Finder(abc.Finder): - def find_module(self, fullname, path): - super().find_module(fullname, path) - - class Loader(abc.Loader): - def load_module(self, fullname): - super().load_module(fullname) - def module_repr(self, module): - super().module_repr(module) - - class ResourceLoader(Loader, abc.ResourceLoader): - def get_data(self, _): - super().get_data(_) - - class InspectLoader(Loader, abc.InspectLoader): - def is_package(self, _): - super().is_package(_) - - def get_code(self, _): - super().get_code(_) - - def get_source(self, _): - super().get_source(_) - - class ExecutionLoader(InspectLoader, abc.ExecutionLoader): - def get_filename(self, _): - super().get_filename(_) - - class SourceLoader(ResourceLoader, ExecutionLoader, abc.SourceLoader): - pass - - class PyLoader(ResourceLoader, InspectLoader, abc.PyLoader): - def source_path(self, _): - super().source_path(_) - - class PyPycLoader(PyLoader, abc.PyPycLoader): - def bytecode_path(self, _): - super().bytecode_path(_) - - def source_mtime(self, _): - super().source_mtime(_) - - def write_bytecode(self, _, _2): - super().write_bytecode(_, _2) - - def raises_NotImplementedError(self, ins, *args): - for method_name in args: - method = getattr(ins, method_name) - arg_count = len(inspect.getfullargspec(method)[0]) - 1 - args = [''] * arg_count - try: - method(*args) - except NotImplementedError: - pass - else: - msg = "{}.{} did not raise NotImplementedError" - self.fail(msg.format(ins.__class__.__name__, method_name)) - - def test_Loader(self): - self.raises_NotImplementedError(self.Loader(), 'load_module') - - # XXX misplaced; should be somewhere else - def test_Finder(self): - self.raises_NotImplementedError(self.Finder(), 'find_module') - - def test_ResourceLoader(self): - self.raises_NotImplementedError(self.ResourceLoader(), 'load_module', - 'get_data') - - def test_InspectLoader(self): - self.raises_NotImplementedError(self.InspectLoader(), 'load_module', - 'is_package', 'get_code', 'get_source') - - def test_ExecutionLoader(self): - self.raises_NotImplementedError(self.ExecutionLoader(), 'load_module', - 'is_package', 'get_code', 'get_source', - 'get_filename') - - def test_SourceLoader(self): - ins = self.SourceLoader() - # Required abstractmethods. - self.raises_NotImplementedError(ins, 'get_filename', 'get_data') - # Optional abstractmethods. - self.raises_NotImplementedError(ins,'path_stats', 'set_data') - - def test_PyLoader(self): - self.raises_NotImplementedError(self.PyLoader(), 'source_path', - 'get_data', 'is_package') - - def test_PyPycLoader(self): - self.raises_NotImplementedError(self.PyPycLoader(), 'source_path', - 'source_mtime', 'bytecode_path', - 'write_bytecode') - - -def test_main(): - from test.support import run_unittest - run_unittest(PyLoaderTests, PyLoaderCompatTests, - PyLoaderInterfaceTests, - PyPycLoaderTests, PyPycLoaderInterfaceTests, - SkipWritingBytecodeTests, RegeneratedBytecodeTests, - BadBytecodeFailureTests, MissingPathsTests, - SourceOnlyLoaderTests, - SourceLoaderBytecodeTests, - SourceLoaderGetSourceTests, - AbstractMethodImplTests) - - -if __name__ == '__main__': - test_main() diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py index 241173fb44..bb78d2ed86 100644 --- a/Lib/test/test_importlib/source/test_case_sensitivity.py +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -1,9 +1,9 @@ """Test case-sensitivity (PEP 235).""" -from importlib import _bootstrap -from importlib import machinery from .. import util from . import util as source_util -import imp + +from importlib import _bootstrap +from importlib import machinery import os import sys from test import support as test_support diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py index d59969a8ce..66ad96e68e 100644 --- a/Lib/test/test_importlib/source/test_file_loader.py +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -1,21 +1,22 @@ from importlib import machinery import importlib import importlib.abc +import importlib.util from .. import abc from .. import util from . import util as source_util import errno -import imp import marshal import os import py_compile import shutil import stat import sys +import types import unittest -from test.support import make_legacy_pyc +from test.support import make_legacy_pyc, unload class SimpleTest(unittest.TestCase): @@ -26,23 +27,13 @@ class SimpleTest(unittest.TestCase): """ def test_load_module_API(self): - # If fullname is not specified that assume self.name is desired. - class TesterMixin(importlib.abc.Loader): - def load_module(self, fullname): return fullname - def module_repr(self, module): return '<module>' - - class Tester(importlib.abc.FileLoader, TesterMixin): - def get_code(self, _): pass - def get_source(self, _): pass - def is_package(self, _): pass + class Tester(importlib.abc.FileLoader): + def get_source(self, _): return 'attr = 42' + def is_package(self, _): return False - name = 'mod_name' - loader = Tester(name, 'some_path') - self.assertEqual(name, loader.load_module()) - self.assertEqual(name, loader.load_module(None)) - self.assertEqual(name, loader.load_module(name)) - with self.assertRaises(ImportError): - loader.load_module(loader.name + 'XXX') + loader = Tester('blah', 'blah.py') + self.addCleanup(unload, 'blah') + module = loader.load_module() # Should not raise an exception. def test_get_filename_API(self): # If fullname is not set then assume self.path is desired. @@ -122,7 +113,7 @@ class SimpleTest(unittest.TestCase): value = '<test>' name = '_temp' with source_util.create_modules(name) as mapping: - orig_module = imp.new_module(name) + orig_module = types.ModuleType(name) for attr in attributes: setattr(orig_module, attr, value) with open(mapping[name], 'w') as file: @@ -154,11 +145,11 @@ class SimpleTest(unittest.TestCase): loader = machinery.SourceFileLoader('_temp', file_path) mod = loader.load_module('_temp') self.assertEqual(file_path, mod.__file__) - self.assertEqual(imp.cache_from_source(file_path), + self.assertEqual(importlib.util.cache_from_source(file_path), mod.__cached__) finally: os.unlink(file_path) - pycache = os.path.dirname(imp.cache_from_source(file_path)) + pycache = os.path.dirname(importlib.util.cache_from_source(file_path)) if os.path.exists(pycache): shutil.rmtree(pycache) @@ -167,7 +158,7 @@ class SimpleTest(unittest.TestCase): # truncated rather than raise an OverflowError. with source_util.create_modules('_temp') as mapping: source = mapping['_temp'] - compiled = imp.cache_from_source(source) + compiled = importlib.util.cache_from_source(source) with open(source, 'w') as f: f.write("x = 5") try: @@ -204,7 +195,7 @@ class BadBytecodeTest(unittest.TestCase): pass py_compile.compile(mapping[name]) if not del_source: - bytecode_path = imp.cache_from_source(mapping[name]) + bytecode_path = importlib.util.cache_from_source(mapping[name]) else: os.unlink(mapping[name]) bytecode_path = make_legacy_pyc(mapping[name]) @@ -332,7 +323,8 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest): def test(name, mapping, bytecode_path): self.import_(mapping[name], name) with open(bytecode_path, 'rb') as bytecode_file: - self.assertEqual(bytecode_file.read(4), imp.get_magic()) + self.assertEqual(bytecode_file.read(4), + importlib.util.MAGIC_NUMBER) self._test_bad_magic(test) @@ -382,7 +374,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest): zeros = b'\x00\x00\x00\x00' with source_util.create_modules('_temp') as mapping: py_compile.compile(mapping['_temp']) - bytecode_path = imp.cache_from_source(mapping['_temp']) + bytecode_path = importlib.util.cache_from_source(mapping['_temp']) with open(bytecode_path, 'r+b') as bytecode_file: bytecode_file.seek(4) bytecode_file.write(zeros) @@ -400,7 +392,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest): with source_util.create_modules('_temp') as mapping: # Create bytecode that will need to be re-created. py_compile.compile(mapping['_temp']) - bytecode_path = imp.cache_from_source(mapping['_temp']) + bytecode_path = importlib.util.cache_from_source(mapping['_temp']) with open(bytecode_path, 'r+b') as bytecode_file: bytecode_file.seek(0) bytecode_file.write(b'\x00\x00\x00\x00') @@ -408,7 +400,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest): os.chmod(bytecode_path, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) try: - # Should not raise IOError! + # Should not raise OSError! self.import_(mapping['_temp'], '_temp') finally: # Make writable for eventual clean-up. @@ -473,13 +465,6 @@ class SourcelessLoaderBadBytecodeTest(BadBytecodeTest): self._test_non_code_marshal(del_source=True) -def test_main(): - from test.support import run_unittest - run_unittest(SimpleTest, - SourceLoaderBadBytecodeTest, - SourcelessLoaderBadBytecodeTest - ) - if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py index 8e4986835d..2d70691e5a 100644 --- a/Lib/test/test_importlib/source/test_finder.py +++ b/Lib/test/test_importlib/source/test_finder.py @@ -3,7 +3,6 @@ from . import util as source_util from importlib import machinery import errno -import imp import os import py_compile import stat diff --git a/Lib/test/test_importlib/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py index 6a78792f07..d320f8e025 100644 --- a/Lib/test/test_importlib/source/test_path_hook.py +++ b/Lib/test/test_importlib/source/test_path_hook.py @@ -1,7 +1,6 @@ from . import util as source_util from importlib import machinery -import imp import unittest diff --git a/Lib/test/test_importlib/source/util.py b/Lib/test/test_importlib/source/util.py index ae65663a67..63cd25adc4 100644 --- a/Lib/test/test_importlib/source/util.py +++ b/Lib/test/test_importlib/source/util.py @@ -2,7 +2,6 @@ from .. import util import contextlib import errno import functools -import imp import os import os.path import sys diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index c620c3771b..979a481b6b 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -1,9 +1,21 @@ -from importlib import abc -from importlib import machinery +import contextlib import inspect +import io +import marshal +import os +import sys +from test import support +import types import unittest +from unittest import mock +from . import util +frozen_init, source_init = util.import_importlib('importlib') +frozen_abc, source_abc = util.import_importlib('importlib.abc') +frozen_util, source_util = util.import_importlib('importlib.util') + +##### Inheritance ############################################################## class InheritanceTests: """Test that the specified class is a subclass/superclass of the expected @@ -14,8 +26,19 @@ class InheritanceTests: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.superclasses = [getattr(self.abc, class_name) + for class_name in self.superclass_names] + if hasattr(self, 'subclass_names'): + # Because test.support.import_fresh_module() creates a new + # importlib._bootstrap per module, inheritance checks fail when + # checking across module boundaries (i.e. the _bootstrap in abc is + # not the same as the one in machinery). That means stealing one of + # the modules from the other to make sure the same instance is used. + self.subclasses = [getattr(self.abc.machinery, class_name) + for class_name in self.subclass_names] assert self.subclasses or self.superclasses, self.__class__ - self.__test = getattr(abc, self.__class__.__name__) + testing = self.__class__.__name__.partition('_')[2] + self.__test = getattr(self.abc, testing) def test_subclasses(self): # Test that the expected subclasses inherit. @@ -29,75 +52,958 @@ class InheritanceTests: self.assertTrue(issubclass(self.__test, superclass), "{0} is not a superclass of {1}".format(superclass, self.__test)) +def create_inheritance_tests(base_class): + def set_frozen(ns): + ns['abc'] = frozen_abc + def set_source(ns): + ns['abc'] = source_abc -class MetaPathFinder(InheritanceTests, unittest.TestCase): + classes = [] + for prefix, ns_set in [('Frozen', set_frozen), ('Source', set_source)]: + classes.append(types.new_class('_'.join([prefix, base_class.__name__]), + (base_class, unittest.TestCase), + exec_body=ns_set)) + return classes - superclasses = [abc.Finder] - subclasses = [machinery.BuiltinImporter, machinery.FrozenImporter, - machinery.PathFinder, machinery.WindowsRegistryFinder] +class MetaPathFinder(InheritanceTests): + superclass_names = ['Finder'] + subclass_names = ['BuiltinImporter', 'FrozenImporter', 'PathFinder', + 'WindowsRegistryFinder'] -class PathEntryFinder(InheritanceTests, unittest.TestCase): +tests = create_inheritance_tests(MetaPathFinder) +Frozen_MetaPathFinderInheritanceTests, Source_MetaPathFinderInheritanceTests = tests - superclasses = [abc.Finder] - subclasses = [machinery.FileFinder] +class PathEntryFinder(InheritanceTests): + superclass_names = ['Finder'] + subclass_names = ['FileFinder'] -class Loader(InheritanceTests, unittest.TestCase): +tests = create_inheritance_tests(PathEntryFinder) +Frozen_PathEntryFinderInheritanceTests, Source_PathEntryFinderInheritanceTests = tests - subclasses = [abc.PyLoader] +class ResourceLoader(InheritanceTests): + superclass_names = ['Loader'] -class ResourceLoader(InheritanceTests, unittest.TestCase): +tests = create_inheritance_tests(ResourceLoader) +Frozen_ResourceLoaderInheritanceTests, Source_ResourceLoaderInheritanceTests = tests - superclasses = [abc.Loader] +class InspectLoader(InheritanceTests): + superclass_names = ['Loader'] + subclass_names = ['BuiltinImporter', 'FrozenImporter', 'ExtensionFileLoader'] -class InspectLoader(InheritanceTests, unittest.TestCase): +tests = create_inheritance_tests(InspectLoader) +Frozen_InspectLoaderInheritanceTests, Source_InspectLoaderInheritanceTests = tests - superclasses = [abc.Loader] - subclasses = [abc.PyLoader, machinery.BuiltinImporter, - machinery.FrozenImporter, machinery.ExtensionFileLoader] +class ExecutionLoader(InheritanceTests): + superclass_names = ['InspectLoader'] + subclass_names = ['ExtensionFileLoader'] -class ExecutionLoader(InheritanceTests, unittest.TestCase): +tests = create_inheritance_tests(ExecutionLoader) +Frozen_ExecutionLoaderInheritanceTests, Source_ExecutionLoaderInheritanceTests = tests - superclasses = [abc.InspectLoader] - subclasses = [abc.PyLoader] +class FileLoader(InheritanceTests): + superclass_names = ['ResourceLoader', 'ExecutionLoader'] + subclass_names = ['SourceFileLoader', 'SourcelessFileLoader'] -class FileLoader(InheritanceTests, unittest.TestCase): +tests = create_inheritance_tests(FileLoader) +Frozen_FileLoaderInheritanceTests, Source_FileLoaderInheritanceTests = tests - superclasses = [abc.ResourceLoader, abc.ExecutionLoader] - subclasses = [machinery.SourceFileLoader, machinery.SourcelessFileLoader] +class SourceLoader(InheritanceTests): + superclass_names = ['ResourceLoader', 'ExecutionLoader'] + subclass_names = ['SourceFileLoader'] -class SourceLoader(InheritanceTests, unittest.TestCase): +tests = create_inheritance_tests(SourceLoader) +Frozen_SourceLoaderInheritanceTests, Source_SourceLoaderInheritanceTests = tests - superclasses = [abc.ResourceLoader, abc.ExecutionLoader] - subclasses = [machinery.SourceFileLoader] +##### Default return values #################################################### +def make_abc_subclasses(base_class): + classes = [] + for kind, abc in [('Frozen', frozen_abc), ('Source', source_abc)]: + name = '_'.join([kind, base_class.__name__]) + base_classes = base_class, getattr(abc, base_class.__name__) + classes.append(types.new_class(name, base_classes)) + return classes +def make_return_value_tests(base_class, test_class): + frozen_class, source_class = make_abc_subclasses(base_class) + tests = [] + for prefix, class_in_test in [('Frozen', frozen_class), ('Source', source_class)]: + def set_ns(ns): + ns['ins'] = class_in_test() + tests.append(types.new_class('_'.join([prefix, test_class.__name__]), + (test_class, unittest.TestCase), + exec_body=set_ns)) + return tests -class PyLoader(InheritanceTests, unittest.TestCase): - superclasses = [abc.Loader, abc.ResourceLoader, abc.ExecutionLoader] +class MetaPathFinder: + def find_module(self, fullname, path): + return super().find_module(fullname, path) -class PyPycLoader(InheritanceTests, unittest.TestCase): +Frozen_MPF, Source_MPF = make_abc_subclasses(MetaPathFinder) - superclasses = [abc.PyLoader] +class MetaPathFinderDefaultsTests: + + def test_find_module(self): + # Default should return None. + self.assertIsNone(self.ins.find_module('something', None)) + + def test_invalidate_caches(self): + # Calling the method is a no-op. + self.ins.invalidate_caches() + + +tests = make_return_value_tests(MetaPathFinder, MetaPathFinderDefaultsTests) +Frozen_MPFDefaultTests, Source_MPFDefaultTests = tests + + +class PathEntryFinder: + + def find_loader(self, fullname): + return super().find_loader(fullname) + +Frozen_PEF, Source_PEF = make_abc_subclasses(PathEntryFinder) + + +class PathEntryFinderDefaultsTests: + + def test_find_loader(self): + self.assertEqual((None, []), self.ins.find_loader('something')) + + def find_module(self): + self.assertEqual(None, self.ins.find_module('something')) + + def test_invalidate_caches(self): + # Should be a no-op. + self.ins.invalidate_caches() + + +tests = make_return_value_tests(PathEntryFinder, PathEntryFinderDefaultsTests) +Frozen_PEFDefaultTests, Source_PEFDefaultTests = tests + + +class Loader: + + def load_module(self, fullname): + return super().load_module(fullname) + + +Frozen_L, Source_L = make_abc_subclasses(Loader) + + +class LoaderDefaultsTests: + + def test_load_module(self): + with self.assertRaises(ImportError): + self.ins.load_module('something') + + def test_module_repr(self): + mod = types.ModuleType('blah') + with self.assertRaises(NotImplementedError): + self.ins.module_repr(mod) + original_repr = repr(mod) + mod.__loader__ = self.ins + # Should still return a proper repr. + self.assertTrue(repr(mod)) + + +tests = make_return_value_tests(Loader, LoaderDefaultsTests) +Frozen_LDefaultTests, SourceLDefaultTests = tests + + +class ResourceLoader(Loader): + + def get_data(self, path): + return super().get_data(path) + + +Frozen_RL, Source_RL = make_abc_subclasses(ResourceLoader) + + +class ResourceLoaderDefaultsTests: + + def test_get_data(self): + with self.assertRaises(IOError): + self.ins.get_data('/some/path') + + +tests = make_return_value_tests(ResourceLoader, ResourceLoaderDefaultsTests) +Frozen_RLDefaultTests, Source_RLDefaultTests = tests + + +class InspectLoader(Loader): + + def is_package(self, fullname): + return super().is_package(fullname) + + def get_source(self, fullname): + return super().get_source(fullname) + + +Frozen_IL, Source_IL = make_abc_subclasses(InspectLoader) + + +class InspectLoaderDefaultsTests: + + def test_is_package(self): + with self.assertRaises(ImportError): + self.ins.is_package('blah') + + def test_get_source(self): + with self.assertRaises(ImportError): + self.ins.get_source('blah') + + +tests = make_return_value_tests(InspectLoader, InspectLoaderDefaultsTests) +Frozen_ILDefaultTests, Source_ILDefaultTests = tests + + +class ExecutionLoader(InspectLoader): + + def get_filename(self, fullname): + return super().get_filename(fullname) + +Frozen_EL, Source_EL = make_abc_subclasses(ExecutionLoader) + + +class ExecutionLoaderDefaultsTests: + + def test_get_filename(self): + with self.assertRaises(ImportError): + self.ins.get_filename('blah') + + +tests = make_return_value_tests(ExecutionLoader, InspectLoaderDefaultsTests) +Frozen_ELDefaultTests, Source_ELDefaultsTests = tests + +##### Loader concrete methods ################################################## +class LoaderConcreteMethodTests: + + def test_init_module_attrs(self): + loader = self.LoaderSubclass() + module = types.ModuleType('blah') + loader.init_module_attrs(module) + self.assertEqual(module.__loader__, loader) + + +class Frozen_LoaderConcreateMethodTests(LoaderConcreteMethodTests, unittest.TestCase): + LoaderSubclass = Frozen_L + +class Source_LoaderConcreateMethodTests(LoaderConcreteMethodTests, unittest.TestCase): + LoaderSubclass = Source_L + + +##### InspectLoader concrete methods ########################################### +class InspectLoaderSourceToCodeTests: + + def source_to_module(self, data, path=None): + """Help with source_to_code() tests.""" + module = types.ModuleType('blah') + loader = self.InspectLoaderSubclass() + if path is None: + code = loader.source_to_code(data) + else: + code = loader.source_to_code(data, path) + exec(code, module.__dict__) + return module + + def test_source_to_code_source(self): + # Since compile() can handle strings, so should source_to_code(). + source = 'attr = 42' + module = self.source_to_module(source) + self.assertTrue(hasattr(module, 'attr')) + self.assertEqual(module.attr, 42) + + def test_source_to_code_bytes(self): + # Since compile() can handle bytes, so should source_to_code(). + source = b'attr = 42' + module = self.source_to_module(source) + self.assertTrue(hasattr(module, 'attr')) + self.assertEqual(module.attr, 42) + + def test_source_to_code_path(self): + # Specifying a path should set it for the code object. + path = 'path/to/somewhere' + loader = self.InspectLoaderSubclass() + code = loader.source_to_code('', path) + self.assertEqual(code.co_filename, path) + + def test_source_to_code_no_path(self): + # Not setting a path should still work and be set to <string> since that + # is a pre-existing practice as a default to compile(). + loader = self.InspectLoaderSubclass() + code = loader.source_to_code('') + self.assertEqual(code.co_filename, '<string>') + + +class Frozen_ILSourceToCodeTests(InspectLoaderSourceToCodeTests, unittest.TestCase): + InspectLoaderSubclass = Frozen_IL + +class Source_ILSourceToCodeTests(InspectLoaderSourceToCodeTests, unittest.TestCase): + InspectLoaderSubclass = Source_IL + + +class InspectLoaderGetCodeTests: + + def test_get_code(self): + # Test success. + module = types.ModuleType('blah') + with mock.patch.object(self.InspectLoaderSubclass, 'get_source') as mocked: + mocked.return_value = 'attr = 42' + loader = self.InspectLoaderSubclass() + code = loader.get_code('blah') + exec(code, module.__dict__) + self.assertEqual(module.attr, 42) + + def test_get_code_source_is_None(self): + # If get_source() is None then this should be None. + with mock.patch.object(self.InspectLoaderSubclass, 'get_source') as mocked: + mocked.return_value = None + loader = self.InspectLoaderSubclass() + code = loader.get_code('blah') + self.assertIsNone(code) + + def test_get_code_source_not_found(self): + # If there is no source then there is no code object. + loader = self.InspectLoaderSubclass() + with self.assertRaises(ImportError): + loader.get_code('blah') + + +class Frozen_ILGetCodeTests(InspectLoaderGetCodeTests, unittest.TestCase): + InspectLoaderSubclass = Frozen_IL + +class Source_ILGetCodeTests(InspectLoaderGetCodeTests, unittest.TestCase): + InspectLoaderSubclass = Source_IL + + +class InspectLoaderInitModuleTests: + + def mock_is_package(self, return_value): + return mock.patch.object(self.InspectLoaderSubclass, 'is_package', + return_value=return_value) + + def init_module_attrs(self, name): + loader = self.InspectLoaderSubclass() + module = types.ModuleType(name) + loader.init_module_attrs(module) + self.assertEqual(module.__loader__, loader) + return module + + def test_package(self): + # If a package, then __package__ == __name__, __path__ == [] + with self.mock_is_package(True): + name = 'blah' + module = self.init_module_attrs(name) + self.assertEqual(module.__package__, name) + self.assertEqual(module.__path__, []) + + def test_toplevel(self): + # If a module is top-level, __package__ == '' + with self.mock_is_package(False): + name = 'blah' + module = self.init_module_attrs(name) + self.assertEqual(module.__package__, '') + + def test_submodule(self): + # If a module is contained within a package then set __package__ to the + # package name. + with self.mock_is_package(False): + name = 'pkg.mod' + module = self.init_module_attrs(name) + self.assertEqual(module.__package__, 'pkg') + + def test_is_package_ImportError(self): + # If is_package() raises ImportError, __package__ should be None and + # __path__ should not be set. + with self.mock_is_package(False) as mocked_method: + mocked_method.side_effect = ImportError + name = 'mod' + module = self.init_module_attrs(name) + self.assertIsNone(module.__package__) + self.assertFalse(hasattr(module, '__path__')) + + +class Frozen_ILInitModuleTests(InspectLoaderInitModuleTests, unittest.TestCase): + InspectLoaderSubclass = Frozen_IL + +class Source_ILInitModuleTests(InspectLoaderInitModuleTests, unittest.TestCase): + InspectLoaderSubclass = Source_IL + + +class InspectLoaderLoadModuleTests: + + """Test InspectLoader.load_module().""" + + module_name = 'blah' + + def setUp(self): + support.unload(self.module_name) + self.addCleanup(support.unload, self.module_name) + + def mock_get_code(self): + return mock.patch.object(self.InspectLoaderSubclass, 'get_code') + + def test_get_code_ImportError(self): + # If get_code() raises ImportError, it should propagate. + with self.mock_get_code() as mocked_get_code: + mocked_get_code.side_effect = ImportError + with self.assertRaises(ImportError): + loader = self.InspectLoaderSubclass() + loader.load_module(self.module_name) + + def test_get_code_None(self): + # If get_code() returns None, raise ImportError. + with self.mock_get_code() as mocked_get_code: + mocked_get_code.return_value = None + with self.assertRaises(ImportError): + loader = self.InspectLoaderSubclass() + loader.load_module(self.module_name) + + def test_module_returned(self): + # The loaded module should be returned. + code = compile('attr = 42', '<string>', 'exec') + with self.mock_get_code() as mocked_get_code: + mocked_get_code.return_value = code + loader = self.InspectLoaderSubclass() + module = loader.load_module(self.module_name) + self.assertEqual(module, sys.modules[self.module_name]) + + +class Frozen_ILLoadModuleTests(InspectLoaderLoadModuleTests, unittest.TestCase): + InspectLoaderSubclass = Frozen_IL + +class Source_ILLoadModuleTests(InspectLoaderLoadModuleTests, unittest.TestCase): + InspectLoaderSubclass = Source_IL + + +##### ExecutionLoader concrete methods ######################################### +class ExecutionLoaderGetCodeTests: + + def mock_methods(self, *, get_source=False, get_filename=False): + source_mock_context, filename_mock_context = None, None + if get_source: + source_mock_context = mock.patch.object(self.ExecutionLoaderSubclass, + 'get_source') + if get_filename: + filename_mock_context = mock.patch.object(self.ExecutionLoaderSubclass, + 'get_filename') + return source_mock_context, filename_mock_context + + def test_get_code(self): + path = 'blah.py' + source_mock_context, filename_mock_context = self.mock_methods( + get_source=True, get_filename=True) + with source_mock_context as source_mock, filename_mock_context as name_mock: + source_mock.return_value = 'attr = 42' + name_mock.return_value = path + loader = self.ExecutionLoaderSubclass() + code = loader.get_code('blah') + self.assertEqual(code.co_filename, path) + module = types.ModuleType('blah') + exec(code, module.__dict__) + self.assertEqual(module.attr, 42) + + def test_get_code_source_is_None(self): + # If get_source() is None then this should be None. + source_mock_context, _ = self.mock_methods(get_source=True) + with source_mock_context as mocked: + mocked.return_value = None + loader = self.ExecutionLoaderSubclass() + code = loader.get_code('blah') + self.assertIsNone(code) + + def test_get_code_source_not_found(self): + # If there is no source then there is no code object. + loader = self.ExecutionLoaderSubclass() + with self.assertRaises(ImportError): + loader.get_code('blah') + + def test_get_code_no_path(self): + # If get_filename() raises ImportError then simply skip setting the path + # on the code object. + source_mock_context, filename_mock_context = self.mock_methods( + get_source=True, get_filename=True) + with source_mock_context as source_mock, filename_mock_context as name_mock: + source_mock.return_value = 'attr = 42' + name_mock.side_effect = ImportError + loader = self.ExecutionLoaderSubclass() + code = loader.get_code('blah') + self.assertEqual(code.co_filename, '<string>') + module = types.ModuleType('blah') + exec(code, module.__dict__) + self.assertEqual(module.attr, 42) + + +class Frozen_ELGetCodeTests(ExecutionLoaderGetCodeTests, unittest.TestCase): + ExecutionLoaderSubclass = Frozen_EL + +class Source_ELGetCodeTests(ExecutionLoaderGetCodeTests, unittest.TestCase): + ExecutionLoaderSubclass = Source_EL + + +class ExecutionLoaderInitModuleTests: + + def mock_is_package(self, return_value): + return mock.patch.object(self.ExecutionLoaderSubclass, 'is_package', + return_value=return_value) + + @contextlib.contextmanager + def mock_methods(self, is_package, filename): + is_package_manager = self.mock_is_package(is_package) + get_filename_manager = mock.patch.object(self.ExecutionLoaderSubclass, + 'get_filename', return_value=filename) + with is_package_manager as mock_is_package: + with get_filename_manager as mock_get_filename: + yield {'is_package': mock_is_package, + 'get_filename': mock_get_filename} + + def test_toplevel(self): + # Verify __loader__, __file__, and __package__; no __path__. + name = 'blah' + path = os.path.join('some', 'path', '{}.py'.format(name)) + with self.mock_methods(False, path): + loader = self.ExecutionLoaderSubclass() + module = types.ModuleType(name) + loader.init_module_attrs(module) + self.assertIs(module.__loader__, loader) + self.assertEqual(module.__file__, path) + self.assertEqual(module.__package__, '') + self.assertFalse(hasattr(module, '__path__')) + + def test_package(self): + # Verify __loader__, __file__, __package__, and __path__. + name = 'pkg' + path = os.path.join('some', 'pkg', '__init__.py') + with self.mock_methods(True, path): + loader = self.ExecutionLoaderSubclass() + module = types.ModuleType(name) + loader.init_module_attrs(module) + self.assertIs(module.__loader__, loader) + self.assertEqual(module.__file__, path) + self.assertEqual(module.__package__, 'pkg') + self.assertEqual(module.__path__, [os.path.dirname(path)]) + + def test_submodule(self): + # Verify __package__ and not __path__; test_toplevel() takes care of + # other attributes. + name = 'pkg.submodule' + path = os.path.join('some', 'pkg', 'submodule.py') + with self.mock_methods(False, path): + loader = self.ExecutionLoaderSubclass() + module = types.ModuleType(name) + loader.init_module_attrs(module) + self.assertEqual(module.__package__, 'pkg') + self.assertEqual(module.__file__, path) + self.assertFalse(hasattr(module, '__path__')) + + def test_get_filename_ImportError(self): + # If get_filename() raises ImportError, don't set __file__. + name = 'blah' + path = 'blah.py' + with self.mock_methods(False, path) as mocked_methods: + mocked_methods['get_filename'].side_effect = ImportError + loader = self.ExecutionLoaderSubclass() + module = types.ModuleType(name) + loader.init_module_attrs(module) + self.assertFalse(hasattr(module, '__file__')) + + +class Frozen_ELInitModuleTests(ExecutionLoaderInitModuleTests, unittest.TestCase): + ExecutionLoaderSubclass = Frozen_EL + +class Source_ELInitModuleTests(ExecutionLoaderInitModuleTests, unittest.TestCase): + ExecutionLoaderSubclass = Source_EL + + +##### SourceLoader concrete methods ############################################ +class SourceLoader: + + # Globals that should be defined for all modules. + source = (b"_ = '::'.join([__name__, __file__, __cached__, __package__, " + b"repr(__loader__)])") + + def __init__(self, path): + self.path = path + + def get_data(self, path): + if path != self.path: + raise IOError + return self.source + + def get_filename(self, fullname): + return self.path + + def module_repr(self, module): + return '<module>' + + +Frozen_SourceOnlyL, Source_SourceOnlyL = make_abc_subclasses(SourceLoader) + + +class SourceLoader(SourceLoader): + + source_mtime = 1 + + def __init__(self, path, magic=None): + super().__init__(path) + self.bytecode_path = self.util.cache_from_source(self.path) + self.source_size = len(self.source) + if magic is None: + magic = self.util.MAGIC_NUMBER + data = bytearray(magic) + data.extend(self.init._w_long(self.source_mtime)) + data.extend(self.init._w_long(self.source_size)) + code_object = compile(self.source, self.path, 'exec', + dont_inherit=True) + data.extend(marshal.dumps(code_object)) + self.bytecode = bytes(data) + self.written = {} + + def get_data(self, path): + if path == self.path: + return super().get_data(path) + elif path == self.bytecode_path: + return self.bytecode + else: + raise OSError + + def path_stats(self, path): + if path != self.path: + raise IOError + return {'mtime': self.source_mtime, 'size': self.source_size} + + def set_data(self, path, data): + self.written[path] = bytes(data) + return path == self.bytecode_path + + +Frozen_SL, Source_SL = make_abc_subclasses(SourceLoader) +Frozen_SL.util = frozen_util +Source_SL.util = source_util +Frozen_SL.init = frozen_init +Source_SL.init = source_init + + +class SourceLoaderTestHarness: + + def setUp(self, *, is_package=True, **kwargs): + self.package = 'pkg' + if is_package: + self.path = os.path.join(self.package, '__init__.py') + self.name = self.package + else: + module_name = 'mod' + self.path = os.path.join(self.package, '.'.join(['mod', 'py'])) + self.name = '.'.join([self.package, module_name]) + self.cached = self.util.cache_from_source(self.path) + self.loader = self.loader_mock(self.path, **kwargs) + + def verify_module(self, module): + self.assertEqual(module.__name__, self.name) + self.assertEqual(module.__file__, self.path) + self.assertEqual(module.__cached__, self.cached) + self.assertEqual(module.__package__, self.package) + self.assertEqual(module.__loader__, self.loader) + values = module._.split('::') + self.assertEqual(values[0], self.name) + self.assertEqual(values[1], self.path) + self.assertEqual(values[2], self.cached) + self.assertEqual(values[3], self.package) + self.assertEqual(values[4], repr(self.loader)) + + def verify_code(self, code_object): + module = types.ModuleType(self.name) + module.__file__ = self.path + module.__cached__ = self.cached + module.__package__ = self.package + module.__loader__ = self.loader + module.__path__ = [] + exec(code_object, module.__dict__) + self.verify_module(module) + + +class SourceOnlyLoaderTests(SourceLoaderTestHarness): + + """Test importlib.abc.SourceLoader for source-only loading. + + Reload testing is subsumed by the tests for + importlib.util.module_for_loader. + + """ + + def test_get_source(self): + # Verify the source code is returned as a string. + # If an OSError is raised by get_data then raise ImportError. + expected_source = self.loader.source.decode('utf-8') + self.assertEqual(self.loader.get_source(self.name), expected_source) + def raise_OSError(path): + raise OSError + self.loader.get_data = raise_OSError + with self.assertRaises(ImportError) as cm: + self.loader.get_source(self.name) + self.assertEqual(cm.exception.name, self.name) + + def test_is_package(self): + # Properly detect when loading a package. + self.setUp(is_package=False) + self.assertFalse(self.loader.is_package(self.name)) + self.setUp(is_package=True) + self.assertTrue(self.loader.is_package(self.name)) + self.assertFalse(self.loader.is_package(self.name + '.__init__')) + + def test_get_code(self): + # Verify the code object is created. + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + def test_source_to_code(self): + # Verify the compiled code object. + code = self.loader.source_to_code(self.loader.source, self.path) + self.verify_code(code) + + def test_load_module(self): + # Loading a module should set __name__, __loader__, __package__, + # __path__ (for packages), __file__, and __cached__. + # The module should also be put into sys.modules. + with util.uncache(self.name): + module = self.loader.load_module(self.name) + self.verify_module(module) + self.assertEqual(module.__path__, [os.path.dirname(self.path)]) + self.assertIn(self.name, sys.modules) + + def test_package_settings(self): + # __package__ needs to be set, while __path__ is set on if the module + # is a package. + # Testing the values for a package are covered by test_load_module. + self.setUp(is_package=False) + with util.uncache(self.name): + module = self.loader.load_module(self.name) + self.verify_module(module) + self.assertTrue(not hasattr(module, '__path__')) + + def test_get_source_encoding(self): + # Source is considered encoded in UTF-8 by default unless otherwise + # specified by an encoding line. + source = "_ = 'ü'" + self.loader.source = source.encode('utf-8') + returned_source = self.loader.get_source(self.name) + self.assertEqual(returned_source, source) + source = "# coding: latin-1\n_ = ü" + self.loader.source = source.encode('latin-1') + returned_source = self.loader.get_source(self.name) + self.assertEqual(returned_source, source) + + +class Frozen_SourceOnlyLTests(SourceOnlyLoaderTests, unittest.TestCase): + loader_mock = Frozen_SourceOnlyL + util = frozen_util + +class Source_SourceOnlyLTests(SourceOnlyLoaderTests, unittest.TestCase): + loader_mock = Source_SourceOnlyL + util = source_util + + +@unittest.skipIf(sys.dont_write_bytecode, "sys.dont_write_bytecode is true") +class SourceLoaderBytecodeTests(SourceLoaderTestHarness): + + """Test importlib.abc.SourceLoader's use of bytecode. + + Source-only testing handled by SourceOnlyLoaderTests. + + """ + + def verify_code(self, code_object, *, bytecode_written=False): + super().verify_code(code_object) + if bytecode_written: + self.assertIn(self.cached, self.loader.written) + data = bytearray(self.util.MAGIC_NUMBER) + data.extend(self.init._w_long(self.loader.source_mtime)) + data.extend(self.init._w_long(self.loader.source_size)) + data.extend(marshal.dumps(code_object)) + self.assertEqual(self.loader.written[self.cached], bytes(data)) + + def test_code_with_everything(self): + # When everything should work. + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + def test_no_bytecode(self): + # If no bytecode exists then move on to the source. + self.loader.bytecode_path = "<does not exist>" + # Sanity check + with self.assertRaises(OSError): + bytecode_path = self.util.cache_from_source(self.path) + self.loader.get_data(bytecode_path) + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + + def test_code_bad_timestamp(self): + # Bytecode is only used when the timestamp matches the source EXACTLY. + for source_mtime in (0, 2): + assert source_mtime != self.loader.source_mtime + original = self.loader.source_mtime + self.loader.source_mtime = source_mtime + # If bytecode is used then EOFError would be raised by marshal. + self.loader.bytecode = self.loader.bytecode[8:] + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + self.loader.source_mtime = original + + def test_code_bad_magic(self): + # Skip over bytecode with a bad magic number. + self.setUp(magic=b'0000') + # If bytecode is used then EOFError would be raised by marshal. + self.loader.bytecode = self.loader.bytecode[8:] + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + + def test_dont_write_bytecode(self): + # Bytecode is not written if sys.dont_write_bytecode is true. + # Can assume it is false already thanks to the skipIf class decorator. + try: + sys.dont_write_bytecode = True + self.loader.bytecode_path = "<does not exist>" + code_object = self.loader.get_code(self.name) + self.assertNotIn(self.cached, self.loader.written) + finally: + sys.dont_write_bytecode = False + + def test_no_set_data(self): + # If set_data is not defined, one can still read bytecode. + self.setUp(magic=b'0000') + original_set_data = self.loader.__class__.mro()[1].set_data + try: + del self.loader.__class__.mro()[1].set_data + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + finally: + self.loader.__class__.mro()[1].set_data = original_set_data + + def test_set_data_raises_exceptions(self): + # Raising NotImplementedError or OSError is okay for set_data. + def raise_exception(exc): + def closure(*args, **kwargs): + raise exc + return closure + + self.setUp(magic=b'0000') + self.loader.set_data = raise_exception(NotImplementedError) + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + +class Frozen_SLBytecodeTests(SourceLoaderBytecodeTests, unittest.TestCase): + loader_mock = Frozen_SL + init = frozen_init + util = frozen_util + +class SourceSLBytecodeTests(SourceLoaderBytecodeTests, unittest.TestCase): + loader_mock = Source_SL + init = source_init + util = source_util + + +class SourceLoaderGetSourceTests: + + """Tests for importlib.abc.SourceLoader.get_source().""" + + def test_default_encoding(self): + # Should have no problems with UTF-8 text. + name = 'mod' + mock = self.SourceOnlyLoaderMock('mod.file') + source = 'x = "ü"' + mock.source = source.encode('utf-8') + returned_source = mock.get_source(name) + self.assertEqual(returned_source, source) + + def test_decoded_source(self): + # Decoding should work. + name = 'mod' + mock = self.SourceOnlyLoaderMock("mod.file") + source = "# coding: Latin-1\nx='ü'" + assert source.encode('latin-1') != source.encode('utf-8') + mock.source = source.encode('latin-1') + returned_source = mock.get_source(name) + self.assertEqual(returned_source, source) + + def test_universal_newlines(self): + # PEP 302 says universal newlines should be used. + name = 'mod' + mock = self.SourceOnlyLoaderMock('mod.file') + source = "x = 42\r\ny = -13\r\n" + mock.source = source.encode('utf-8') + expect = io.IncrementalNewlineDecoder(None, True).decode(source) + self.assertEqual(mock.get_source(name), expect) + + +class Frozen_SourceOnlyLGetSourceTests(SourceLoaderGetSourceTests, unittest.TestCase): + SourceOnlyLoaderMock = Frozen_SourceOnlyL + +class Source_SourceOnlyLGetSourceTests(SourceLoaderGetSourceTests, unittest.TestCase): + SourceOnlyLoaderMock = Source_SourceOnlyL + + +class SourceLoaderInitModuleAttrTests: + + """Tests for importlib.abc.SourceLoader.init_module_attrs().""" + + def test_init_module_attrs(self): + # If __file__ set, __cached__ == importlib.util.cached_from_source(__file__). + name = 'blah' + path = 'blah.py' + loader = self.SourceOnlyLoaderMock(path) + module = types.ModuleType(name) + loader.init_module_attrs(module) + self.assertEqual(module.__loader__, loader) + self.assertEqual(module.__package__, '') + self.assertEqual(module.__file__, path) + self.assertEqual(module.__cached__, self.util.cache_from_source(path)) + + def test_no_get_filename(self): + # No __file__, no __cached__. + with mock.patch.object(self.SourceOnlyLoaderMock, 'get_filename') as mocked: + mocked.side_effect = ImportError + name = 'blah' + loader = self.SourceOnlyLoaderMock('blah.py') + module = types.ModuleType(name) + loader.init_module_attrs(module) + self.assertFalse(hasattr(module, '__file__')) + self.assertFalse(hasattr(module, '__cached__')) + + +class Frozen_SLInitModuleAttrTests(SourceLoaderInitModuleAttrTests, unittest.TestCase): + SourceOnlyLoaderMock = Frozen_SourceOnlyL + util = frozen_util + + # Difficult to test under source thanks to cross-module mocking needs. + @mock.patch('importlib._bootstrap.cache_from_source') + def test_cache_from_source_NotImplementedError(self, mock_cache_from_source): + # If importlib.util.cache_from_source() raises NotImplementedError don't set + # __cached__. + mock_cache_from_source.side_effect = NotImplementedError + name = 'blah' + path = 'blah.py' + loader = self.SourceOnlyLoaderMock(path) + module = types.ModuleType(name) + loader.init_module_attrs(module) + self.assertEqual(module.__file__, path) + self.assertFalse(hasattr(module, '__cached__')) + + +class Source_SLInitModuleAttrTests(SourceLoaderInitModuleAttrTests, unittest.TestCase): + SourceOnlyLoaderMock = Source_SourceOnlyL + util = source_util -def test_main(): - from test.support import run_unittest - classes = [] - for class_ in globals().values(): - if (inspect.isclass(class_) and - issubclass(class_, unittest.TestCase) and - issubclass(class_, InheritanceTests)): - classes.append(class_) - run_unittest(*classes) if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index b1a58944f3..0c0e851bbc 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -1,6 +1,7 @@ from . import util -import imp + import importlib +from importlib import _bootstrap from importlib import machinery import sys from test import support @@ -98,7 +99,7 @@ class FindLoaderTests(unittest.TestCase): # If a module with __loader__ is in sys.modules, then return it. name = 'some_mod' with util.uncache(name): - module = imp.new_module(name) + module = types.ModuleType(name) loader = 'a loader!' module.__loader__ = loader sys.modules[name] = module @@ -109,12 +110,26 @@ class FindLoaderTests(unittest.TestCase): # If sys.modules[name].__loader__ is None, raise ValueError. name = 'some_mod' with util.uncache(name): - module = imp.new_module(name) + module = types.ModuleType(name) module.__loader__ = None sys.modules[name] = module with self.assertRaises(ValueError): importlib.find_loader(name) + def test_sys_modules_loader_is_not_set(self): + # Should raise ValueError + # Issue #17099 + name = 'some_mod' + with util.uncache(name): + module = types.ModuleType(name) + try: + del module.__loader__ + except AttributeError: + pass + sys.modules[name] = module + with self.assertRaises(ValueError): + importlib.find_loader(name) + def test_success(self): # Return the loader found on sys.meta_path. name = 'some_mod' @@ -136,6 +151,34 @@ class FindLoaderTests(unittest.TestCase): self.assertIsNone(importlib.find_loader('nevergoingtofindthismodule')) +class ReloadTests(unittest.TestCase): + + """Test module reloading for builtin and extension modules.""" + + def test_reload_modules(self): + for mod in ('tokenize', 'time', 'marshal'): + with self.subTest(module=mod): + with support.CleanImport(mod): + module = importlib.import_module(mod) + importlib.reload(module) + + def test_module_replaced(self): + def code(): + import sys + module = type(sys)('top_level') + module.spam = 3 + sys.modules['top_level'] = module + mock = util.mock_modules('top_level', + module_code={'top_level': code}) + with mock: + with util.import_state(meta_path=[mock]): + module = importlib.import_module('top_level') + reloaded = importlib.reload(module) + actual = sys.modules['top_level'] + self.assertEqual(actual.spam, 3) + self.assertEqual(reloaded.spam, 3) + + class InvalidateCacheTests(unittest.TestCase): def test_method_called(self): @@ -162,7 +205,7 @@ class InvalidateCacheTests(unittest.TestCase): def test_method_lacking(self): # There should be no issues if the method is not defined. key = 'gobbledeegook' - sys.path_importer_cache[key] = imp.NullImporter('abc') + sys.path_importer_cache[key] = None self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key)) importlib.invalidate_caches() # Shouldn't trigger an exception. @@ -182,21 +225,13 @@ class StartupTests(unittest.TestCase): # Issue #17098: all modules should have __loader__ defined. for name, module in sys.modules.items(): if isinstance(module, types.ModuleType): - if name in sys.builtin_module_names: - self.assertEqual(importlib.machinery.BuiltinImporter, - module.__loader__) - elif imp.is_frozen(name): - self.assertEqual(importlib.machinery.FrozenImporter, - module.__loader__) - -def test_main(): - from test.support import run_unittest - run_unittest(ImportModuleTests, - FindLoaderTests, - InvalidateCacheTests, - FrozenImportlibTests, - StartupTests) + self.assertTrue(hasattr(module, '__loader__'), + '{!r} lacks a __loader__ attribute'.format(name)) + if importlib.machinery.BuiltinImporter.find_module(name): + self.assertIsNot(module.__loader__, None) + elif importlib.machinery.FrozenImporter.find_module(name): + self.assertIsNot(module.__loader__, None) if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index efc8977fb4..5fcbdae573 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -1,23 +1,121 @@ from importlib import util from . import util as test_util -import imp + +import os import sys +from test import support import types import unittest +import warnings + + +class DecodeSourceBytesTests(unittest.TestCase): + + source = "string ='ü'" + + def test_ut8_default(self): + source_bytes = self.source.encode('utf-8') + self.assertEqual(util.decode_source(source_bytes), self.source) + + def test_specified_encoding(self): + source = '# coding=latin-1\n' + self.source + source_bytes = source.encode('latin-1') + assert source_bytes != source.encode('utf-8') + self.assertEqual(util.decode_source(source_bytes), source) + + def test_universal_newlines(self): + source = '\r\n'.join([self.source, self.source]) + source_bytes = source.encode('utf-8') + self.assertEqual(util.decode_source(source_bytes), + '\n'.join([self.source, self.source])) + + +class ModuleToLoadTests(unittest.TestCase): + + module_name = 'ModuleManagerTest_module' + + def setUp(self): + support.unload(self.module_name) + self.addCleanup(support.unload, self.module_name) + + def test_new_module(self): + # Test a new module is created, inserted into sys.modules, has + # __initializing__ set to True after entering the context manager, + # and __initializing__ set to False after exiting. + with util.module_to_load(self.module_name) as module: + self.assertIn(self.module_name, sys.modules) + self.assertIs(sys.modules[self.module_name], module) + self.assertTrue(module.__initializing__) + self.assertFalse(module.__initializing__) + + def test_new_module_failed(self): + # Test the module is removed from sys.modules. + try: + with util.module_to_load(self.module_name) as module: + self.assertIn(self.module_name, sys.modules) + raise exception + except Exception: + self.assertNotIn(self.module_name, sys.modules) + else: + self.fail('importlib.util.module_to_load swallowed an exception') + + def test_reload(self): + # Test that the same module is in sys.modules. + created_module = types.ModuleType(self.module_name) + sys.modules[self.module_name] = created_module + with util.module_to_load(self.module_name) as module: + self.assertIs(module, created_module) + + def test_reload_failed(self): + # Test that the module was left in sys.modules. + created_module = types.ModuleType(self.module_name) + sys.modules[self.module_name] = created_module + try: + with util.module_to_load(self.module_name) as module: + raise Exception + except Exception: + self.assertIn(self.module_name, sys.modules) + else: + self.fail('importlib.util.module_to_load swallowed an exception') + + def test_reset_name(self): + # If reset_name is true then module.__name__ = name, else leave it be. + odd_name = 'not your typical name' + created_module = types.ModuleType(self.module_name) + created_module.__name__ = odd_name + sys.modules[self.module_name] = created_module + with util.module_to_load(self.module_name) as module: + self.assertEqual(module.__name__, self.module_name) + created_module.__name__ = odd_name + with util.module_to_load(self.module_name, reset_name=False) as module: + self.assertEqual(module.__name__, odd_name) class ModuleForLoaderTests(unittest.TestCase): """Tests for importlib.util.module_for_loader.""" + @staticmethod + def module_for_loader(func): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', PendingDeprecationWarning) + return util.module_for_loader(func) + + def test_warning(self): + # Should raise a PendingDeprecationWarning when used. + with warnings.catch_warnings(): + warnings.simplefilter('error', PendingDeprecationWarning) + with self.assertRaises(PendingDeprecationWarning): + func = util.module_for_loader(lambda x: x) + def return_module(self, name): - fxn = util.module_for_loader(lambda self, module: module) + fxn = self.module_for_loader(lambda self, module: module) return fxn(self, name) def raise_exception(self, name): def to_wrap(self, module): raise ImportError - fxn = util.module_for_loader(to_wrap) + fxn = self.module_for_loader(to_wrap) try: fxn(self, name) except ImportError: @@ -35,12 +133,23 @@ class ModuleForLoaderTests(unittest.TestCase): def test_reload(self): # Test that a module is reused if already in sys.modules. + class FakeLoader: + def is_package(self, name): + return True + @self.module_for_loader + def load_module(self, module): + return module name = 'a.b.c' - module = imp.new_module('a.b.c') + module = types.ModuleType('a.b.c') + module.__loader__ = 42 + module.__package__ = 42 with test_util.uncache(name): sys.modules[name] = module - returned_module = self.return_module(name) + loader = FakeLoader() + returned_module = loader.load_module(name) self.assertIs(returned_module, sys.modules[name]) + self.assertEqual(module.__loader__, loader) + self.assertEqual(module.__package__, name) def test_new_module_failure(self): # Test that a module is removed from sys.modules if added but an @@ -53,7 +162,7 @@ class ModuleForLoaderTests(unittest.TestCase): def test_reload_failure(self): # Test that a failure on reload leaves the module in-place. name = 'a.b.c' - module = imp.new_module(name) + module = types.ModuleType(name) with test_util.uncache(name): sys.modules[name] = module self.raise_exception(name) @@ -61,7 +170,7 @@ class ModuleForLoaderTests(unittest.TestCase): def test_decorator_attrs(self): def fxn(self, module): pass - wrapped = util.module_for_loader(fxn) + wrapped = self.module_for_loader(fxn) self.assertEqual(wrapped.__name__, fxn.__name__) self.assertEqual(wrapped.__qualname__, fxn.__qualname__) @@ -87,7 +196,7 @@ class ModuleForLoaderTests(unittest.TestCase): self._pkg = is_package def is_package(self, name): return self._pkg - @util.module_for_loader + @self.module_for_loader def load_module(self, module): return module @@ -124,26 +233,26 @@ class SetPackageTests(unittest.TestCase): def test_top_level(self): # __package__ should be set to the empty string if a top-level module. # Implicitly tests when package is set to None. - module = imp.new_module('module') + module = types.ModuleType('module') module.__package__ = None self.verify(module, '') def test_package(self): # Test setting __package__ for a package. - module = imp.new_module('pkg') + module = types.ModuleType('pkg') module.__path__ = ['<path>'] module.__package__ = None self.verify(module, 'pkg') def test_submodule(self): # Test __package__ for a module in a package. - module = imp.new_module('pkg.mod') + module = types.ModuleType('pkg.mod') module.__package__ = None self.verify(module, 'pkg') def test_setting_if_missing(self): # __package__ should be set if it is missing. - module = imp.new_module('mod') + module = types.ModuleType('mod') if hasattr(module, '__package__'): delattr(module, '__package__') self.verify(module, '') @@ -151,7 +260,7 @@ class SetPackageTests(unittest.TestCase): def test_leaving_alone(self): # If __package__ is set and not None then leave it alone. for value in (True, False): - module = imp.new_module('mod') + module = types.ModuleType('mod') module.__package__ = value self.verify(module, value) @@ -162,6 +271,37 @@ class SetPackageTests(unittest.TestCase): self.assertEqual(wrapped.__qualname__, fxn.__qualname__) +class SetLoaderTests(unittest.TestCase): + + """Tests importlib.util.set_loader().""" + + class DummyLoader: + @util.set_loader + def load_module(self, module): + return self.module + + def test_no_attribute(self): + loader = self.DummyLoader() + loader.module = types.ModuleType('blah') + try: + del loader.module.__loader__ + except AttributeError: + pass + self.assertEqual(loader, loader.load_module('blah').__loader__) + + def test_attribute_is_None(self): + loader = self.DummyLoader() + loader.module = types.ModuleType('blah') + loader.module.__loader__ = None + self.assertEqual(loader, loader.load_module('blah').__loader__) + + def test_not_reset(self): + loader = self.DummyLoader() + loader.module = types.ModuleType('blah') + loader.module.__loader__ = 42 + self.assertEqual(42, loader.load_module('blah').__loader__) + + class ResolveNameTests(unittest.TestCase): """Tests importlib.util.resolve_name().""" @@ -195,14 +335,131 @@ class ResolveNameTests(unittest.TestCase): util.resolve_name('..bacon', 'spam') -def test_main(): - from test import support - support.run_unittest( - ModuleForLoaderTests, - SetPackageTests, - ResolveNameTests - ) +class MagicNumberTests(unittest.TestCase): + + def test_length(self): + # Should be 4 bytes. + self.assertEqual(len(util.MAGIC_NUMBER), 4) + + def test_incorporates_rn(self): + # The magic number uses \r\n to come out wrong when splitting on lines. + self.assertTrue(util.MAGIC_NUMBER.endswith(b'\r\n')) + + +class PEP3147Tests(unittest.TestCase): + """Tests of PEP 3147-related functions: + cache_from_source and source_from_cache. + + """ + + tag = sys.implementation.cache_tag + + @unittest.skipUnless(sys.implementation.cache_tag is not None, + 'requires sys.implementation.cache_tag not be None') + def test_cache_from_source(self): + # Given the path to a .py file, return the path to its PEP 3147 + # defined .pyc file (i.e. under __pycache__). + path = os.path.join('foo', 'bar', 'baz', 'qux.py') + expect = os.path.join('foo', 'bar', 'baz', '__pycache__', + 'qux.{}.pyc'.format(self.tag)) + self.assertEqual(util.cache_from_source(path, True), expect) + + def test_cache_from_source_no_cache_tag(self): + # No cache tag means NotImplementedError. + with support.swap_attr(sys.implementation, 'cache_tag', None): + with self.assertRaises(NotImplementedError): + util.cache_from_source('whatever.py') + + def test_cache_from_source_no_dot(self): + # Directory with a dot, filename without dot. + path = os.path.join('foo.bar', 'file') + expect = os.path.join('foo.bar', '__pycache__', + 'file{}.pyc'.format(self.tag)) + self.assertEqual(util.cache_from_source(path, True), expect) + + def test_cache_from_source_optimized(self): + # Given the path to a .py file, return the path to its PEP 3147 + # defined .pyo file (i.e. under __pycache__). + path = os.path.join('foo', 'bar', 'baz', 'qux.py') + expect = os.path.join('foo', 'bar', 'baz', '__pycache__', + 'qux.{}.pyo'.format(self.tag)) + self.assertEqual(util.cache_from_source(path, False), expect) + + def test_cache_from_source_cwd(self): + path = 'foo.py' + expect = os.path.join('__pycache__', 'foo.{}.pyc'.format(self.tag)) + self.assertEqual(util.cache_from_source(path, True), expect) + + def test_cache_from_source_override(self): + # When debug_override is not None, it can be any true-ish or false-ish + # value. + path = os.path.join('foo', 'bar', 'baz.py') + partial_expect = os.path.join('foo', 'bar', '__pycache__', + 'baz.{}.py'.format(self.tag)) + self.assertEqual(util.cache_from_source(path, []), partial_expect + 'o') + self.assertEqual(util.cache_from_source(path, [17]), + partial_expect + 'c') + # However if the bool-ishness can't be determined, the exception + # propagates. + class Bearish: + def __bool__(self): raise RuntimeError + with self.assertRaises(RuntimeError): + util.cache_from_source('/foo/bar/baz.py', Bearish()) + + @unittest.skipUnless(os.sep == '\\' and os.altsep == '/', + 'test meaningful only where os.altsep is defined') + def test_sep_altsep_and_sep_cache_from_source(self): + # Windows path and PEP 3147 where sep is right of altsep. + self.assertEqual( + util.cache_from_source('\\foo\\bar\\baz/qux.py', True), + '\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag)) + + @unittest.skipUnless(sys.implementation.cache_tag is not None, + 'requires sys.implementation.cache_tag to not be ' + 'None') + def test_source_from_cache(self): + # Given the path to a PEP 3147 defined .pyc file, return the path to + # its source. This tests the good path. + path = os.path.join('foo', 'bar', 'baz', '__pycache__', + 'qux.{}.pyc'.format(self.tag)) + expect = os.path.join('foo', 'bar', 'baz', 'qux.py') + self.assertEqual(util.source_from_cache(path), expect) + + def test_source_from_cache_no_cache_tag(self): + # If sys.implementation.cache_tag is None, raise NotImplementedError. + path = os.path.join('blah', '__pycache__', 'whatever.pyc') + with support.swap_attr(sys.implementation, 'cache_tag', None): + with self.assertRaises(NotImplementedError): + util.source_from_cache(path) + + def test_source_from_cache_bad_path(self): + # When the path to a pyc file is not in PEP 3147 format, a ValueError + # is raised. + self.assertRaises( + ValueError, util.source_from_cache, '/foo/bar/bazqux.pyc') + + def test_source_from_cache_no_slash(self): + # No slashes at all in path -> ValueError + self.assertRaises( + ValueError, util.source_from_cache, 'foo.cpython-32.pyc') + + def test_source_from_cache_too_few_dots(self): + # Too few dots in final path component -> ValueError + self.assertRaises( + ValueError, util.source_from_cache, '__pycache__/foo.pyc') + + def test_source_from_cache_too_many_dots(self): + # Too many dots in final path component -> ValueError + self.assertRaises( + ValueError, util.source_from_cache, + '__pycache__/foo.cpython-32.foo.pyc') + + def test_source_from_cache_no__pycache__(self): + # Another problem with the path -> ValueError + self.assertRaises( + ValueError, util.source_from_cache, + '/foo/bar/foo.cpython-32.foo.pyc') if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index ef32f7d690..91a63613c1 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -1,9 +1,18 @@ from contextlib import contextmanager -import imp import os.path from test import support import unittest import sys +import types + + +def import_importlib(module_name): + """Import a module from importlib both w/ and w/o _frozen_importlib.""" + fresh = ('importlib',) if '.' in module_name else () + frozen = support.import_fresh_module(module_name) + source = support.import_fresh_module(module_name, fresh=fresh, + blocked=('_frozen_importlib',)) + return frozen, source CASE_INSENSITIVE_FS = True @@ -98,7 +107,7 @@ class mock_modules: package = name.rsplit('.', 1)[0] else: package = import_name - module = imp.new_module(import_name) + module = types.ModuleType(import_name) module.__loader__ = self module.__file__ = '<mock __file__>' module.__package__ = package diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 5cbec9bb17..fb5fe17e22 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -8,10 +8,16 @@ import datetime import collections import os import shutil +import functools +import importlib from os.path import normcase +try: + from concurrent.futures import ThreadPoolExecutor +except ImportError: + ThreadPoolExecutor = None from test.support import run_unittest, TESTFN, DirsOnSysPath - +from test.script_helper import assert_python_ok, assert_python_failure from test import inspect_fodder as mod from test import inspect_fodder2 as mod2 @@ -418,14 +424,14 @@ class TestBuggyCases(GetSourceBase): unicodedata.__file__[-4:] in (".pyc", ".pyo"), "unicodedata is not an external binary module") def test_findsource_binary(self): - self.assertRaises(IOError, inspect.getsource, unicodedata) - self.assertRaises(IOError, inspect.findsource, unicodedata) + self.assertRaises(OSError, inspect.getsource, unicodedata) + self.assertRaises(OSError, inspect.findsource, unicodedata) def test_findsource_code_in_linecache(self): lines = ["x=1"] co = compile(lines[0], "_dynamically_created_file", "exec") - self.assertRaises(IOError, inspect.findsource, co) - self.assertRaises(IOError, inspect.getsource, co) + self.assertRaises(OSError, inspect.findsource, co) + self.assertRaises(OSError, inspect.getsource, co) linecache.cache[co.co_filename] = (1, None, lines, co.co_filename) try: self.assertEqual(inspect.findsource(co), (lines,0)) @@ -650,6 +656,14 @@ class TestClassesAndFunctions(unittest.TestCase): if isinstance(builtin, type): inspect.classify_class_attrs(builtin) + def test_classify_VirtualAttribute(self): + class VA: + @types.DynamicClassAttribute + def ham(self): + return 'eggs' + should_find = inspect.Attribute('ham', 'data', VA, VA.__dict__['ham']) + self.assertIn(should_find, inspect.classify_class_attrs(VA)) + def test_getmembers_descriptors(self): class A(object): dd = _BrokenDataDescriptor() @@ -693,6 +707,13 @@ class TestClassesAndFunctions(unittest.TestCase): self.assertIn(('f', b.f), inspect.getmembers(b)) self.assertIn(('f', b.f), inspect.getmembers(b, inspect.ismethod)) + def test_getmembers_VirtualAttribute(self): + class A: + @types.DynamicClassAttribute + def eggs(self): + return 'spam' + self.assertIn(('eggs', A.__dict__['eggs']), inspect.getmembers(A)) + _global_ref = object() class TestGetClosureVars(unittest.TestCase): @@ -1080,6 +1101,15 @@ class TestGetattrStatic(unittest.TestCase): self.assertEqual(inspect.getattr_static(Thing, 'x'), Thing.x) + def test_classVirtualAttribute(self): + class Thing(object): + @types.DynamicClassAttribute + def x(self): + return self._x + _x = object() + + self.assertEqual(inspect.getattr_static(Thing, 'x'), Thing.__dict__['x']) + def test_inherited_classattribute(self): class Thing(object): x = object() @@ -1736,6 +1766,17 @@ class TestSignatureObject(unittest.TestCase): ((('b', ..., ..., "positional_or_keyword"),), ...)) + # Test we handle __signature__ partway down the wrapper stack + def wrapped_foo_call(): + pass + wrapped_foo_call.__wrapped__ = Foo.__call__ + + self.assertEqual(self.signature(wrapped_foo_call), + ((('a', ..., ..., "positional_or_keyword"), + ('b', ..., ..., "positional_or_keyword")), + ...)) + + def test_signature_on_class(self): class C: def __init__(self, a): @@ -1850,6 +1891,10 @@ class TestSignatureObject(unittest.TestCase): self.assertEqual(self.signature(Wrapped), ((('a', ..., ..., "positional_or_keyword"),), ...)) + # wrapper loop: + Wrapped.__wrapped__ = Wrapped + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + self.signature(Wrapped) def test_signature_on_lambdas(self): self.assertEqual(self.signature((lambda a=10: a)), @@ -2301,6 +2346,103 @@ class TestBoundArguments(unittest.TestCase): self.assertNotEqual(ba, ba4) +class TestUnwrap(unittest.TestCase): + + def test_unwrap_one(self): + def func(a, b): + return a + b + wrapper = functools.lru_cache(maxsize=20)(func) + self.assertIs(inspect.unwrap(wrapper), func) + + def test_unwrap_several(self): + def func(a, b): + return a + b + wrapper = func + for __ in range(10): + @functools.wraps(wrapper) + def wrapper(): + pass + self.assertIsNot(wrapper.__wrapped__, func) + self.assertIs(inspect.unwrap(wrapper), func) + + def test_stop(self): + def func1(a, b): + return a + b + @functools.wraps(func1) + def func2(): + pass + @functools.wraps(func2) + def wrapper(): + pass + func2.stop_here = 1 + unwrapped = inspect.unwrap(wrapper, + stop=(lambda f: hasattr(f, "stop_here"))) + self.assertIs(unwrapped, func2) + + def test_cycle(self): + def func1(): pass + func1.__wrapped__ = func1 + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + inspect.unwrap(func1) + + def func2(): pass + func2.__wrapped__ = func1 + func1.__wrapped__ = func2 + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + inspect.unwrap(func1) + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + inspect.unwrap(func2) + + def test_unhashable(self): + def func(): pass + func.__wrapped__ = None + class C: + __hash__ = None + __wrapped__ = func + self.assertIsNone(inspect.unwrap(C())) + +class TestMain(unittest.TestCase): + def test_only_source(self): + module = importlib.import_module('unittest') + rc, out, err = assert_python_ok('-m', 'inspect', + 'unittest') + lines = out.decode().splitlines() + # ignore the final newline + self.assertEqual(lines[:-1], inspect.getsource(module).splitlines()) + self.assertEqual(err, b'') + + @unittest.skipIf(ThreadPoolExecutor is None, + 'threads required to test __qualname__ for source files') + def test_qualname_source(self): + rc, out, err = assert_python_ok('-m', 'inspect', + 'concurrent.futures:ThreadPoolExecutor') + lines = out.decode().splitlines() + # ignore the final newline + self.assertEqual(lines[:-1], + inspect.getsource(ThreadPoolExecutor).splitlines()) + self.assertEqual(err, b'') + + def test_builtins(self): + module = importlib.import_module('unittest') + _, out, err = assert_python_failure('-m', 'inspect', + 'sys') + lines = err.decode().splitlines() + self.assertEqual(lines, ["Can't get info for builtin modules."]) + + def test_details(self): + module = importlib.import_module('unittest') + rc, out, err = assert_python_ok('-m', 'inspect', + 'unittest', '--details') + output = out.decode() + # Just a quick sanity check on the output + self.assertIn(module.__name__, output) + self.assertIn(module.__file__, output) + self.assertIn(module.__cached__, output) + self.assertEqual(err, b'') + + + + def test_main(): run_unittest( TestDecorators, TestRetrievingSourceCode, TestOneliners, TestBuggyCases, @@ -2308,7 +2450,7 @@ def test_main(): TestGetcallargsFunctions, TestGetcallargsMethods, TestGetcallargsUnboundMethods, TestGetattrStatic, TestGetGeneratorState, TestNoEOL, TestSignatureObject, TestSignatureBind, TestParameterObject, - TestBoundArguments, TestGetClosureVars + TestBoundArguments, TestGetClosureVars, TestUnwrap, TestMain ) if __name__ == "__main__": diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index c198bcc740..4e00622325 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -228,6 +228,47 @@ class IntTestCases(unittest.TestCase): self.assertRaises(TypeError, int, base=10) self.assertRaises(TypeError, int, base=0) + def test_int_base_limits(self): + """Testing the supported limits of the int() base parameter.""" + self.assertEqual(int('0', 5), 0) + with self.assertRaises(ValueError): + int('0', 1) + with self.assertRaises(ValueError): + int('0', 37) + with self.assertRaises(ValueError): + int('0', -909) # An old magic value base from Python 2. + with self.assertRaises(ValueError): + int('0', base=0-(2**234)) + with self.assertRaises(ValueError): + int('0', base=2**234) + # Bases 2 through 36 are supported. + for base in range(2,37): + self.assertEqual(int('0', base=base), 0) + + def test_int_base_bad_types(self): + """Not integer types are not valid bases; issue16772.""" + with self.assertRaises(TypeError): + int('0', 5.5) + with self.assertRaises(TypeError): + int('0', 5.0) + + def test_int_base_indexable(self): + class MyIndexable(object): + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + + # Check out of range bases. + for base in 2**100, -2**100, 1, 37: + with self.assertRaises(ValueError): + int('43', base) + + # Check in-range bases. + self.assertEqual(int('101', base=MyIndexable(2)), 5) + self.assertEqual(int('101', base=MyIndexable(10)), 101) + self.assertEqual(int('101', base=MyIndexable(36)), 1 + 36**2) + def test_non_numeric_input_types(self): # Test possible non-numeric types for the argument x, including # subclasses of the explicitly documented accepted types. diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 9b8920211e..c1ea6f245f 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -165,7 +165,7 @@ class CloseFailureIO(MockRawIO): def close(self): if not self.closed: self.closed = 1 - raise IOError + raise OSError class CCloseFailureIO(CloseFailureIO, io.RawIOBase): pass @@ -601,9 +601,9 @@ class IOTest(unittest.TestCase): def test_flush_error_on_close(self): f = self.open(support.TESTFN, "wb", buffering=0) def bad_flush(): - raise IOError() + raise OSError() f.flush = bad_flush - self.assertRaises(IOError, f.close) # exception not swallowed + self.assertRaises(OSError, f.close) # exception not swallowed self.assertTrue(f.closed) def test_multi_close(self): @@ -762,7 +762,7 @@ class CommonBufferedTests: if s: # The destructor *may* have printed an unraisable error, check it self.assertEqual(len(s.splitlines()), 1) - self.assertTrue(s.startswith("Exception IOError: "), s) + self.assertTrue(s.startswith("Exception OSError: "), s) self.assertTrue(s.endswith(" ignored"), s) def test_repr(self): @@ -778,22 +778,22 @@ class CommonBufferedTests: def test_flush_error_on_close(self): raw = self.MockRawIO() def bad_flush(): - raise IOError() + raise OSError() raw.flush = bad_flush b = self.tp(raw) - self.assertRaises(IOError, b.close) # exception not swallowed + self.assertRaises(OSError, b.close) # exception not swallowed self.assertTrue(b.closed) def test_close_error_on_close(self): raw = self.MockRawIO() def bad_flush(): - raise IOError('flush') + raise OSError('flush') def bad_close(): - raise IOError('close') + raise OSError('close') raw.close = bad_close b = self.tp(raw) b.flush = bad_flush - with self.assertRaises(IOError) as err: # exception not swallowed + with self.assertRaises(OSError) as err: # exception not swallowed b.close() self.assertEqual(err.exception.args, ('close',)) self.assertEqual(err.exception.__context__.args, ('flush',)) @@ -833,6 +833,14 @@ class SizeofTest: bufio = self.tp(rawio, buffer_size=bufsize2) self.assertEqual(sys.getsizeof(bufio), size + bufsize2) + @support.cpython_only + def test_buffer_freeing(self) : + bufsize = 4096 + rawio = self.MockRawIO() + bufio = self.tp(rawio, buffer_size=bufsize) + size = sys.getsizeof(bufio) - bufsize + bufio.close() + self.assertEqual(sys.getsizeof(bufio), size) class BufferedReaderTest(unittest.TestCase, CommonBufferedTests): read_mode = "rb" @@ -1007,8 +1015,8 @@ class BufferedReaderTest(unittest.TestCase, CommonBufferedTests): def test_misbehaved_io(self): rawio = self.MisbehavedRawIO((b"abc", b"d", b"efg")) bufio = self.tp(rawio) - self.assertRaises(IOError, bufio.seek, 0) - self.assertRaises(IOError, bufio.tell) + self.assertRaises(OSError, bufio.seek, 0) + self.assertRaises(OSError, bufio.tell) def test_no_extraneous_read(self): # Issue #9550; when the raw IO object has satisfied the read request, @@ -1059,17 +1067,18 @@ class CBufferedReaderTest(BufferedReaderTest, SizeofTest): bufio = self.tp(rawio) # _pyio.BufferedReader seems to implement reading different, so that # checking this is not so easy. - self.assertRaises(IOError, bufio.read, 10) + self.assertRaises(OSError, bufio.read, 10) def test_garbage_collection(self): # C BufferedReader objects are collected. # The Python version has __del__, so it ends into gc.garbage instead - rawio = self.FileIO(support.TESTFN, "w+b") - f = self.tp(rawio) - f.f = f - wr = weakref.ref(f) - del f - support.gc_collect() + with support.check_warnings(('', ResourceWarning)): + rawio = self.FileIO(support.TESTFN, "w+b") + f = self.tp(rawio) + f.f = f + wr = weakref.ref(f) + del f + support.gc_collect() self.assertTrue(wr() is None, wr) def test_args_error(self): @@ -1312,9 +1321,9 @@ class BufferedWriterTest(unittest.TestCase, CommonBufferedTests): def test_misbehaved_io(self): rawio = self.MisbehavedRawIO() bufio = self.tp(rawio, 5) - self.assertRaises(IOError, bufio.seek, 0) - self.assertRaises(IOError, bufio.tell) - self.assertRaises(IOError, bufio.write, b"abcdef") + self.assertRaises(OSError, bufio.seek, 0) + self.assertRaises(OSError, bufio.tell) + self.assertRaises(OSError, bufio.write, b"abcdef") def test_max_buffer_size_removal(self): with self.assertRaises(TypeError): @@ -1323,11 +1332,11 @@ class BufferedWriterTest(unittest.TestCase, CommonBufferedTests): def test_write_error_on_close(self): raw = self.MockRawIO() def bad_write(b): - raise IOError() + raise OSError() raw.write = bad_write b = self.tp(raw) b.write(b'spam') - self.assertRaises(IOError, b.close) # exception not swallowed + self.assertRaises(OSError, b.close) # exception not swallowed self.assertTrue(b.closed) @@ -1358,13 +1367,14 @@ class CBufferedWriterTest(BufferedWriterTest, SizeofTest): # C BufferedWriter objects are collected, and collecting them flushes # all data to disk. # The Python version has __del__, so it ends into gc.garbage instead - rawio = self.FileIO(support.TESTFN, "w+b") - f = self.tp(rawio) - f.write(b"123xxx") - f.x = f - wr = weakref.ref(f) - del f - support.gc_collect() + with support.check_warnings(('', ResourceWarning)): + rawio = self.FileIO(support.TESTFN, "w+b") + f = self.tp(rawio) + f.write(b"123xxx") + f.x = f + wr = weakref.ref(f) + del f + support.gc_collect() self.assertTrue(wr() is None, wr) with self.open(support.TESTFN, "rb") as f: self.assertEqual(f.read(), b"123xxx") @@ -1397,14 +1407,14 @@ class BufferedRWPairTest(unittest.TestCase): def readable(self): return False - self.assertRaises(IOError, self.tp, NotReadable(), self.MockRawIO()) + self.assertRaises(OSError, self.tp, NotReadable(), self.MockRawIO()) def test_constructor_with_not_writeable(self): class NotWriteable(MockRawIO): def writable(self): return False - self.assertRaises(IOError, self.tp, self.MockRawIO(), NotWriteable()) + self.assertRaises(OSError, self.tp, self.MockRawIO(), NotWriteable()) def test_read(self): pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) @@ -2165,7 +2175,7 @@ class TextIOWrapperTest(unittest.TestCase): if s: # The destructor *may* have printed an unraisable error, check it self.assertEqual(len(s.splitlines()), 1) - self.assertTrue(s.startswith("Exception IOError: "), s) + self.assertTrue(s.startswith("Exception OSError: "), s) self.assertTrue(s.endswith(" ignored"), s) # Systematic tests of the text I/O API @@ -2237,7 +2247,7 @@ class TextIOWrapperTest(unittest.TestCase): f.seek(0) for line in f: self.assertEqual(line, "\xff\n") - self.assertRaises(IOError, f.tell) + self.assertRaises(OSError, f.tell) self.assertEqual(f.tell(), p2) f.close() @@ -2341,7 +2351,7 @@ class TextIOWrapperTest(unittest.TestCase): def readable(self): return False txt = self.TextIOWrapper(UnReadable()) - self.assertRaises(IOError, txt.read) + self.assertRaises(OSError, txt.read) def test_read_one_by_one(self): txt = self.TextIOWrapper(self.BytesIO(b"AA\r\nBB")) @@ -2516,9 +2526,9 @@ class TextIOWrapperTest(unittest.TestCase): def test_flush_error_on_close(self): txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") def bad_flush(): - raise IOError() + raise OSError() txt.flush = bad_flush - self.assertRaises(IOError, txt.close) # exception not swallowed + self.assertRaises(OSError, txt.close) # exception not swallowed self.assertTrue(txt.closed) def test_multi_close(self): @@ -2599,14 +2609,15 @@ class CTextIOWrapperTest(TextIOWrapperTest): # C TextIOWrapper objects are collected, and collecting them flushes # all data to disk. # The Python version has __del__, so it ends in gc.garbage instead. - rawio = io.FileIO(support.TESTFN, "wb") - b = self.BufferedWriter(rawio) - t = self.TextIOWrapper(b, encoding="ascii") - t.write("456def") - t.x = t - wr = weakref.ref(t) - del t - support.gc_collect() + with support.check_warnings(('', ResourceWarning)): + rawio = io.FileIO(support.TESTFN, "wb") + b = self.BufferedWriter(rawio) + t = self.TextIOWrapper(b, encoding="ascii") + t.write("456def") + t.x = t + wr = weakref.ref(t) + del t + support.gc_collect() self.assertTrue(wr() is None, wr) with self.open(support.TESTFN, "rb") as f: self.assertEqual(f.read(), b"456def") @@ -2896,7 +2907,7 @@ class MiscIOTest(unittest.TestCase): for fd in fds: try: os.close(fd) - except EnvironmentError as e: + except OSError as e: if e.errno != errno.EBADF: raise self.addCleanup(cleanup_fds) @@ -3062,15 +3073,18 @@ class SignalsTest(unittest.TestCase): try: wio = self.io.open(w, **fdopen_kwargs) t.start() - signal.alarm(1) # Fill the pipe enough that the write will be blocking. # It will be interrupted by the timer armed above. Since the # other thread has read one byte, the low-level write will # return with a successful (partial) result rather than an EINTR. # The buffered IO layer must check for pending signal # handlers, which in this case will invoke alarm_interrupt(). - self.assertRaises(ZeroDivisionError, - wio.write, item * (support.PIPE_MAX_SIZE // len(item) + 1)) + signal.alarm(1) + try: + self.assertRaises(ZeroDivisionError, + wio.write, item * (support.PIPE_MAX_SIZE // len(item) + 1)) + finally: + signal.alarm(0) t.join() # We got one byte, get another one and check that it isn't a # repeat of the first one. @@ -3084,7 +3098,7 @@ class SignalsTest(unittest.TestCase): # buffer, and block again. try: wio.close() - except IOError as e: + except OSError as e: if e.errno != errno.EBADF: raise @@ -3212,7 +3226,7 @@ class SignalsTest(unittest.TestCase): # buffer, and could block (in case of failure). try: wio.close() - except IOError as e: + except OSError as e: if e.errno != errno.EBADF: raise diff --git a/Lib/test/test_ioctl.py b/Lib/test/test_ioctl.py index 531c9afbb5..efe9f516cc 100644 --- a/Lib/test/test_ioctl.py +++ b/Lib/test/test_ioctl.py @@ -8,7 +8,7 @@ get_attribute(termios, 'TIOCGPGRP') #Can't run tests without this feature try: tty = open("/dev/tty", "rb") -except IOError: +except OSError: raise unittest.SkipTest("Unable to open /dev/tty") else: # Skip if another process is in foreground @@ -86,8 +86,6 @@ class IoctlTests(unittest.TestCase): os.close(mfd) os.close(sfd) -def test_main(): - run_unittest(IoctlTests) if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_iterlen.py b/Lib/test/test_iterlen.py index 9101f6c884..152f5fc0cb 100644 --- a/Lib/test/test_iterlen.py +++ b/Lib/test/test_iterlen.py @@ -45,31 +45,21 @@ import unittest from test import support from itertools import repeat from collections import deque -from builtins import len as _len +from operator import length_hint n = 10 -def len(obj): - try: - return _len(obj) - except TypeError: - try: - # note: this is an internal undocumented API, - # don't rely on it in your own programs - return obj.__length_hint__() - except AttributeError: - raise TypeError class TestInvariantWithoutMutations: def test_invariant(self): it = self.it for i in reversed(range(1, n+1)): - self.assertEqual(len(it), i) + self.assertEqual(length_hint(it), i) next(it) - self.assertEqual(len(it), 0) + self.assertEqual(length_hint(it), 0) self.assertRaises(StopIteration, next, it) - self.assertEqual(len(it), 0) + self.assertEqual(length_hint(it), 0) class TestTemporarilyImmutable(TestInvariantWithoutMutations): @@ -78,12 +68,12 @@ class TestTemporarilyImmutable(TestInvariantWithoutMutations): # length immutability during iteration it = self.it - self.assertEqual(len(it), n) + self.assertEqual(length_hint(it), n) next(it) - self.assertEqual(len(it), n-1) + self.assertEqual(length_hint(it), n-1) self.mutate() self.assertRaises(RuntimeError, next, it) - self.assertEqual(len(it), 0) + self.assertEqual(length_hint(it), 0) ## ------- Concrete Type Tests ------- @@ -92,10 +82,6 @@ class TestRepeat(TestInvariantWithoutMutations, unittest.TestCase): def setUp(self): self.it = repeat(None, n) - def test_no_len_for_infinite_repeat(self): - # The repeat() object can also be infinite - self.assertRaises(TypeError, len, repeat(None)) - class TestXrange(TestInvariantWithoutMutations, unittest.TestCase): def setUp(self): @@ -167,14 +153,15 @@ class TestList(TestInvariantWithoutMutations, unittest.TestCase): it = iter(d) next(it) next(it) - self.assertEqual(len(it), n-2) + self.assertEqual(length_hint(it), n - 2) d.append(n) - self.assertEqual(len(it), n-1) # grow with append + self.assertEqual(length_hint(it), n - 1) # grow with append d[1:] = [] - self.assertEqual(len(it), 0) + self.assertEqual(length_hint(it), 0) self.assertEqual(list(it), []) d.extend(range(20)) - self.assertEqual(len(it), 0) + self.assertEqual(length_hint(it), 0) + class TestListReversed(TestInvariantWithoutMutations, unittest.TestCase): @@ -186,32 +173,41 @@ class TestListReversed(TestInvariantWithoutMutations, unittest.TestCase): it = reversed(d) next(it) next(it) - self.assertEqual(len(it), n-2) + self.assertEqual(length_hint(it), n - 2) d.append(n) - self.assertEqual(len(it), n-2) # ignore append + self.assertEqual(length_hint(it), n - 2) # ignore append d[1:] = [] - self.assertEqual(len(it), 0) + self.assertEqual(length_hint(it), 0) self.assertEqual(list(it), []) # confirm invariant d.extend(range(20)) - self.assertEqual(len(it), 0) + self.assertEqual(length_hint(it), 0) ## -- Check to make sure exceptions are not suppressed by __length_hint__() class BadLen(object): - def __iter__(self): return iter(range(10)) + def __iter__(self): + return iter(range(10)) + def __len__(self): raise RuntimeError('hello') + class BadLengthHint(object): - def __iter__(self): return iter(range(10)) + def __iter__(self): + return iter(range(10)) + def __length_hint__(self): raise RuntimeError('hello') + class NoneLengthHint(object): - def __iter__(self): return iter(range(10)) + def __iter__(self): + return iter(range(10)) + def __length_hint__(self): - return None + return NotImplemented + class TestLengthHintExceptions(unittest.TestCase): diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 514a6b7a20..2672b00c56 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1729,9 +1729,8 @@ class TestVariousIteratorArgs(unittest.TestCase): class LengthTransparency(unittest.TestCase): def test_repeat(self): - from test.test_iterlen import len - self.assertEqual(len(repeat(None, 50)), 50) - self.assertRaises(TypeError, len, repeat(None)) + self.assertEqual(operator.length_hint(repeat(None, 50)), 50) + self.assertEqual(operator.length_hint(repeat(None), 12), 12) class RegressionTests(unittest.TestCase): diff --git a/Lib/test/test_json/test_enum.py b/Lib/test/test_json/test_enum.py new file mode 100644 index 0000000000..10f414898b --- /dev/null +++ b/Lib/test/test_json/test_enum.py @@ -0,0 +1,120 @@ +from enum import Enum, IntEnum +from math import isnan +from test.test_json import PyTest, CTest + +SMALL = 1 +BIG = 1<<32 +HUGE = 1<<64 +REALLY_HUGE = 1<<96 + +class BigNum(IntEnum): + small = SMALL + big = BIG + huge = HUGE + really_huge = REALLY_HUGE + +E = 2.718281 +PI = 3.141593 +TAU = 2 * PI + +class FloatNum(float, Enum): + e = E + pi = PI + tau = TAU + +INF = float('inf') +NEG_INF = float('-inf') +NAN = float('nan') + +class WierdNum(float, Enum): + inf = INF + neg_inf = NEG_INF + nan = NAN + +class TestEnum: + + def test_floats(self): + for enum in FloatNum: + self.assertEqual(self.dumps(enum), repr(enum.value)) + self.assertEqual(float(self.dumps(enum)), enum) + self.assertEqual(self.loads(self.dumps(enum)), enum) + + def test_weird_floats(self): + for enum, expected in zip(WierdNum, ('Infinity', '-Infinity', 'NaN')): + self.assertEqual(self.dumps(enum), expected) + if not isnan(enum): + self.assertEqual(float(self.dumps(enum)), enum) + self.assertEqual(self.loads(self.dumps(enum)), enum) + else: + self.assertTrue(isnan(float(self.dumps(enum)))) + self.assertTrue(isnan(self.loads(self.dumps(enum)))) + + def test_ints(self): + for enum in BigNum: + self.assertEqual(self.dumps(enum), str(enum.value)) + self.assertEqual(int(self.dumps(enum)), enum) + self.assertEqual(self.loads(self.dumps(enum)), enum) + + def test_list(self): + self.assertEqual(self.dumps(list(BigNum)), + str([SMALL, BIG, HUGE, REALLY_HUGE])) + self.assertEqual(self.loads(self.dumps(list(BigNum))), + list(BigNum)) + self.assertEqual(self.dumps(list(FloatNum)), + str([E, PI, TAU])) + self.assertEqual(self.loads(self.dumps(list(FloatNum))), + list(FloatNum)) + self.assertEqual(self.dumps(list(WierdNum)), + '[Infinity, -Infinity, NaN]') + self.assertEqual(self.loads(self.dumps(list(WierdNum)))[:2], + list(WierdNum)[:2]) + self.assertTrue(isnan(self.loads(self.dumps(list(WierdNum)))[2])) + + def test_dict_keys(self): + s, b, h, r = BigNum + e, p, t = FloatNum + i, j, n = WierdNum + d = { + s:'tiny', b:'large', h:'larger', r:'largest', + e:"Euler's number", p:'pi', t:'tau', + i:'Infinity', j:'-Infinity', n:'NaN', + } + nd = self.loads(self.dumps(d)) + self.assertEqual(nd[str(SMALL)], 'tiny') + self.assertEqual(nd[str(BIG)], 'large') + self.assertEqual(nd[str(HUGE)], 'larger') + self.assertEqual(nd[str(REALLY_HUGE)], 'largest') + self.assertEqual(nd[repr(E)], "Euler's number") + self.assertEqual(nd[repr(PI)], 'pi') + self.assertEqual(nd[repr(TAU)], 'tau') + self.assertEqual(nd['Infinity'], 'Infinity') + self.assertEqual(nd['-Infinity'], '-Infinity') + self.assertEqual(nd['NaN'], 'NaN') + + def test_dict_values(self): + d = dict( + tiny=BigNum.small, + large=BigNum.big, + larger=BigNum.huge, + largest=BigNum.really_huge, + e=FloatNum.e, + pi=FloatNum.pi, + tau=FloatNum.tau, + i=WierdNum.inf, + j=WierdNum.neg_inf, + n=WierdNum.nan, + ) + nd = self.loads(self.dumps(d)) + self.assertEqual(nd['tiny'], SMALL) + self.assertEqual(nd['large'], BIG) + self.assertEqual(nd['larger'], HUGE) + self.assertEqual(nd['largest'], REALLY_HUGE) + self.assertEqual(nd['e'], E) + self.assertEqual(nd['pi'], PI) + self.assertEqual(nd['tau'], TAU) + self.assertEqual(nd['i'], INF) + self.assertEqual(nd['j'], NEG_INF) + self.assertTrue(isnan(nd['n'])) + +class TestPyEnum(TestEnum, PyTest): pass +class TestCEnum(TestEnum, CTest): pass diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index 3dd877afdb..7caafdbddd 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -1,4 +1,5 @@ from test.test_json import PyTest, CTest +import re # 2007-10-05 JSONDOCS = [ @@ -100,6 +101,94 @@ class TestFail: #This is for python encoder self.assertRaises(TypeError, self.dumps, data, indent=True) + def test_truncated_input(self): + test_cases = [ + ('', 'Expecting value', 0), + ('[', 'Expecting value', 1), + ('[42', "Expecting ',' delimiter", 3), + ('[42,', 'Expecting value', 4), + ('["', 'Unterminated string starting at', 1), + ('["spam', 'Unterminated string starting at', 1), + ('["spam"', "Expecting ',' delimiter", 7), + ('["spam",', 'Expecting value', 8), + ('{', 'Expecting property name enclosed in double quotes', 1), + ('{"', 'Unterminated string starting at', 1), + ('{"spam', 'Unterminated string starting at', 1), + ('{"spam"', "Expecting ':' delimiter", 7), + ('{"spam":', 'Expecting value', 8), + ('{"spam":42', "Expecting ',' delimiter", 10), + ('{"spam":42,', 'Expecting property name enclosed in double quotes', 11), + ] + test_cases += [ + ('"', 'Unterminated string starting at', 0), + ('"spam', 'Unterminated string starting at', 0), + ] + for data, msg, idx in test_cases: + self.assertRaisesRegex(ValueError, + r'^{0}: line 1 column {1} \(char {2}\)'.format( + re.escape(msg), idx + 1, idx), + self.loads, data) + + def test_unexpected_data(self): + test_cases = [ + ('[,', 'Expecting value', 1), + ('{"spam":[}', 'Expecting value', 9), + ('[42:', "Expecting ',' delimiter", 3), + ('[42 "spam"', "Expecting ',' delimiter", 4), + ('[42,]', 'Expecting value', 4), + ('{"spam":[42}', "Expecting ',' delimiter", 11), + ('["]', 'Unterminated string starting at', 1), + ('["spam":', "Expecting ',' delimiter", 7), + ('["spam",]', 'Expecting value', 8), + ('{:', 'Expecting property name enclosed in double quotes', 1), + ('{,', 'Expecting property name enclosed in double quotes', 1), + ('{42', 'Expecting property name enclosed in double quotes', 1), + ('[{]', 'Expecting property name enclosed in double quotes', 2), + ('{"spam",', "Expecting ':' delimiter", 7), + ('{"spam"}', "Expecting ':' delimiter", 7), + ('[{"spam"]', "Expecting ':' delimiter", 8), + ('{"spam":}', 'Expecting value', 8), + ('[{"spam":]', 'Expecting value', 9), + ('{"spam":42 "ham"', "Expecting ',' delimiter", 11), + ('[{"spam":42]', "Expecting ',' delimiter", 11), + ('{"spam":42,}', 'Expecting property name enclosed in double quotes', 11), + ] + for data, msg, idx in test_cases: + self.assertRaisesRegex(ValueError, + r'^{0}: line 1 column {1} \(char {2}\)'.format( + re.escape(msg), idx + 1, idx), + self.loads, data) + + def test_extra_data(self): + test_cases = [ + ('[]]', 'Extra data', 2), + ('{}}', 'Extra data', 2), + ('[],[]', 'Extra data', 2), + ('{},{}', 'Extra data', 2), + ] + test_cases += [ + ('42,"spam"', 'Extra data', 2), + ('"spam",42', 'Extra data', 6), + ] + for data, msg, idx in test_cases: + self.assertRaisesRegex(ValueError, + r'^{0}: line 1 column {1} - line 1 column {2}' + r' \(char {3} - {4}\)'.format( + re.escape(msg), idx + 1, len(data) + 1, idx, len(data)), + self.loads, data) + + def test_linecol(self): + test_cases = [ + ('!', 1, 1, 0), + (' !', 1, 2, 1), + ('\n!', 2, 1, 1), + ('\n \n\n !', 4, 6, 10), + ] + for data, line, col, idx in test_cases: + self.assertRaisesRegex(ValueError, + r'^Expecting value: line {0} column {1}' + r' \(char {2}\)$'.format(line, col, idx), + self.loads, data) class TestPyFail(TestFail, PyTest): pass class TestCFail(TestFail, CTest): pass diff --git a/Lib/test/test_json/test_indent.py b/Lib/test/test_json/test_indent.py index a4d4d20142..e07856f33c 100644 --- a/Lib/test/test_json/test_indent.py +++ b/Lib/test/test_json/test_indent.py @@ -32,6 +32,8 @@ class TestIndent: d1 = self.dumps(h) d2 = self.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) d3 = self.dumps(h, indent='\t', sort_keys=True, separators=(',', ': ')) + d4 = self.dumps(h, indent=2, sort_keys=True) + d5 = self.dumps(h, indent='\t', sort_keys=True) h1 = self.loads(d1) h2 = self.loads(d2) @@ -42,6 +44,8 @@ class TestIndent: self.assertEqual(h3, h) self.assertEqual(d2, expect.expandtabs(2)) self.assertEqual(d3, expect) + self.assertEqual(d4, d2) + self.assertEqual(d5, d3) def test_indent0(self): h = {3: 1} diff --git a/Lib/test/test_keyword.py b/Lib/test/test_keyword.py new file mode 100644 index 0000000000..af99f52c63 --- /dev/null +++ b/Lib/test/test_keyword.py @@ -0,0 +1,138 @@ +import keyword +import unittest +from test import support +import filecmp +import os +import sys +import subprocess +import shutil +import textwrap + +KEYWORD_FILE = support.findfile('keyword.py') +GRAMMAR_FILE = os.path.join(os.path.split(__file__)[0], + '..', '..', 'Python', 'graminit.c') +TEST_PY_FILE = 'keyword_test.py' +GRAMMAR_TEST_FILE = 'graminit_test.c' +PY_FILE_WITHOUT_KEYWORDS = 'minimal_keyword.py' +NONEXISTENT_FILE = 'not_here.txt' + + +class Test_iskeyword(unittest.TestCase): + def test_true_is_a_keyword(self): + self.assertTrue(keyword.iskeyword('True')) + + def test_uppercase_true_is_not_a_keyword(self): + self.assertFalse(keyword.iskeyword('TRUE')) + + def test_none_value_is_not_a_keyword(self): + self.assertFalse(keyword.iskeyword(None)) + + # This is probably an accident of the current implementation, but should be + # preserved for backward compatibility. + def test_changing_the_kwlist_does_not_affect_iskeyword(self): + oldlist = keyword.kwlist + self.addCleanup(setattr, keyword, 'kwlist', oldlist) + keyword.kwlist = ['its', 'all', 'eggs', 'beans', 'and', 'a', 'slice'] + self.assertFalse(keyword.iskeyword('eggs')) + + +class TestKeywordGeneration(unittest.TestCase): + + def _copy_file_without_generated_keywords(self, source_file, dest_file): + with open(source_file, 'rb') as fp: + lines = fp.readlines() + nl = lines[0][len(lines[0].strip()):] + with open(dest_file, 'wb') as fp: + fp.writelines(lines[:lines.index(b"#--start keywords--" + nl) + 1]) + fp.writelines(lines[lines.index(b"#--end keywords--" + nl):]) + + def _generate_keywords(self, grammar_file, target_keyword_py_file): + proc = subprocess.Popen([sys.executable, + KEYWORD_FILE, + grammar_file, + target_keyword_py_file], stderr=subprocess.PIPE) + stderr = proc.communicate()[1] + return proc.returncode, stderr + + @unittest.skipIf(not os.path.exists(GRAMMAR_FILE), + 'test only works from source build directory') + def test_real_grammar_and_keyword_file(self): + self._copy_file_without_generated_keywords(KEYWORD_FILE, TEST_PY_FILE) + self.addCleanup(support.unlink, TEST_PY_FILE) + self.assertFalse(filecmp.cmp(KEYWORD_FILE, TEST_PY_FILE)) + self.assertEqual((0, b''), self._generate_keywords(GRAMMAR_FILE, + TEST_PY_FILE)) + self.assertTrue(filecmp.cmp(KEYWORD_FILE, TEST_PY_FILE)) + + def test_grammar(self): + self._copy_file_without_generated_keywords(KEYWORD_FILE, TEST_PY_FILE) + self.addCleanup(support.unlink, TEST_PY_FILE) + with open(GRAMMAR_TEST_FILE, 'w') as fp: + # Some of these are probably implementation accidents. + fp.writelines(textwrap.dedent("""\ + {2, 1}, + {11, "encoding_decl", 0, 2, states_79, + "\000\000\040\000\000\000\000\000\000\000\000\000" + "\000\000\000\000\000\000\000\000\000"}, + {1, "jello"}, + {326, 0}, + {1, "turnip"}, + \t{1, "This one is tab indented" + {278, 0}, + {1, "crazy but legal" + "also legal" {1, " + {1, "continue"}, + {1, "lemon"}, + {1, "tomato"}, + {1, "wigii"}, + {1, 'no good'} + {283, 0}, + {1, "too many spaces"}""")) + self.addCleanup(support.unlink, GRAMMAR_TEST_FILE) + self._generate_keywords(GRAMMAR_TEST_FILE, TEST_PY_FILE) + expected = [ + " 'This one is tab indented',", + " 'also legal',", + " 'continue',", + " 'crazy but legal',", + " 'jello',", + " 'lemon',", + " 'tomato',", + " 'turnip',", + " 'wigii',", + ] + with open(TEST_PY_FILE) as fp: + lines = fp.read().splitlines() + start = lines.index("#--start keywords--") + 1 + end = lines.index("#--end keywords--") + actual = lines[start:end] + self.assertEqual(actual, expected) + + def test_empty_grammar_results_in_no_keywords(self): + self._copy_file_without_generated_keywords(KEYWORD_FILE, + PY_FILE_WITHOUT_KEYWORDS) + self.addCleanup(support.unlink, PY_FILE_WITHOUT_KEYWORDS) + shutil.copyfile(KEYWORD_FILE, TEST_PY_FILE) + self.addCleanup(support.unlink, TEST_PY_FILE) + self.assertEqual((0, b''), self._generate_keywords(os.devnull, + TEST_PY_FILE)) + self.assertTrue(filecmp.cmp(TEST_PY_FILE, PY_FILE_WITHOUT_KEYWORDS)) + + def test_keywords_py_without_markers_produces_error(self): + rc, stderr = self._generate_keywords(os.devnull, os.devnull) + self.assertNotEqual(rc, 0) + self.assertRegex(stderr, b'does not contain format markers') + + def test_missing_grammar_file_produces_error(self): + rc, stderr = self._generate_keywords(NONEXISTENT_FILE, KEYWORD_FILE) + self.assertNotEqual(rc, 0) + self.assertRegex(stderr, b'(?ms)' + NONEXISTENT_FILE.encode()) + + def test_missing_keywords_py_file_produces_error(self): + rc, stderr = self._generate_keywords(os.devnull, NONEXISTENT_FILE) + self.assertNotEqual(rc, 0) + self.assertRegex(stderr, b'(?ms)' + NONEXISTENT_FILE.encode()) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_keywordonlyarg.py b/Lib/test/test_keywordonlyarg.py index 0503a7fc6a..e4a44c1d8f 100644 --- a/Lib/test/test_keywordonlyarg.py +++ b/Lib/test/test_keywordonlyarg.py @@ -176,6 +176,18 @@ class KeywordOnlyArgTestCase(unittest.TestCase): return __a self.assertEqual(X().f(), 42) + def test_default_evaluation_order(self): + # See issue 16967 + a = 42 + with self.assertRaises(NameError) as err: + def f(v=a, x=b, *, y=c, z=d): + pass + self.assertEqual(str(err.exception), "name 'b' is not defined") + with self.assertRaises(NameError) as err: + f = lambda v=a, x=b, *, y=c, z=d: None + self.assertEqual(str(err.exception), "name 'b' is not defined") + + def test_main(): run_unittest(KeywordOnlyArgTestCase) diff --git a/Lib/test/test_kqueue.py b/Lib/test/test_kqueue.py index e5e6058334..bafdeba6ff 100644 --- a/Lib/test/test_kqueue.py +++ b/Lib/test/test_kqueue.py @@ -94,7 +94,7 @@ class TestKQueue(unittest.TestCase): client.setblocking(False) try: client.connect(('127.0.0.1', serverSocket.getsockname()[1])) - except socket.error as e: + except OSError as e: self.assertEqual(e.args[0], errno.EINPROGRESS) else: #raise AssertionError("Connect should have raised EINPROGRESS") @@ -185,6 +185,33 @@ class TestKQueue(unittest.TestCase): b.close() kq.close() + def test_close(self): + open_file = open(__file__, "rb") + self.addCleanup(open_file.close) + fd = open_file.fileno() + kqueue = select.kqueue() + + # test fileno() method and closed attribute + self.assertIsInstance(kqueue.fileno(), int) + self.assertFalse(kqueue.closed) + + # test close() + kqueue.close() + self.assertTrue(kqueue.closed) + self.assertRaises(ValueError, kqueue.fileno) + + # close() can be called more than once + kqueue.close() + + # operations must fail with ValueError("I/O operation on closed ...") + self.assertRaises(ValueError, kqueue.control, None, 4) + + def test_fd_non_inheritable(self): + kqueue = select.kqueue() + self.addCleanup(kqueue.close) + self.assertEqual(os.get_inheritable(kqueue.fileno()), False) + + def test_main(): support.run_unittest(TestKQueue) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 63ee697250..5b276e76ff 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -159,7 +159,7 @@ def setUpModule(): # Seeking is not enough of a test: you must write and flush, too! f.write(b'x') f.flush() - except (IOError, OverflowError): + except (OSError, OverflowError): raise unittest.SkipTest("filesystem does not have " "largefile support") finally: diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index ae4ca18444..ffd0c8bf2b 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -26,6 +26,7 @@ import logging.handlers import logging.config import codecs +import configparser import datetime import pickle import io @@ -58,7 +59,9 @@ try: import smtpd from urllib.parse import urlparse, parse_qs from socketserver import (ThreadingUDPServer, DatagramRequestHandler, - ThreadingTCPServer, StreamRequestHandler) + ThreadingTCPServer, StreamRequestHandler, + ThreadingUnixStreamServer, + ThreadingUnixDatagramServer) except ImportError: threading = None try: @@ -75,13 +78,12 @@ try: except ImportError: pass - class BaseTest(unittest.TestCase): """Base class for logging tests.""" log_format = "%(name)s -> %(levelname)s: %(message)s" - expected_log_pat = r"^([\w.]+) -> ([\w]+): ([\d]+)$" + expected_log_pat = r"^([\w.]+) -> (\w+): (\d+)$" message_num = 0 def setUp(self): @@ -93,7 +95,8 @@ class BaseTest(unittest.TestCase): self.saved_handlers = logging._handlers.copy() self.saved_handler_list = logging._handlerList[:] self.saved_loggers = saved_loggers = logger_dict.copy() - self.saved_level_names = logging._levelNames.copy() + self.saved_name_to_level = logging._nameToLevel.copy() + self.saved_level_to_name = logging._levelToName.copy() self.logger_states = logger_states = {} for name in saved_loggers: logger_states[name] = getattr(saved_loggers[name], @@ -135,8 +138,10 @@ class BaseTest(unittest.TestCase): self.root_logger.setLevel(self.original_logging_level) logging._acquireLock() try: - logging._levelNames.clear() - logging._levelNames.update(self.saved_level_names) + logging._levelToName.clear() + logging._levelToName.update(self.saved_level_to_name) + logging._nameToLevel.clear() + logging._nameToLevel.update(self.saved_name_to_level) logging._handlers.clear() logging._handlers.update(self.saved_handlers) logging._handlerList[:] = self.saved_handler_list @@ -150,12 +155,12 @@ class BaseTest(unittest.TestCase): finally: logging._releaseLock() - def assert_log_lines(self, expected_values, stream=None): + def assert_log_lines(self, expected_values, stream=None, pat=None): """Match the collected log lines against the regular expression self.expected_log_pat, and compare the extracted group values to the expected_values list of tuples.""" stream = stream or self.stream - pat = re.compile(self.expected_log_pat) + pat = re.compile(pat or self.expected_log_pat) actual_lines = stream.getvalue().splitlines() self.assertEqual(len(actual_lines), len(expected_values)) for actual, expected in zip(actual_lines, expected_values): @@ -430,7 +435,7 @@ class CustomLevelsAndFiltersTest(BaseTest): """Test various filtering possibilities with custom logging levels.""" # Skip the logger name group. - expected_log_pat = r"^[\w.]+ -> ([\w]+): ([\d]+)$" + expected_log_pat = r"^[\w.]+ -> (\w+): (\d+)$" def setUp(self): BaseTest.setUp(self) @@ -560,7 +565,7 @@ class HandlerTest(BaseTest): self.assertEqual(h.facility, h.LOG_USER) self.assertTrue(h.unixsocket) h.close() - except socket.error: # syslogd might not be available + except OSError: # syslogd might not be available pass for method in ('GET', 'POST', 'PUT'): if method == 'PUT': @@ -647,41 +652,6 @@ class StreamHandlerTest(BaseTest): # -- if it proves to be of wider utility than just test_logging if threading: - class TestSMTPChannel(smtpd.SMTPChannel): - """ - This derived class has had to be created because smtpd does not - support use of custom channel maps, although they are allowed by - asyncore's design. Issue #11959 has been raised to address this, - and if resolved satisfactorily, some of this code can be removed. - """ - def __init__(self, server, conn, addr, sockmap): - asynchat.async_chat.__init__(self, conn, sockmap) - self.smtp_server = server - self.conn = conn - self.addr = addr - self.data_size_limit = None - self.received_lines = [] - self.smtp_state = self.COMMAND - self.seen_greeting = '' - self.mailfrom = None - self.rcpttos = [] - self.received_data = '' - self.fqdn = socket.getfqdn() - self.num_bytes = 0 - try: - self.peer = conn.getpeername() - except socket.error as err: - # a race condition may occur if the other end is closing - # before we can get the peername - self.close() - if err.args[0] != errno.ENOTCONN: - raise - return - self.push('220 %s %s' % (self.fqdn, smtpd.__version__)) - self.set_terminator(b'\r\n') - self.extended_smtp = False - - class TestSMTPServer(smtpd.SMTPServer): """ This class implements a test SMTP server. @@ -702,37 +672,14 @@ if threading: :func:`asyncore.loop`. This avoids changing the :mod:`asyncore` module's global state. """ - channel_class = TestSMTPChannel def __init__(self, addr, handler, poll_interval, sockmap): - self._localaddr = addr - self._remoteaddr = None - self.data_size_limit = None - self.sockmap = sockmap - asyncore.dispatcher.__init__(self, map=sockmap) - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setblocking(0) - self.set_socket(sock, map=sockmap) - # try to re-use a server port if possible - self.set_reuse_addr() - self.bind(addr) - self.port = sock.getsockname()[1] - self.listen(5) - except: - self.close() - raise + smtpd.SMTPServer.__init__(self, addr, None, map=sockmap) + self.port = self.socket.getsockname()[1] self._handler = handler self._thread = None self.poll_interval = poll_interval - def handle_accepted(self, conn, addr): - """ - Redefined only because the base class does not pass in a - map, forcing use of a global in :mod:`asyncore`. - """ - channel = self.channel_class(self, conn, addr, self.sockmap) - def process_message(self, peer, mailfrom, rcpttos, data): """ Delegates to the handler passed in to the server's constructor. @@ -763,8 +710,8 @@ if threading: :func:`asyncore.loop`. """ try: - asyncore.loop(poll_interval, map=self.sockmap) - except select.error: + asyncore.loop(poll_interval, map=self._map) + except OSError: # On FreeBSD 8, closing the server repeatably # raises this error. We swallow it if the # server has been closed. @@ -871,7 +818,7 @@ if threading: sock, addr = self.socket.accept() if self.sslctx: sock = self.sslctx.wrap_socket(sock, server_side=True) - except socket.error as e: + except OSError as e: # socket errors are silenced by the caller, print them here sys.stderr.write("Got an error:\n%s\n" % e) raise @@ -908,6 +855,9 @@ if threading: super(TestTCPServer, self).server_bind() self.port = self.socket.getsockname()[1] + class TestUnixStreamServer(TestTCPServer): + address_family = socket.AF_UNIX + class TestUDPServer(ControlMixin, ThreadingUDPServer): """ A UDP server which is controllable using :class:`ControlMixin`. @@ -937,7 +887,7 @@ if threading: if data: try: super(DelegatingUDPRequestHandler, self).finish() - except socket.error: + except OSError: if not self.server._closed: raise @@ -955,6 +905,9 @@ if threading: super(TestUDPServer, self).server_close() self._closed = True + class TestUnixDatagramServer(TestUDPServer): + address_family = socket.AF_UNIX + # - end of server_helper section @unittest.skipUnless(threading, 'Threading required for this test.') @@ -991,7 +944,7 @@ class MemoryHandlerTest(BaseTest): """Tests for the MemoryHandler.""" # Do not bother with a logger name group. - expected_log_pat = r"^[\w.]+ -> ([\w]+): ([\d]+)$" + expected_log_pat = r"^[\w.]+ -> (\w+): (\d+)$" def setUp(self): BaseTest.setUp(self) @@ -1044,7 +997,7 @@ class ConfigFileTest(BaseTest): """Reading logging config from a .ini-style config file.""" - expected_log_pat = r"^([\w]+) \+\+ ([\w]+)$" + expected_log_pat = r"^(\w+) \+\+ (\w+)$" # config0 is a standard configuration. config0 = """ @@ -1292,6 +1245,24 @@ class ConfigFileTest(BaseTest): # Original logger output is empty. self.assert_log_lines([]) + def test_config0_using_cp_ok(self): + # A simple config file which overrides the default settings. + with captured_stdout() as output: + file = io.StringIO(textwrap.dedent(self.config0)) + cp = configparser.ConfigParser() + cp.read_file(file) + logging.config.fileConfig(cp) + logger = logging.getLogger() + # Won't output anything + logger.info(self.next_message()) + # Outputs a message + logger.error(self.next_message()) + self.assert_log_lines([ + ('ERROR', '2'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + def test_config1_ok(self, config=config1): # A config file defining a sub-parser as well. with captured_stdout() as output: @@ -1381,7 +1352,7 @@ class ConfigFileTest(BaseTest): def test_logger_disabling(self): self.apply_config(self.disable_test) - logger = logging.getLogger('foo') + logger = logging.getLogger('some_pristine_logger') self.assertFalse(logger.disabled) self.apply_config(self.disable_test) self.assertTrue(logger.disabled) @@ -1394,17 +1365,23 @@ class SocketHandlerTest(BaseTest): """Test for SocketHandler objects.""" + if threading: + server_class = TestTCPServer + address = ('localhost', 0) + def setUp(self): """Set up a TCP server to receive log messages, and a SocketHandler pointing to that server's address and port.""" BaseTest.setUp(self) - addr = ('localhost', 0) - self.server = server = TestTCPServer(addr, self.handle_socket, - 0.01) + self.server = server = self.server_class(self.address, + self.handle_socket, 0.01) server.start() server.ready.wait() - self.sock_hdlr = logging.handlers.SocketHandler('localhost', - server.port) + hcls = logging.handlers.SocketHandler + if isinstance(server.server_address, tuple): + self.sock_hdlr = hcls('localhost', server.port) + else: + self.sock_hdlr = hcls(server.server_address, None) self.log_output = '' self.root_logger.removeHandler(self.root_logger.handlers[0]) self.root_logger.addHandler(self.sock_hdlr) @@ -1444,35 +1421,69 @@ class SocketHandlerTest(BaseTest): self.assertEqual(self.log_output, "spam\neggs\n") def test_noserver(self): + # Avoid timing-related failures due to SocketHandler's own hard-wired + # one-second timeout on socket.create_connection() (issue #16264). + self.sock_hdlr.retryStart = 2.5 # Kill the server self.server.stop(2.0) - #The logging call should try to connect, which should fail + # The logging call should try to connect, which should fail try: raise RuntimeError('Deliberate mistake') except RuntimeError: self.root_logger.exception('Never sent') self.root_logger.error('Never sent, either') now = time.time() - self.assertTrue(self.sock_hdlr.retryTime > now) + self.assertGreater(self.sock_hdlr.retryTime, now) time.sleep(self.sock_hdlr.retryTime - now + 0.001) self.root_logger.error('Nor this') +def _get_temp_domain_socket(): + fd, fn = tempfile.mkstemp(prefix='test_logging_', suffix='.sock') + os.close(fd) + # just need a name - file can't be present, or we'll get an + # 'address already in use' error. + os.remove(fn) + return fn + +@unittest.skipUnless(threading, 'Threading required for this test.') +class UnixSocketHandlerTest(SocketHandlerTest): + + """Test for SocketHandler with unix sockets.""" + + if threading: + server_class = TestUnixStreamServer + + def setUp(self): + # override the definition in the base class + self.address = _get_temp_domain_socket() + SocketHandlerTest.setUp(self) + + def tearDown(self): + SocketHandlerTest.tearDown(self) + os.remove(self.address) @unittest.skipUnless(threading, 'Threading required for this test.') class DatagramHandlerTest(BaseTest): """Test for DatagramHandler.""" + if threading: + server_class = TestUDPServer + address = ('localhost', 0) + def setUp(self): """Set up a UDP server to receive log messages, and a DatagramHandler pointing to that server's address and port.""" BaseTest.setUp(self) - addr = ('localhost', 0) - self.server = server = TestUDPServer(addr, self.handle_datagram, 0.01) + self.server = server = self.server_class(self.address, + self.handle_datagram, 0.01) server.start() server.ready.wait() - self.sock_hdlr = logging.handlers.DatagramHandler('localhost', - server.port) + hcls = logging.handlers.DatagramHandler + if isinstance(server.server_address, tuple): + self.sock_hdlr = hcls('localhost', server.port) + else: + self.sock_hdlr = hcls(server.server_address, None) self.log_output = '' self.root_logger.removeHandler(self.root_logger.handlers[0]) self.root_logger.addHandler(self.sock_hdlr) @@ -1507,21 +1518,44 @@ class DatagramHandlerTest(BaseTest): @unittest.skipUnless(threading, 'Threading required for this test.') +class UnixDatagramHandlerTest(DatagramHandlerTest): + + """Test for DatagramHandler using Unix sockets.""" + + if threading: + server_class = TestUnixDatagramServer + + def setUp(self): + # override the definition in the base class + self.address = _get_temp_domain_socket() + DatagramHandlerTest.setUp(self) + + def tearDown(self): + DatagramHandlerTest.tearDown(self) + os.remove(self.address) + +@unittest.skipUnless(threading, 'Threading required for this test.') class SysLogHandlerTest(BaseTest): """Test for SysLogHandler using UDP.""" + if threading: + server_class = TestUDPServer + address = ('localhost', 0) + def setUp(self): """Set up a UDP server to receive log messages, and a SysLogHandler pointing to that server's address and port.""" BaseTest.setUp(self) - addr = ('localhost', 0) - self.server = server = TestUDPServer(addr, self.handle_datagram, - 0.01) + self.server = server = self.server_class(self.address, + self.handle_datagram, 0.01) server.start() server.ready.wait() - self.sl_hdlr = logging.handlers.SysLogHandler(('localhost', - server.port)) + hcls = logging.handlers.SysLogHandler + if isinstance(server.server_address, tuple): + self.sl_hdlr = hcls(('localhost', server.port)) + else: + self.sl_hdlr = hcls(server.server_address) self.log_output = '' self.root_logger.removeHandler(self.root_logger.handlers[0]) self.root_logger.addHandler(self.sl_hdlr) @@ -1559,6 +1593,23 @@ class SysLogHandlerTest(BaseTest): @unittest.skipUnless(threading, 'Threading required for this test.') +class UnixSysLogHandlerTest(SysLogHandlerTest): + + """Test for SysLogHandler with Unix sockets.""" + + if threading: + server_class = TestUnixDatagramServer + + def setUp(self): + # override the definition in the base class + self.address = _get_temp_domain_socket() + SysLogHandlerTest.setUp(self) + + def tearDown(self): + SysLogHandlerTest.tearDown(self) + os.remove(self.address) + +@unittest.skipUnless(threading, 'Threading required for this test.') class HTTPHandlerTest(BaseTest): """Test for HTTPHandler.""" @@ -1779,7 +1830,7 @@ class WarningsTest(BaseTest): logger.removeHandler(h) s = stream.getvalue() h.close() - self.assertTrue(s.find("UserWarning: I'm warning you...\n") > 0) + self.assertGreater(s.find("UserWarning: I'm warning you...\n"), 0) #See if an explicit file uses the original implementation a_file = io.StringIO() @@ -1817,7 +1868,7 @@ class ConfigDictTest(BaseTest): """Reading logging config from a dictionary.""" - expected_log_pat = r"^([\w]+) \+\+ ([\w]+)$" + expected_log_pat = r"^(\w+) \+\+ (\w+)$" # config0 is a standard configuration. config0 = { @@ -2389,6 +2440,32 @@ class ConfigDictTest(BaseTest): }, } + # As config0, but with properties + config14 = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + '.': { + 'foo': 'bar', + 'terminator': '!\n', + } + }, + }, + 'root' : { + 'level' : 'WARNING', + 'handlers' : ['hand1'], + }, + } + out_of_order = { "version": 1, "formatters": { @@ -2656,11 +2733,20 @@ class ConfigDictTest(BaseTest): def test_config13_failure(self): self.assertRaises(Exception, self.apply_config, self.config13) + def test_config14_ok(self): + with captured_stdout() as output: + self.apply_config(self.config14) + h = logging._handlers['hand1'] + self.assertEqual(h.foo, 'bar') + self.assertEqual(h.terminator, '!\n') + logging.warning('Exclamation') + self.assertTrue(output.getvalue().endswith('Exclamation!\n')) + @unittest.skipUnless(threading, 'listen() needs threading to work') - def setup_via_listener(self, text): + def setup_via_listener(self, text, verify=None): text = text.encode("utf-8") # Ask for a randomly assigned port (by using port 0) - t = logging.config.listen(0) + t = logging.config.listen(0, verify) t.start() t.ready.wait() # Now get the port allocated @@ -2720,6 +2806,69 @@ class ConfigDictTest(BaseTest): # Original logger output is empty. self.assert_log_lines([]) + @unittest.skipUnless(threading, 'Threading required for this test.') + def test_listen_verify(self): + + def verify_fail(stuff): + return None + + def verify_reverse(stuff): + return stuff[::-1] + + logger = logging.getLogger("compiler.parser") + to_send = textwrap.dedent(ConfigFileTest.config1) + # First, specify a verification function that will fail. + # We expect to see no output, since our configuration + # never took effect. + with captured_stdout() as output: + self.setup_via_listener(to_send, verify_fail) + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + self.assert_log_lines([], stream=output) + # Original logger output has the stuff we logged. + self.assert_log_lines([ + ('INFO', '1'), + ('ERROR', '2'), + ], pat=r"^[\w.]+ -> (\w+): (\d+)$") + + # Now, perform no verification. Our configuration + # should take effect. + + with captured_stdout() as output: + self.setup_via_listener(to_send) # no verify callable specified + logger = logging.getLogger("compiler.parser") + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + self.assert_log_lines([ + ('INFO', '3'), + ('ERROR', '4'), + ], stream=output) + # Original logger output still has the stuff we logged before. + self.assert_log_lines([ + ('INFO', '1'), + ('ERROR', '2'), + ], pat=r"^[\w.]+ -> (\w+): (\d+)$") + + # Now, perform verification which transforms the bytes. + + with captured_stdout() as output: + self.setup_via_listener(to_send[::-1], verify_reverse) + logger = logging.getLogger("compiler.parser") + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + self.assert_log_lines([ + ('INFO', '5'), + ('ERROR', '6'), + ], stream=output) + # Original logger output still has the stuff we logged before. + self.assert_log_lines([ + ('INFO', '1'), + ('ERROR', '2'), + ], pat=r"^[\w.]+ -> (\w+): (\d+)$") + def test_out_of_order(self): self.apply_config(self.out_of_order) handler = logging.getLogger('mymodule').handlers[0] @@ -2779,14 +2928,14 @@ class ChildLoggerTest(BaseTest): l2 = logging.getLogger('def.ghi') c1 = r.getChild('xyz') c2 = r.getChild('uvw.xyz') - self.assertTrue(c1 is logging.getLogger('xyz')) - self.assertTrue(c2 is logging.getLogger('uvw.xyz')) + self.assertIs(c1, logging.getLogger('xyz')) + self.assertIs(c2, logging.getLogger('uvw.xyz')) c1 = l1.getChild('def') c2 = c1.getChild('ghi') c3 = l1.getChild('def.ghi') - self.assertTrue(c1 is logging.getLogger('abc.def')) - self.assertTrue(c2 is logging.getLogger('abc.def.ghi')) - self.assertTrue(c2 is c3) + self.assertIs(c1, logging.getLogger('abc.def')) + self.assertIs(c2, logging.getLogger('abc.def.ghi')) + self.assertIs(c2, c3) class DerivedLogRecord(logging.LogRecord): @@ -2829,7 +2978,7 @@ class LogRecordFactoryTest(BaseTest): class QueueHandlerTest(BaseTest): # Do not bother with a logger name group. - expected_log_pat = r"^[\w.]+ -> ([\w]+): ([\d]+)$" + expected_log_pat = r"^[\w.]+ -> (\w+): (\d+)$" def setUp(self): BaseTest.setUp(self) @@ -3123,13 +3272,13 @@ class ShutdownTest(BaseTest): self.assertEqual('0 - release', self.called[-1]) def test_with_ioerror_in_acquire(self): - self._test_with_failure_in_method('acquire', IOError) + self._test_with_failure_in_method('acquire', OSError) def test_with_ioerror_in_flush(self): - self._test_with_failure_in_method('flush', IOError) + self._test_with_failure_in_method('flush', OSError) def test_with_ioerror_in_close(self): - self._test_with_failure_in_method('close', IOError) + self._test_with_failure_in_method('close', OSError) def test_with_valueerror_in_acquire(self): self._test_with_failure_in_method('acquire', ValueError) @@ -3336,6 +3485,12 @@ class BasicConfigTest(unittest.TestCase): self.assertEqual(logging.root.level, self.original_logging_level) def test_filename(self): + + def cleanup(h1, h2, fn): + h1.close() + h2.close() + os.remove(fn) + logging.basicConfig(filename='test.log') self.assertEqual(len(logging.root.handlers), 1) @@ -3343,17 +3498,23 @@ class BasicConfigTest(unittest.TestCase): self.assertIsInstance(handler, logging.FileHandler) expected = logging.FileHandler('test.log', 'a') - self.addCleanup(expected.close) self.assertEqual(handler.stream.mode, expected.stream.mode) self.assertEqual(handler.stream.name, expected.stream.name) + self.addCleanup(cleanup, handler, expected, 'test.log') def test_filemode(self): + + def cleanup(h1, h2, fn): + h1.close() + h2.close() + os.remove(fn) + logging.basicConfig(filename='test.log', filemode='wb') handler = logging.root.handlers[0] expected = logging.FileHandler('test.log', 'wb') - self.addCleanup(expected.close) self.assertEqual(handler.stream.mode, expected.stream.mode) + self.addCleanup(cleanup, handler, expected, 'test.log') def test_stream(self): stream = io.StringIO() @@ -3809,6 +3970,63 @@ class TimedRotatingFileHandlerTest(BaseFileTest): assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, self.fn, 'W7', delay=True) + def test_compute_rollover_daily_attime(self): + currentTime = 0 + atTime = datetime.time(12, 0, 0) + rh = logging.handlers.TimedRotatingFileHandler( + self.fn, when='MIDNIGHT', interval=1, backupCount=0, utc=True, + atTime=atTime) + try: + actual = rh.computeRollover(currentTime) + self.assertEqual(actual, currentTime + 12 * 60 * 60) + + actual = rh.computeRollover(currentTime + 13 * 60 * 60) + self.assertEqual(actual, currentTime + 36 * 60 * 60) + finally: + rh.close() + + #@unittest.skipIf(True, 'Temporarily skipped while failures investigated.') + def test_compute_rollover_weekly_attime(self): + currentTime = int(time.time()) + today = currentTime - currentTime % 86400 + + atTime = datetime.time(12, 0, 0) + + wday = time.gmtime(today).tm_wday + for day in range(7): + rh = logging.handlers.TimedRotatingFileHandler( + self.fn, when='W%d' % day, interval=1, backupCount=0, utc=True, + atTime=atTime) + try: + if wday > day: + # The rollover day has already passed this week, so we + # go over into next week + expected = (7 - wday + day) + else: + expected = (day - wday) + # At this point expected is in days from now, convert to seconds + expected *= 24 * 60 * 60 + # Add in the rollover time + expected += 12 * 60 * 60 + # Add in adjustment for today + expected += today + actual = rh.computeRollover(today) + if actual != expected: + print('failed in timezone: %d' % time.timezone) + print('local vars: %s' % locals()) + self.assertEqual(actual, expected) + if day == wday: + # goes into following week + expected += 7 * 24 * 60 * 60 + actual = rh.computeRollover(today + 13 * 60 * 60) + if actual != expected: + print('failed in timezone: %d' % time.timezone) + print('local vars: %s' % locals()) + self.assertEqual(actual, expected) + finally: + rh.close() + + def secs(**kw): return datetime.timedelta(**kw) // datetime.timedelta(seconds=1) @@ -3867,7 +4085,7 @@ class NTEventLogHandlerTest(BaseTest): h.handle(r) h.close() # Now see if the event is recorded - self.assertTrue(num_recs < win32evtlog.GetNumberOfEventLogRecords(elh)) + self.assertLess(num_recs, win32evtlog.GetNumberOfEventLogRecords(elh)) flags = win32evtlog.EVENTLOG_BACKWARDS_READ | \ win32evtlog.EVENTLOG_SEQUENTIAL_READ found = False @@ -3900,7 +4118,8 @@ def test_main(): SMTPHandlerTest, FileHandlerTest, RotatingFileHandlerTest, LastResortTest, LogRecordTest, ExceptionTest, SysLogHandlerTest, HTTPHandlerTest, NTEventLogHandlerTest, - TimedRotatingFileHandlerTest + TimedRotatingFileHandlerTest, UnixSocketHandlerTest, + UnixDatagramHandlerTest, UnixSysLogHandlerTest ) if __name__ == "__main__": diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index baf1d6a3b2..6c30fed7c9 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -1079,7 +1079,7 @@ class LongTest(unittest.TestCase): self.assertRaises(OverflowError, (256).to_bytes, 1, 'big', signed=True) self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=False) self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=True) - self.assertRaises(OverflowError, (-1).to_bytes, 2, 'big', signed=False), + self.assertRaises(OverflowError, (-1).to_bytes, 2, 'big', signed=False) self.assertRaises(OverflowError, (-1).to_bytes, 2, 'little', signed=False) self.assertEqual((0).to_bytes(0, 'big'), b'') self.assertEqual((1).to_bytes(5, 'big'), b'\x00\x00\x00\x00\x01') diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index a13cf3bd09..20d85828e6 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -669,6 +669,20 @@ class FileTestCase(unittest.TestCase): with LZMAFile(BytesIO(COMPRESSED_XZ[:128])) as f: self.assertRaises(EOFError, f.read) + def test_read_truncated(self): + # Drop stream footer: CRC (4 bytes), index size (4 bytes), + # flags (2 bytes) and magic number (2 bytes). + truncated = COMPRESSED_XZ[:-12] + with LZMAFile(BytesIO(truncated)) as f: + self.assertRaises(EOFError, f.read) + with LZMAFile(BytesIO(truncated)) as f: + self.assertEqual(f.read(len(INPUT)), INPUT) + self.assertRaises(EOFError, f.read, 1) + # Incomplete 12-byte header. + for i in range(12): + with LZMAFile(BytesIO(truncated[:i])) as f: + self.assertRaises(EOFError, f.read, 1) + def test_read_bad_args(self): f = LZMAFile(BytesIO(COMPRESSED_XZ)) f.close() diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 39e8643414..6c679ce436 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -596,7 +596,7 @@ class TestMaildir(TestMailbox, unittest.TestCase): def setUp(self): TestMailbox.setUp(self) - if os.name in ('nt', 'os2') or sys.platform == 'cygwin': + if (os.name == 'nt') or (sys.platform == 'cygwin'): self._box.colon = '!' def assertMailboxEmpty(self): diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 7e37f39c6a..255eb87bb3 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -24,37 +24,13 @@ class HelperMixin: class IntTestCase(unittest.TestCase, HelperMixin): def test_ints(self): - # Test the full range of Python ints. - n = sys.maxsize + # Test a range of Python ints larger than the machine word size. + n = sys.maxsize ** 2 while n: for expected in (-n, n): self.helper(expected) n = n >> 1 - def test_int64(self): - # Simulate int marshaling on a 64-bit box. This is most interesting if - # we're running the test on a 32-bit box, of course. - - def to_little_endian_string(value, nbytes): - b = bytearray() - for i in range(nbytes): - b.append(value & 0xff) - value >>= 8 - return b - - maxint64 = (1 << 63) - 1 - minint64 = -maxint64-1 - - for base in maxint64, minint64, -maxint64, -(minint64 >> 1): - while base: - s = b'I' + to_little_endian_string(base, 8) - got = marshal.loads(s) - self.assertEqual(base, got) - if base == -1: # a fixed-point for shifting right 1 - base = 0 - else: - base >>= 1 - def test_bool(self): for b in (True, False): self.helper(b) @@ -201,10 +177,14 @@ class BugsTestCase(unittest.TestCase): except Exception: pass - def test_loads_recursion(self): + def test_loads_2x_code(self): s = b'c' + (b'X' * 4*4) + b'{' * 2**20 self.assertRaises(ValueError, marshal.loads, s) + def test_loads_recursion(self): + s = b'c' + (b'X' * 4*5) + b'{' * 2**20 + self.assertRaises(ValueError, marshal.loads, s) + def test_recursion_limit(self): # Create a deeply nested structure. head = last = [] @@ -282,15 +262,20 @@ class BugsTestCase(unittest.TestCase): def test_bad_reader(self): class BadReader(io.BytesIO): - def read(self, n=-1): - b = super().read(n) + def readinto(self, buf): + n = super().readinto(buf) if n is not None and n > 4: - b += b' ' * 10**6 - return b + n += 10**6 + return n for value in (1.0, 1j, b'0123456789', '0123456789'): self.assertRaises(ValueError, marshal.load, BadReader(marshal.dumps(value))) + def _test_eof(self): + data = marshal.dumps(("hello", "dolly", None)) + for i in range(len(data)): + self.assertRaises(EOFError, marshal.loads, data[0: i]) + LARGE_SIZE = 2**31 pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4 @@ -335,6 +320,122 @@ class LargeValuesTestCase(unittest.TestCase): def test_bytearray(self, size): self.check_unmarshallable(bytearray(size)) +def CollectObjectIDs(ids, obj): + """Collect object ids seen in a structure""" + if id(obj) in ids: + return + ids.add(id(obj)) + if isinstance(obj, (list, tuple, set, frozenset)): + for e in obj: + CollectObjectIDs(ids, e) + elif isinstance(obj, dict): + for k, v in obj.items(): + CollectObjectIDs(ids, k) + CollectObjectIDs(ids, v) + return len(ids) + +class InstancingTestCase(unittest.TestCase, HelperMixin): + intobj = 123321 + floatobj = 1.2345 + strobj = "abcde"*3 + dictobj = {"hello":floatobj, "goodbye":floatobj, floatobj:"hello"} + + def helper3(self, rsample, recursive=False, simple=False): + #we have two instances + sample = (rsample, rsample) + + n0 = CollectObjectIDs(set(), sample) + + s3 = marshal.dumps(sample, 3) + n3 = CollectObjectIDs(set(), marshal.loads(s3)) + + #same number of instances generated + self.assertEqual(n3, n0) + + if not recursive: + #can compare with version 2 + s2 = marshal.dumps(sample, 2) + n2 = CollectObjectIDs(set(), marshal.loads(s2)) + #old format generated more instances + self.assertGreater(n2, n0) + + #if complex objects are in there, old format is larger + if not simple: + self.assertGreater(len(s2), len(s3)) + else: + self.assertGreaterEqual(len(s2), len(s3)) + + def testInt(self): + self.helper(self.intobj) + self.helper3(self.intobj, simple=True) + + def testFloat(self): + self.helper(self.floatobj) + self.helper3(self.floatobj) + + def testStr(self): + self.helper(self.strobj) + self.helper3(self.strobj) + + def testDict(self): + self.helper(self.dictobj) + self.helper3(self.dictobj) + + def testModule(self): + with open(__file__, "rb") as f: + code = f.read() + if __file__.endswith(".py"): + code = compile(code, __file__, "exec") + self.helper(code) + self.helper3(code) + + def testRecursion(self): + d = dict(self.dictobj) + d["self"] = d + self.helper3(d, recursive=True) + l = [self.dictobj] + l.append(l) + self.helper3(l, recursive=True) + +class CompatibilityTestCase(unittest.TestCase): + def _test(self, version): + with open(__file__, "rb") as f: + code = f.read() + if __file__.endswith(".py"): + code = compile(code, __file__, "exec") + data = marshal.dumps(code, version) + marshal.loads(data) + + def test0To3(self): + self._test(0) + + def test1To3(self): + self._test(1) + + def test2To3(self): + self._test(2) + + def test3To3(self): + self._test(3) + +class InterningTestCase(unittest.TestCase, HelperMixin): + strobj = "this is an interned string" + strobj = sys.intern(strobj) + + def testIntern(self): + s = marshal.loads(marshal.dumps(self.strobj)) + self.assertEqual(s, self.strobj) + self.assertEqual(id(s), id(self.strobj)) + s2 = sys.intern(s) + self.assertEqual(id(s2), id(s)) + + def testNoIntern(self): + s = marshal.loads(marshal.dumps(self.strobj, 2)) + self.assertEqual(s, self.strobj) + self.assertNotEqual(id(s), id(self.strobj)) + s2 = sys.intern(s) + self.assertNotEqual(id(s2), id(s)) + def test_main(): support.run_unittest(IntTestCase, diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index d611a3138b..4de4f65329 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -520,12 +520,12 @@ class TextIOTestMixin: def test_relative_seek(self): memio = self.ioclass() - self.assertRaises(IOError, memio.seek, -1, 1) - self.assertRaises(IOError, memio.seek, 3, 1) - self.assertRaises(IOError, memio.seek, -3, 1) - self.assertRaises(IOError, memio.seek, -1, 2) - self.assertRaises(IOError, memio.seek, 1, 1) - self.assertRaises(IOError, memio.seek, 1, 2) + self.assertRaises(OSError, memio.seek, -1, 1) + self.assertRaises(OSError, memio.seek, 3, 1) + self.assertRaises(OSError, memio.seek, -3, 1) + self.assertRaises(OSError, memio.seek, -1, 2) + self.assertRaises(OSError, memio.seek, 1, 1) + self.assertRaises(OSError, memio.seek, 1, 2) def test_textio_properties(self): memio = self.ioclass() diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index ee6b15ac14..ffd4f5851a 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -352,6 +352,15 @@ class AbstractMemoryTests: self.assertIs(wr(), None) self.assertIs(L[0], b) + def test_reversed(self): + for tp in self._types: + b = tp(self._source) + m = self._view(b) + aslist = list(reversed(m.tolist())) + self.assertEqual(list(reversed(m)), aslist) + self.assertEqual(list(reversed(m)), list(m[::-1])) + + # Variations on source objects for the buffer: bytes-like objects, then arrays # with itemsize > 1. # NOTE: support for multi-dimensional objects is unimplemented. diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index d066368726..b1cc97371e 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1,11 +1,12 @@ from test.support import (TESTFN, run_unittest, import_module, unlink, - requires, _2G, _4G) + requires, _2G, _4G, gc_collect) import unittest import os import re import itertools import socket import sys +import weakref # Skip test if we can't import mmap. mmap = import_module('mmap') @@ -245,7 +246,7 @@ class MmapTests(unittest.TestCase): def test_bad_file_desc(self): # Try opening a bad file descriptor... - self.assertRaises(mmap.error, mmap.mmap, -2, 4096) + self.assertRaises(OSError, mmap.mmap, -2, 4096) def test_tougher_find(self): # Do a tougher .find() test. SF bug 515943 pointed out that, in 2.2, @@ -658,7 +659,7 @@ class MmapTests(unittest.TestCase): m = mmap.mmap(f.fileno(), 0) f.close() try: - m.resize(0) # will raise WindowsError + m.resize(0) # will raise OSError except: pass try: @@ -673,7 +674,7 @@ class MmapTests(unittest.TestCase): # parameters to _get_osfhandle. s = socket.socket() try: - with self.assertRaises(mmap.error): + with self.assertRaises(OSError): m = mmap.mmap(s.fileno(), 10) finally: s.close() @@ -684,14 +685,23 @@ class MmapTests(unittest.TestCase): self.assertTrue(m.closed) def test_context_manager_exception(self): - # Test that the IOError gets passed through + # Test that the OSError gets passed through with self.assertRaises(Exception) as exc: with mmap.mmap(-1, 10) as m: - raise IOError - self.assertIsInstance(exc.exception, IOError, + raise OSError + self.assertIsInstance(exc.exception, OSError, "wrong exception raised in context manager") self.assertTrue(m.closed, "context manager failed") + def test_weakref(self): + # Check mmap objects are weakrefable + mm = mmap.mmap(-1, 16) + wr = weakref.ref(mm) + self.assertIs(wr(), mm) + del mm + gc_collect() + self.assertIs(wr(), None) + class LargeMmapTests(unittest.TestCase): def setUp(self): @@ -709,7 +719,7 @@ class LargeMmapTests(unittest.TestCase): f.seek(num_zeroes) f.write(tail) f.flush() - except (IOError, OverflowError): + except (OSError, OverflowError): f.close() raise unittest.SkipTest("filesystem does not have largefile support") return f diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py index e5a2525d18..5a9b503485 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module.py @@ -1,6 +1,8 @@ # Test the module type import unittest +import weakref from test.support import run_unittest, gc_collect +from test.script_helper import assert_python_ok import sys ModuleType = type(sys) @@ -33,7 +35,10 @@ class ModuleTests(unittest.TestCase): foo = ModuleType("foo") self.assertEqual(foo.__name__, "foo") self.assertEqual(foo.__doc__, None) - self.assertEqual(foo.__dict__, {"__name__": "foo", "__doc__": None}) + self.assertIs(foo.__loader__, None) + self.assertIs(foo.__package__, None) + self.assertEqual(foo.__dict__, {"__name__": "foo", "__doc__": None, + "__loader__": None, "__package__": None}) def test_ascii_docstring(self): # ASCII docstring @@ -41,7 +46,8 @@ class ModuleTests(unittest.TestCase): self.assertEqual(foo.__name__, "foo") self.assertEqual(foo.__doc__, "foodoc") self.assertEqual(foo.__dict__, - {"__name__": "foo", "__doc__": "foodoc"}) + {"__name__": "foo", "__doc__": "foodoc", + "__loader__": None, "__package__": None}) def test_unicode_docstring(self): # Unicode docstring @@ -49,7 +55,8 @@ class ModuleTests(unittest.TestCase): self.assertEqual(foo.__name__, "foo") self.assertEqual(foo.__doc__, "foodoc\u1234") self.assertEqual(foo.__dict__, - {"__name__": "foo", "__doc__": "foodoc\u1234"}) + {"__name__": "foo", "__doc__": "foodoc\u1234", + "__loader__": None, "__package__": None}) def test_reinit(self): # Reinitialization should not replace the __dict__ @@ -61,10 +68,10 @@ class ModuleTests(unittest.TestCase): self.assertEqual(foo.__doc__, "foodoc") self.assertEqual(foo.bar, 42) self.assertEqual(foo.__dict__, - {"__name__": "foo", "__doc__": "foodoc", "bar": 42}) + {"__name__": "foo", "__doc__": "foodoc", "bar": 42, + "__loader__": None, "__package__": None}) self.assertTrue(foo.__dict__ is d) - @unittest.expectedFailure def test_dont_clear_dict(self): # See issue 7140. def f(): @@ -89,6 +96,14 @@ a = A(destroyed)""" gc_collect() self.assertEqual(destroyed, [1]) + def test_weakref(self): + m = ModuleType("foo") + wr = weakref.ref(m) + self.assertIs(wr(), m) + del m + gc_collect() + self.assertIs(wr(), None) + def test_module_repr_minimal(self): # reprs when modules have no __file__, __name__, or __loader__ m = ModuleType('foo') @@ -110,13 +125,19 @@ a = A(destroyed)""" m.__file__ = '/tmp/foo.py' self.assertEqual(repr(m), "<module '?' from '/tmp/foo.py'>") + def test_module_repr_with_loader_as_None(self): + m = ModuleType('foo') + assert m.__loader__ is None + self.assertEqual(repr(m), "<module 'foo'>") + def test_module_repr_with_bare_loader_but_no_name(self): m = ModuleType('foo') del m.__name__ # Yes, a class not an instance. m.__loader__ = BareLoader + loader_repr = repr(BareLoader) self.assertEqual( - repr(m), "<module '?' (<class 'test.test_module.BareLoader'>)>") + repr(m), "<module '?' ({})>".format(loader_repr)) def test_module_repr_with_full_loader_but_no_name(self): # m.__loader__.module_repr() will fail because the module has no @@ -126,15 +147,17 @@ a = A(destroyed)""" del m.__name__ # Yes, a class not an instance. m.__loader__ = FullLoader + loader_repr = repr(FullLoader) self.assertEqual( - repr(m), "<module '?' (<class 'test.test_module.FullLoader'>)>") + repr(m), "<module '?' ({})>".format(loader_repr)) def test_module_repr_with_bare_loader(self): m = ModuleType('foo') # Yes, a class not an instance. m.__loader__ = BareLoader + module_repr = repr(BareLoader) self.assertEqual( - repr(m), "<module 'foo' (<class 'test.test_module.BareLoader'>)>") + repr(m), "<module 'foo' ({})>".format(module_repr)) def test_module_repr_with_full_loader(self): m = ModuleType('foo') @@ -167,6 +190,19 @@ a = A(destroyed)""" self.assertEqual(r[:25], "<module 'unittest' from '") self.assertEqual(r[-13:], "__init__.py'>") + def test_module_finalization_at_shutdown(self): + # Module globals and builtins should still be available during shutdown + rc, out, err = assert_python_ok("-c", "from test import final_a") + self.assertFalse(err) + lines = out.splitlines() + self.assertEqual(set(lines), { + b"x = a", + b"x = b", + b"final_a.x = a", + b"final_b.x = b", + b"len = len", + b"shutil.rmtree = rmtree"}) + # frozen and namespace module reprs are tested in importlib. diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index feb7bd595a..91148a6fc5 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -176,57 +176,28 @@ class Test_StreamReader(unittest.TestCase): support.unlink(TESTFN) class Test_StreamWriter(unittest.TestCase): - if len('\U00012345') == 2: # UCS2 - def test_gb18030(self): - s= io.BytesIO() - c = codecs.getwriter('gb18030')(s) - c.write('123') - self.assertEqual(s.getvalue(), b'123') - c.write('\U00012345') - self.assertEqual(s.getvalue(), b'123\x907\x959') - c.write('\U00012345'[0]) - self.assertEqual(s.getvalue(), b'123\x907\x959') - c.write('\U00012345'[1] + '\U00012345' + '\uac00\u00ac') - self.assertEqual(s.getvalue(), - b'123\x907\x959\x907\x959\x907\x959\x827\xcf5\x810\x851') - c.write('\U00012345'[0]) - self.assertEqual(s.getvalue(), - b'123\x907\x959\x907\x959\x907\x959\x827\xcf5\x810\x851') - self.assertRaises(UnicodeError, c.reset) - self.assertEqual(s.getvalue(), - b'123\x907\x959\x907\x959\x907\x959\x827\xcf5\x810\x851') - - def test_utf_8(self): - s= io.BytesIO() - c = codecs.getwriter('utf-8')(s) - c.write('123') - self.assertEqual(s.getvalue(), b'123') - c.write('\U00012345') - self.assertEqual(s.getvalue(), b'123\xf0\x92\x8d\x85') - - # Python utf-8 codec can't buffer surrogate pairs yet. - if 0: - c.write('\U00012345'[0]) - self.assertEqual(s.getvalue(), b'123\xf0\x92\x8d\x85') - c.write('\U00012345'[1] + '\U00012345' + '\uac00\u00ac') - self.assertEqual(s.getvalue(), - b'123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85' - b'\xea\xb0\x80\xc2\xac') - c.write('\U00012345'[0]) - self.assertEqual(s.getvalue(), - b'123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85' - b'\xea\xb0\x80\xc2\xac') - c.reset() - self.assertEqual(s.getvalue(), - b'123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85' - b'\xea\xb0\x80\xc2\xac\xed\xa0\x88') - c.write('\U00012345'[1]) - self.assertEqual(s.getvalue(), - b'123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85' - b'\xea\xb0\x80\xc2\xac\xed\xa0\x88\xed\xbd\x85') - - else: # UCS4 - pass + def test_gb18030(self): + s= io.BytesIO() + c = codecs.getwriter('gb18030')(s) + c.write('123') + self.assertEqual(s.getvalue(), b'123') + c.write('\U00012345') + self.assertEqual(s.getvalue(), b'123\x907\x959') + c.write('\uac00\u00ac') + self.assertEqual(s.getvalue(), + b'123\x907\x959\x827\xcf5\x810\x851') + + def test_utf_8(self): + s= io.BytesIO() + c = codecs.getwriter('utf-8')(s) + c.write('123') + self.assertEqual(s.getvalue(), b'123') + c.write('\U00012345') + self.assertEqual(s.getvalue(), b'123\xf0\x92\x8d\x85') + c.write('\uac00\u00ac') + self.assertEqual(s.getvalue(), + b'123\xf0\x92\x8d\x85' + b'\xea\xb0\x80\xc2\xac') def test_streamwriter_strwrite(self): s = io.BytesIO() diff --git a/Lib/test/test_multiprocessing_fork.py b/Lib/test/test_multiprocessing_fork.py new file mode 100644 index 0000000000..2bf4e75644 --- /dev/null +++ b/Lib/test/test_multiprocessing_fork.py @@ -0,0 +1,7 @@ +import unittest +import test._test_multiprocessing + +test._test_multiprocessing.install_tests_in_module_dict(globals(), 'fork') + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_forkserver.py b/Lib/test/test_multiprocessing_forkserver.py new file mode 100644 index 0000000000..193a04a5fc --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver.py @@ -0,0 +1,7 @@ +import unittest +import test._test_multiprocessing + +test._test_multiprocessing.install_tests_in_module_dict(globals(), 'forkserver') + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn.py b/Lib/test/test_multiprocessing_spawn.py new file mode 100644 index 0000000000..334ae9e8f7 --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn.py @@ -0,0 +1,7 @@ +import unittest +import test._test_multiprocessing + +test._test_multiprocessing.install_tests_in_module_dict(globals(), 'spawn') + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_namespace_pkgs.py b/Lib/test/test_namespace_pkgs.py index 7067b12e8f..4570bee568 100644 --- a/Lib/test/test_namespace_pkgs.py +++ b/Lib/test/test_namespace_pkgs.py @@ -1,7 +1,11 @@ -import sys import contextlib -import unittest +from importlib._bootstrap import NamespaceLoader +import importlib.abc +import importlib.machinery import os +import sys +import types +import unittest from test.test_importlib import util from test.support import run_unittest @@ -286,9 +290,24 @@ class ModuleAndNamespacePackageInSameDir(NamespacePackageTest): self.assertEqual(a_test.attr, 'in module') -def test_main(): - run_unittest(*NamespacePackageTest.__subclasses__()) +class ABCTests(unittest.TestCase): + + def setUp(self): + self.loader = NamespaceLoader('foo', ['pkg'], + importlib.machinery.PathFinder) + + def test_is_package(self): + self.assertTrue(self.loader.is_package('foo')) + + def test_get_code(self): + self.assertTrue(isinstance(self.loader.get_code('foo'), types.CodeType)) + + def test_get_source(self): + self.assertEqual(self.loader.get_source('foo'), '') + + def test_abc_isinstance(self): + self.assertTrue(isinstance(self.loader, importlib.abc.InspectLoader)) if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py index 19fb10a9c7..7cf497ac1c 100644 --- a/Lib/test/test_nntplib.py +++ b/Lib/test/test_nntplib.py @@ -264,7 +264,7 @@ class NetworkedNNTPTestsMixin: return False try: server.help() - except (socket.error, EOFError): + except (OSError, EOFError): return False return True diff --git a/Lib/test/test_normalization.py b/Lib/test/test_normalization.py index 28ede34cd7..ab2eeb77da 100644 --- a/Lib/test/test_normalization.py +++ b/Lib/test/test_normalization.py @@ -43,7 +43,7 @@ class NormalizationTest(unittest.TestCase): try: testdata = open_urlresource(TESTDATAURL, encoding="utf-8", check=check_version) - except (IOError, HTTPException): + except (OSError, HTTPException): self.skipTest("Could not retrieve " + TESTDATAURL) self.addCleanup(testdata.close) for line in testdata: diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index f8098767ce..285ef62a46 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -256,6 +256,40 @@ class TestNtpath(unittest.TestCase): # dialogs (#4804) ntpath.sameopenfile(-1, -1) + def test_ismount(self): + self.assertTrue(ntpath.ismount("c:\\")) + self.assertTrue(ntpath.ismount("C:\\")) + self.assertTrue(ntpath.ismount("c:/")) + self.assertTrue(ntpath.ismount("C:/")) + self.assertTrue(ntpath.ismount("\\\\.\\c:\\")) + self.assertTrue(ntpath.ismount("\\\\.\\C:\\")) + + self.assertTrue(ntpath.ismount(b"c:\\")) + self.assertTrue(ntpath.ismount(b"C:\\")) + self.assertTrue(ntpath.ismount(b"c:/")) + self.assertTrue(ntpath.ismount(b"C:/")) + self.assertTrue(ntpath.ismount(b"\\\\.\\c:\\")) + self.assertTrue(ntpath.ismount(b"\\\\.\\C:\\")) + + with support.temp_dir() as d: + self.assertFalse(ntpath.ismount(d)) + + if sys.platform == "win32": + # + # Make sure the current folder isn't the root folder + # (or any other volume root). The drive-relative + # locations below cannot then refer to mount points + # + drive, path = ntpath.splitdrive(sys.executable) + with support.change_cwd(os.path.dirname(sys.executable)): + self.assertFalse(ntpath.ismount(drive.lower())) + self.assertFalse(ntpath.ismount(drive.upper())) + + self.assertTrue(ntpath.ismount("\\\\localhost\\c$")) + self.assertTrue(ntpath.ismount("\\\\localhost\\c$\\")) + + self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$")) + self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\")) class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase): pathmodule = ntpath diff --git a/Lib/test/test_openpty.py b/Lib/test/test_openpty.py index 63843705b1..47851072c1 100644 --- a/Lib/test/test_openpty.py +++ b/Lib/test/test_openpty.py @@ -4,7 +4,7 @@ import os, unittest from test.support import run_unittest if not hasattr(os, "openpty"): - raise unittest.SkipTest("No openpty() available.") + raise unittest.SkipTest("os.openpty() not available.") class OpenptyTest(unittest.TestCase): diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index fa608b9a52..ab58a98365 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -1,8 +1,10 @@ -import operator import unittest from test import support +py_operator = support.import_fresh_module('operator', blocked=['_operator']) +c_operator = support.import_fresh_module('operator', fresh=['_operator']) + class Seq1: def __init__(self, lst): self.lst = lst @@ -32,8 +34,9 @@ class Seq2(object): return other * self.lst -class OperatorTestCase(unittest.TestCase): +class OperatorTestCase: def test_lt(self): + operator = self.module self.assertRaises(TypeError, operator.lt) self.assertRaises(TypeError, operator.lt, 1j, 2j) self.assertFalse(operator.lt(1, 0)) @@ -44,6 +47,7 @@ class OperatorTestCase(unittest.TestCase): self.assertTrue(operator.lt(1, 2.0)) def test_le(self): + operator = self.module self.assertRaises(TypeError, operator.le) self.assertRaises(TypeError, operator.le, 1j, 2j) self.assertFalse(operator.le(1, 0)) @@ -54,6 +58,7 @@ class OperatorTestCase(unittest.TestCase): self.assertTrue(operator.le(1, 2.0)) def test_eq(self): + operator = self.module class C(object): def __eq__(self, other): raise SyntaxError @@ -67,6 +72,7 @@ class OperatorTestCase(unittest.TestCase): self.assertFalse(operator.eq(1, 2.0)) def test_ne(self): + operator = self.module class C(object): def __ne__(self, other): raise SyntaxError @@ -80,6 +86,7 @@ class OperatorTestCase(unittest.TestCase): self.assertTrue(operator.ne(1, 2.0)) def test_ge(self): + operator = self.module self.assertRaises(TypeError, operator.ge) self.assertRaises(TypeError, operator.ge, 1j, 2j) self.assertTrue(operator.ge(1, 0)) @@ -90,6 +97,7 @@ class OperatorTestCase(unittest.TestCase): self.assertFalse(operator.ge(1, 2.0)) def test_gt(self): + operator = self.module self.assertRaises(TypeError, operator.gt) self.assertRaises(TypeError, operator.gt, 1j, 2j) self.assertTrue(operator.gt(1, 0)) @@ -100,22 +108,26 @@ class OperatorTestCase(unittest.TestCase): self.assertFalse(operator.gt(1, 2.0)) def test_abs(self): + operator = self.module self.assertRaises(TypeError, operator.abs) self.assertRaises(TypeError, operator.abs, None) self.assertEqual(operator.abs(-1), 1) self.assertEqual(operator.abs(1), 1) def test_add(self): + operator = self.module self.assertRaises(TypeError, operator.add) self.assertRaises(TypeError, operator.add, None, None) self.assertTrue(operator.add(3, 4) == 7) def test_bitwise_and(self): + operator = self.module self.assertRaises(TypeError, operator.and_) self.assertRaises(TypeError, operator.and_, None, None) self.assertTrue(operator.and_(0xf, 0xa) == 0xa) def test_concat(self): + operator = self.module self.assertRaises(TypeError, operator.concat) self.assertRaises(TypeError, operator.concat, None, None) self.assertTrue(operator.concat('py', 'thon') == 'python') @@ -125,12 +137,14 @@ class OperatorTestCase(unittest.TestCase): self.assertRaises(TypeError, operator.concat, 13, 29) def test_countOf(self): + operator = self.module self.assertRaises(TypeError, operator.countOf) self.assertRaises(TypeError, operator.countOf, None, None) self.assertTrue(operator.countOf([1, 2, 1, 3, 1, 4], 3) == 1) self.assertTrue(operator.countOf([1, 2, 1, 3, 1, 4], 5) == 0) def test_delitem(self): + operator = self.module a = [4, 3, 2, 1] self.assertRaises(TypeError, operator.delitem, a) self.assertRaises(TypeError, operator.delitem, a, None) @@ -138,33 +152,39 @@ class OperatorTestCase(unittest.TestCase): self.assertTrue(a == [4, 2, 1]) def test_floordiv(self): + operator = self.module self.assertRaises(TypeError, operator.floordiv, 5) self.assertRaises(TypeError, operator.floordiv, None, None) self.assertTrue(operator.floordiv(5, 2) == 2) def test_truediv(self): + operator = self.module self.assertRaises(TypeError, operator.truediv, 5) self.assertRaises(TypeError, operator.truediv, None, None) self.assertTrue(operator.truediv(5, 2) == 2.5) def test_getitem(self): + operator = self.module a = range(10) self.assertRaises(TypeError, operator.getitem) self.assertRaises(TypeError, operator.getitem, a, None) self.assertTrue(operator.getitem(a, 2) == 2) def test_indexOf(self): + operator = self.module self.assertRaises(TypeError, operator.indexOf) self.assertRaises(TypeError, operator.indexOf, None, None) self.assertTrue(operator.indexOf([4, 3, 2, 1], 3) == 1) self.assertRaises(ValueError, operator.indexOf, [4, 3, 2, 1], 0) def test_invert(self): + operator = self.module self.assertRaises(TypeError, operator.invert) self.assertRaises(TypeError, operator.invert, None) self.assertEqual(operator.inv(4), -5) def test_lshift(self): + operator = self.module self.assertRaises(TypeError, operator.lshift) self.assertRaises(TypeError, operator.lshift, None, 42) self.assertTrue(operator.lshift(5, 1) == 10) @@ -172,16 +192,19 @@ class OperatorTestCase(unittest.TestCase): self.assertRaises(ValueError, operator.lshift, 2, -1) def test_mod(self): + operator = self.module self.assertRaises(TypeError, operator.mod) self.assertRaises(TypeError, operator.mod, None, 42) self.assertTrue(operator.mod(5, 2) == 1) def test_mul(self): + operator = self.module self.assertRaises(TypeError, operator.mul) self.assertRaises(TypeError, operator.mul, None, None) self.assertTrue(operator.mul(5, 2) == 10) def test_neg(self): + operator = self.module self.assertRaises(TypeError, operator.neg) self.assertRaises(TypeError, operator.neg, None) self.assertEqual(operator.neg(5), -5) @@ -190,11 +213,13 @@ class OperatorTestCase(unittest.TestCase): self.assertEqual(operator.neg(-0), 0) def test_bitwise_or(self): + operator = self.module self.assertRaises(TypeError, operator.or_) self.assertRaises(TypeError, operator.or_, None, None) self.assertTrue(operator.or_(0xa, 0x5) == 0xf) def test_pos(self): + operator = self.module self.assertRaises(TypeError, operator.pos) self.assertRaises(TypeError, operator.pos, None) self.assertEqual(operator.pos(5), 5) @@ -203,14 +228,15 @@ class OperatorTestCase(unittest.TestCase): self.assertEqual(operator.pos(-0), 0) def test_pow(self): + operator = self.module self.assertRaises(TypeError, operator.pow) self.assertRaises(TypeError, operator.pow, None, None) self.assertEqual(operator.pow(3,5), 3**5) - self.assertEqual(operator.__pow__(3,5), 3**5) self.assertRaises(TypeError, operator.pow, 1) self.assertRaises(TypeError, operator.pow, 1, 2, 3) def test_rshift(self): + operator = self.module self.assertRaises(TypeError, operator.rshift) self.assertRaises(TypeError, operator.rshift, None, 42) self.assertTrue(operator.rshift(5, 1) == 2) @@ -218,12 +244,14 @@ class OperatorTestCase(unittest.TestCase): self.assertRaises(ValueError, operator.rshift, 2, -1) def test_contains(self): + operator = self.module self.assertRaises(TypeError, operator.contains) self.assertRaises(TypeError, operator.contains, None, None) self.assertTrue(operator.contains(range(4), 2)) self.assertFalse(operator.contains(range(4), 5)) def test_setitem(self): + operator = self.module a = list(range(3)) self.assertRaises(TypeError, operator.setitem, a) self.assertRaises(TypeError, operator.setitem, a, None, None) @@ -232,11 +260,13 @@ class OperatorTestCase(unittest.TestCase): self.assertRaises(IndexError, operator.setitem, a, 4, 2) def test_sub(self): + operator = self.module self.assertRaises(TypeError, operator.sub) self.assertRaises(TypeError, operator.sub, None, None) self.assertTrue(operator.sub(5, 2) == 3) def test_truth(self): + operator = self.module class C(object): def __bool__(self): raise SyntaxError @@ -248,11 +278,13 @@ class OperatorTestCase(unittest.TestCase): self.assertFalse(operator.truth([])) def test_bitwise_xor(self): + operator = self.module self.assertRaises(TypeError, operator.xor) self.assertRaises(TypeError, operator.xor, None, None) self.assertTrue(operator.xor(0xb, 0xc) == 0x7) def test_is(self): + operator = self.module a = b = 'xyzpdq' c = a[:3] + b[3:] self.assertRaises(TypeError, operator.is_) @@ -260,6 +292,7 @@ class OperatorTestCase(unittest.TestCase): self.assertFalse(operator.is_(a,c)) def test_is_not(self): + operator = self.module a = b = 'xyzpdq' c = a[:3] + b[3:] self.assertRaises(TypeError, operator.is_not) @@ -267,6 +300,7 @@ class OperatorTestCase(unittest.TestCase): self.assertTrue(operator.is_not(a,c)) def test_attrgetter(self): + operator = self.module class A: pass a = A() @@ -316,6 +350,7 @@ class OperatorTestCase(unittest.TestCase): self.assertEqual(f(a), ('arthur', 'thomas', 'johnson')) def test_itemgetter(self): + operator = self.module a = 'ABCDE' f = operator.itemgetter(2) self.assertEqual(f(a), 'C') @@ -350,12 +385,15 @@ class OperatorTestCase(unittest.TestCase): self.assertRaises(TypeError, operator.itemgetter(2, 'x', 5), data) def test_methodcaller(self): + operator = self.module self.assertRaises(TypeError, operator.methodcaller) class A: def foo(self, *args, **kwds): return args[0] + args[1] def bar(self, f=42): return f + def baz(*args, **kwds): + return kwds['name'], kwds['self'] a = A() f = operator.methodcaller('foo') self.assertRaises(IndexError, f, a) @@ -366,8 +404,11 @@ class OperatorTestCase(unittest.TestCase): self.assertRaises(TypeError, f, a, a) f = operator.methodcaller('bar', f=5) self.assertEqual(f(a), 5) + f = operator.methodcaller('baz', name='spam', self='eggs') + self.assertEqual(f(a), ('spam', 'eggs')) def test_inplace(self): + operator = self.module class C(object): def __iadd__ (self, other): return "iadd" def __iand__ (self, other): return "iand" @@ -396,37 +437,48 @@ class OperatorTestCase(unittest.TestCase): self.assertEqual(operator.itruediv (c, 5), "itruediv") self.assertEqual(operator.ixor (c, 5), "ixor") self.assertEqual(operator.iconcat (c, c), "iadd") - self.assertEqual(operator.__iadd__ (c, 5), "iadd") - self.assertEqual(operator.__iand__ (c, 5), "iand") - self.assertEqual(operator.__ifloordiv__(c, 5), "ifloordiv") - self.assertEqual(operator.__ilshift__ (c, 5), "ilshift") - self.assertEqual(operator.__imod__ (c, 5), "imod") - self.assertEqual(operator.__imul__ (c, 5), "imul") - self.assertEqual(operator.__ior__ (c, 5), "ior") - self.assertEqual(operator.__ipow__ (c, 5), "ipow") - self.assertEqual(operator.__irshift__ (c, 5), "irshift") - self.assertEqual(operator.__isub__ (c, 5), "isub") - self.assertEqual(operator.__itruediv__ (c, 5), "itruediv") - self.assertEqual(operator.__ixor__ (c, 5), "ixor") - self.assertEqual(operator.__iconcat__ (c, c), "iadd") - -def test_main(verbose=None): - import sys - test_classes = ( - OperatorTestCase, - ) - - support.run_unittest(*test_classes) - - # verify reference counting - if verbose and hasattr(sys, "gettotalrefcount"): - import gc - counts = [None] * 5 - for i in range(len(counts)): - support.run_unittest(*test_classes) - gc.collect() - counts[i] = sys.gettotalrefcount() - print(counts) + + def test_length_hint(self): + operator = self.module + class X(object): + def __init__(self, value): + self.value = value + + def __length_hint__(self): + if type(self.value) is type: + raise self.value + else: + return self.value + + self.assertEqual(operator.length_hint([], 2), 0) + self.assertEqual(operator.length_hint(iter([1, 2, 3])), 3) + + self.assertEqual(operator.length_hint(X(2)), 2) + self.assertEqual(operator.length_hint(X(NotImplemented), 4), 4) + self.assertEqual(operator.length_hint(X(TypeError), 12), 12) + with self.assertRaises(TypeError): + operator.length_hint(X("abc")) + with self.assertRaises(ValueError): + operator.length_hint(X(-2)) + with self.assertRaises(LookupError): + operator.length_hint(X(LookupError)) + + def test_dunder_is_original(self): + operator = self.module + + names = [name for name in dir(operator) if not name.startswith('_')] + for name in names: + orig = getattr(operator, name) + dunder = getattr(operator, '__' + name.strip('_') + '__', None) + if dunder: + self.assertIs(dunder, orig) + +class PyOperatorTestCase(OperatorTestCase, unittest.TestCase): + module = py_operator + +@unittest.skipUnless(c_operator, 'requires _operator') +class COperatorTestCase(OperatorTestCase, unittest.TestCase): + module = c_operator if __name__ == "__main__": - test_main(verbose=True) + unittest.main() diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index 78de278f76..94730119e9 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -730,7 +730,7 @@ class TestStandard(BaseTest): def test_short_and_long_option_split(self): self.assertParseOK(["-a", "xyz", "--foo", "bar"], {'a': 'xyz', 'boo': None, 'foo': ["bar"]}, - []), + []) def test_short_option_split_long_option_append(self): self.assertParseOK(["--foo=bar", "-b", "123", "--foo", "baz"], @@ -740,15 +740,15 @@ class TestStandard(BaseTest): def test_short_option_split_one_positional_arg(self): self.assertParseOK(["-a", "foo", "bar"], {'a': "foo", 'boo': None, 'foo': None}, - ["bar"]), + ["bar"]) def test_short_option_consumes_separator(self): self.assertParseOK(["-a", "--", "foo", "bar"], {'a': "--", 'boo': None, 'foo': None}, - ["foo", "bar"]), + ["foo", "bar"]) self.assertParseOK(["-a", "--", "--foo", "bar"], {'a': "--", 'boo': None, 'foo': ["bar"]}, - []), + []) def test_short_option_joined_and_separator(self): self.assertParseOK(["-ab", "--", "--foo", "bar"], diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index d2424d7865..f1d652c9a5 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -24,6 +24,9 @@ import itertools import stat import locale import codecs +import decimal +import fractions +import pickle try: import threading except ImportError: @@ -32,6 +35,10 @@ try: import resource except ImportError: resource = None +try: + import fcntl +except ImportError: + fcntl = None from test.script_helper import assert_python_ok @@ -258,6 +265,13 @@ class StatAttributeTests(unittest.TestCase): warnings.simplefilter("ignore", DeprecationWarning) self.check_stat_attributes(fname) + def test_stat_result_pickle(self): + result = os.stat(self.fname) + p = pickle.dumps(result) + self.assertIn(b'\x03cos\nstat_result\n', p) + unpickled = pickle.loads(p) + self.assertEqual(result, unpickled) + def test_statvfs_attributes(self): if not hasattr(os, "statvfs"): return @@ -267,7 +281,7 @@ class StatAttributeTests(unittest.TestCase): except OSError as e: # On AtheOS, glibc always returns ENOSYS if e.errno == errno.ENOSYS: - return + self.skipTest('os.statvfs() failed with ENOSYS') # Make sure direct access works self.assertEqual(result.f_bfree, result[3]) @@ -304,6 +318,21 @@ class StatAttributeTests(unittest.TestCase): except TypeError: pass + @unittest.skipUnless(hasattr(os, 'statvfs'), + "need os.statvfs()") + def test_statvfs_result_pickle(self): + try: + result = os.statvfs(self.fname) + except OSError as e: + # On AtheOS, glibc always returns ENOSYS + if e.errno == errno.ENOSYS: + self.skipTest('os.statvfs() failed with ENOSYS') + + p = pickle.dumps(result) + self.assertIn(b'\x03cos\nstatvfs_result\n', p) + unpickled = pickle.loads(p) + self.assertEqual(result, unpickled) + def test_utime_dir(self): delta = 1000000 st = os.stat(support.TESTFN) @@ -476,9 +505,9 @@ class StatAttributeTests(unittest.TestCase): # Verify that an open file can be stat'ed try: os.stat(r"c:\pagefile.sys") - except WindowsError as e: - if e.errno == 2: # file does not exist; cannot run test - return + except FileNotFoundError: + pass # file does not exist; cannot run test + except OSError as e: self.fail("Could not stat pagefile.sys") @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") @@ -873,6 +902,17 @@ class MakedirTests(unittest.TestCase): os.makedirs(path, mode=mode, exist_ok=True) os.umask(old_mask) + @unittest.skipUnless(hasattr(os, 'chown'), 'test needs os.chown') + def test_chown_uid_gid_arguments_must_be_index(self): + stat = os.stat(support.TESTFN) + uid = stat.st_uid + gid = stat.st_gid + for value in (-1.0, -1j, decimal.Decimal(-1), fractions.Fraction(-2, 2)): + self.assertRaises(TypeError, os.chown, support.TESTFN, value, gid) + self.assertRaises(TypeError, os.chown, support.TESTFN, uid, value) + self.assertIsNone(os.chown(support.TESTFN, uid, gid)) + self.assertIsNone(os.chown(support.TESTFN, -1, -1)) + def test_exist_ok_s_isgid_directory(self): path = os.path.join(support.TESTFN, 'dir1') S_ISGID = stat.S_ISGID @@ -1129,27 +1169,27 @@ class ExecTests(unittest.TestCase): class Win32ErrorTests(unittest.TestCase): def test_rename(self): - self.assertRaises(WindowsError, os.rename, support.TESTFN, support.TESTFN+".bak") + self.assertRaises(OSError, os.rename, support.TESTFN, support.TESTFN+".bak") def test_remove(self): - self.assertRaises(WindowsError, os.remove, support.TESTFN) + self.assertRaises(OSError, os.remove, support.TESTFN) def test_chdir(self): - self.assertRaises(WindowsError, os.chdir, support.TESTFN) + self.assertRaises(OSError, os.chdir, support.TESTFN) def test_mkdir(self): f = open(support.TESTFN, "w") try: - self.assertRaises(WindowsError, os.mkdir, support.TESTFN) + self.assertRaises(OSError, os.mkdir, support.TESTFN) finally: f.close() os.unlink(support.TESTFN) def test_utime(self): - self.assertRaises(WindowsError, os.utime, support.TESTFN, None) + self.assertRaises(OSError, os.utime, support.TESTFN, None) def test_chmod(self): - self.assertRaises(WindowsError, os.chmod, support.TESTFN, 0) + self.assertRaises(OSError, os.chmod, support.TESTFN, 0) class TestInvalidFD(unittest.TestCase): singles = ["fchdir", "dup", "fdopen", "fdatasync", "fstat", @@ -1277,31 +1317,31 @@ if sys.platform != 'win32': if hasattr(os, 'setuid'): def test_setuid(self): if os.getuid() != 0: - self.assertRaises(os.error, os.setuid, 0) + self.assertRaises(OSError, os.setuid, 0) self.assertRaises(OverflowError, os.setuid, 1<<32) if hasattr(os, 'setgid'): def test_setgid(self): if os.getuid() != 0 and not HAVE_WHEEL_GROUP: - self.assertRaises(os.error, os.setgid, 0) + self.assertRaises(OSError, os.setgid, 0) self.assertRaises(OverflowError, os.setgid, 1<<32) if hasattr(os, 'seteuid'): def test_seteuid(self): if os.getuid() != 0: - self.assertRaises(os.error, os.seteuid, 0) + self.assertRaises(OSError, os.seteuid, 0) self.assertRaises(OverflowError, os.seteuid, 1<<32) if hasattr(os, 'setegid'): def test_setegid(self): if os.getuid() != 0 and not HAVE_WHEEL_GROUP: - self.assertRaises(os.error, os.setegid, 0) + self.assertRaises(OSError, os.setegid, 0) self.assertRaises(OverflowError, os.setegid, 1<<32) if hasattr(os, 'setreuid'): def test_setreuid(self): if os.getuid() != 0: - self.assertRaises(os.error, os.setreuid, 0, 0) + self.assertRaises(OSError, os.setreuid, 0, 0) self.assertRaises(OverflowError, os.setreuid, 1<<32, 0) self.assertRaises(OverflowError, os.setreuid, 0, 1<<32) @@ -1315,7 +1355,7 @@ if sys.platform != 'win32': if hasattr(os, 'setregid'): def test_setregid(self): if os.getuid() != 0 and not HAVE_WHEEL_GROUP: - self.assertRaises(os.error, os.setregid, 0, 0) + self.assertRaises(OSError, os.setregid, 0, 0) self.assertRaises(OverflowError, os.setregid, 1<<32, 0) self.assertRaises(OverflowError, os.setregid, 0, 1<<32) @@ -2179,6 +2219,197 @@ class TermsizeTests(unittest.TestCase): self.assertEqual(expected, actual) +class OSErrorTests(unittest.TestCase): + def setUp(self): + class Str(str): + pass + + self.bytes_filenames = [] + self.unicode_filenames = [] + if support.TESTFN_UNENCODABLE is not None: + decoded = support.TESTFN_UNENCODABLE + else: + decoded = support.TESTFN + self.unicode_filenames.append(decoded) + self.unicode_filenames.append(Str(decoded)) + if support.TESTFN_UNDECODABLE is not None: + encoded = support.TESTFN_UNDECODABLE + else: + encoded = os.fsencode(support.TESTFN) + self.bytes_filenames.append(encoded) + self.bytes_filenames.append(memoryview(encoded)) + + self.filenames = self.bytes_filenames + self.unicode_filenames + + def test_oserror_filename(self): + funcs = [ + (self.filenames, os.chdir,), + (self.filenames, os.chmod, 0o777), + (self.filenames, os.lstat,), + (self.filenames, os.open, os.O_RDONLY), + (self.filenames, os.rmdir,), + (self.filenames, os.stat,), + (self.filenames, os.unlink,), + ] + if sys.platform == "win32": + funcs.extend(( + (self.bytes_filenames, os.rename, b"dst"), + (self.bytes_filenames, os.replace, b"dst"), + (self.unicode_filenames, os.rename, "dst"), + (self.unicode_filenames, os.replace, "dst"), + # Issue #16414: Don't test undecodable names with listdir() + # because of a Windows bug. + # + # With the ANSI code page 932, os.listdir(b'\xe7') return an + # empty list (instead of failing), whereas os.listdir(b'\xff') + # raises a FileNotFoundError. It looks like a Windows bug: + # b'\xe7' directory does not exist, FindFirstFileA(b'\xe7') + # fails with ERROR_FILE_NOT_FOUND (2), instead of + # ERROR_PATH_NOT_FOUND (3). + (self.unicode_filenames, os.listdir,), + )) + else: + funcs.extend(( + (self.filenames, os.listdir,), + (self.filenames, os.rename, "dst"), + (self.filenames, os.replace, "dst"), + )) + if hasattr(os, "chown"): + funcs.append((self.filenames, os.chown, 0, 0)) + if hasattr(os, "lchown"): + funcs.append((self.filenames, os.lchown, 0, 0)) + if hasattr(os, "truncate"): + funcs.append((self.filenames, os.truncate, 0)) + if hasattr(os, "chflags"): + funcs.append((self.filenames, os.chflags, 0)) + if hasattr(os, "lchflags"): + funcs.append((self.filenames, os.lchflags, 0)) + if hasattr(os, "chroot"): + funcs.append((self.filenames, os.chroot,)) + if hasattr(os, "link"): + if sys.platform == "win32": + funcs.append((self.bytes_filenames, os.link, b"dst")) + funcs.append((self.unicode_filenames, os.link, "dst")) + else: + funcs.append((self.filenames, os.link, "dst")) + if hasattr(os, "listxattr"): + funcs.extend(( + (self.filenames, os.listxattr,), + (self.filenames, os.getxattr, "user.test"), + (self.filenames, os.setxattr, "user.test", b'user'), + (self.filenames, os.removexattr, "user.test"), + )) + if hasattr(os, "lchmod"): + funcs.append((self.filenames, os.lchmod, 0o777)) + if hasattr(os, "readlink"): + if sys.platform == "win32": + funcs.append((self.unicode_filenames, os.readlink,)) + else: + funcs.append((self.filenames, os.readlink,)) + + for filenames, func, *func_args in funcs: + for name in filenames: + try: + func(name, *func_args) + except OSError as err: + self.assertIs(err.filename, name) + else: + self.fail("No exception thrown by {}".format(func)) + +class CPUCountTests(unittest.TestCase): + def test_cpu_count(self): + cpus = os.cpu_count() + if cpus is not None: + self.assertIsInstance(cpus, int) + self.assertGreater(cpus, 0) + else: + self.skipTest("Could not determine the number of CPUs") + + +class FDInheritanceTests(unittest.TestCase): + def test_get_set_inheritable(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(os.get_inheritable(fd), False) + + os.set_inheritable(fd, True) + self.assertEqual(os.get_inheritable(fd), True) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_get_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(os.get_inheritable(fd), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(os.get_inheritable(fd), True) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_set_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + os.set_inheritable(fd, True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + + def test_open(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(os.get_inheritable(fd), False) + + @unittest.skipUnless(hasattr(os, 'pipe'), "need os.pipe()") + def test_pipe(self): + rfd, wfd = os.pipe() + self.addCleanup(os.close, rfd) + self.addCleanup(os.close, wfd) + self.assertEqual(os.get_inheritable(rfd), False) + self.assertEqual(os.get_inheritable(wfd), False) + + def test_dup(self): + fd1 = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd1) + + fd2 = os.dup(fd1) + self.addCleanup(os.close, fd2) + self.assertEqual(os.get_inheritable(fd2), False) + + @unittest.skipUnless(hasattr(os, 'dup2'), "need os.dup2()") + def test_dup2(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + + # inheritable by default + fd2 = os.open(__file__, os.O_RDONLY) + try: + os.dup2(fd, fd2) + self.assertEqual(os.get_inheritable(fd2), True) + finally: + os.close(fd2) + + # force non-inheritable + fd3 = os.open(__file__, os.O_RDONLY) + try: + os.dup2(fd, fd3, inheritable=False) + self.assertEqual(os.get_inheritable(fd3), False) + finally: + os.close(fd3) + + @unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") + def test_openpty(self): + master_fd, slave_fd = os.openpty() + self.addCleanup(os.close, master_fd) + self.addCleanup(os.close, slave_fd) + self.assertEqual(os.get_inheritable(master_fd), False) + self.assertEqual(os.get_inheritable(slave_fd), False) + + @support.reap_threads def test_main(): support.run_unittest( @@ -2208,7 +2439,10 @@ def test_main(): ExtendedAttributeTests, Win32DeprecatedBytesAPI, TermsizeTests, + OSErrorTests, RemoveDirsTests, + CPUCountTests, + FDInheritanceTests, ) if __name__ == "__main__": diff --git a/Lib/test/test_ossaudiodev.py b/Lib/test/test_ossaudiodev.py index 3908a0506e..c9e2a24767 100644 --- a/Lib/test/test_ossaudiodev.py +++ b/Lib/test/test_ossaudiodev.py @@ -44,7 +44,7 @@ class OSSAudioDevTests(unittest.TestCase): def play_sound_file(self, data, rate, ssize, nchannels): try: dsp = ossaudiodev.open('w') - except IOError as msg: + except OSError as msg: if msg.args[0] in (errno.EACCES, errno.ENOENT, errno.ENODEV, errno.EBUSY): raise unittest.SkipTest(msg) @@ -190,7 +190,7 @@ class OSSAudioDevTests(unittest.TestCase): def test_main(): try: dsp = ossaudiodev.open('w') - except (ossaudiodev.error, IOError) as msg: + except (ossaudiodev.error, OSError) as msg: if msg.args[0] in (errno.EACCES, errno.ENOENT, errno.ENODEV, errno.EBUSY): raise unittest.SkipTest(msg) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 03084e4d12..7993d02ba1 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1,9 +1,9 @@ # A test suite for pdb; not very comprehensive at the moment. import doctest -import imp import pdb import sys +import types import unittest import subprocess import textwrap @@ -205,7 +205,8 @@ def test_pdb_breakpoint_commands(): ... 'enable 1', ... 'clear 1', ... 'commands 2', - ... 'print 42', + ... 'p "42"', + ... 'print("42", 7*6)', # Issue 18764 (not about breakpoints) ... 'end', ... 'continue', # will stop at breakpoint 2 (line 4) ... 'clear', # clear all! @@ -252,11 +253,13 @@ def test_pdb_breakpoint_commands(): (Pdb) clear 1 Deleted breakpoint 1 at <doctest test.test_pdb.test_pdb_breakpoint_commands[0]>:3 (Pdb) commands 2 - (com) print 42 + (com) p "42" + (com) print("42", 7*6) (com) end (Pdb) continue 1 - 42 + '42' + 42 42 > <doctest test.test_pdb.test_pdb_breakpoint_commands[0]>(4)test_function() -> print(2) (Pdb) clear @@ -464,7 +467,7 @@ def test_pdb_skip_modules(): # Module for testing skipping of module that makes a callback -mod = imp.new_module('module_to_skip') +mod = types.ModuleType('module_to_skip') exec('def foo_pony(callback): x = 1; callback(); return None', mod.__dict__) @@ -617,6 +620,36 @@ class PdbTestCase(unittest.TestCase): stderr = stderr and bytes.decode(stderr) return stdout, stderr + def _assert_find_function(self, file_content, func_name, expected): + file_content = textwrap.dedent(file_content) + + with open(support.TESTFN, 'w') as f: + f.write(file_content) + + expected = None if not expected else ( + expected[0], support.TESTFN, expected[1]) + self.assertEqual( + expected, pdb.find_function(func_name, support.TESTFN)) + + def test_find_function_empty_file(self): + self._assert_find_function('', 'foo', None) + + def test_find_function_found(self): + self._assert_find_function( + """\ + def foo(): + pass + + def bar(): + pass + + def quux(): + pass + """, + 'bar', + ('bar', 4), + ) + def test_issue7964(self): # open the file as binary so we can force \r\n newline with open(support.TESTFN, 'wb') as f: diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 1cacdea569..50257923fe 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -5,44 +5,28 @@ from io import StringIO import unittest from math import copysign -def disassemble(func): - f = StringIO() - tmp = sys.stdout - sys.stdout = f - try: - dis.dis(func) - finally: - sys.stdout = tmp - result = f.getvalue() - f.close() - return result +from test.bytecode_helper import BytecodeTestCase -def dis_single(line): - return disassemble(compile(line, '', 'single')) - - -class TestTranforms(unittest.TestCase): +class TestTranforms(BytecodeTestCase): def test_unot(self): # UNARY_NOT POP_JUMP_IF_FALSE --> POP_JUMP_IF_TRUE' def unot(x): if not x == 2: del x - asm = disassemble(unot) - for elem in ('UNARY_NOT', 'POP_JUMP_IF_FALSE'): - self.assertNotIn(elem, asm) - for elem in ('POP_JUMP_IF_TRUE',): - self.assertIn(elem, asm) + self.assertNotInBytecode(unot, 'UNARY_NOT') + self.assertNotInBytecode(unot, 'POP_JUMP_IF_FALSE') + self.assertInBytecode(unot, 'POP_JUMP_IF_TRUE') def test_elim_inversion_of_is_or_in(self): - for line, elem in ( - ('not a is b', '(is not)',), - ('not a in b', '(not in)',), - ('not a is not b', '(is)',), - ('not a not in b', '(in)',), + for line, cmp_op in ( + ('not a is b', 'is not',), + ('not a in b', 'not in',), + ('not a is not b', 'is',), + ('not a not in b', 'in',), ): - asm = dis_single(line) - self.assertIn(elem, asm) + code = compile(line, '', 'single') + self.assertInBytecode(code, 'COMPARE_OP', cmp_op) def test_global_as_constant(self): # LOAD_GLOBAL None/True/False --> LOAD_CONST None/True/False @@ -56,17 +40,14 @@ class TestTranforms(unittest.TestCase): def h(x): False return x - for func, name in ((f, 'None'), (g, 'True'), (h, 'False')): - asm = disassemble(func) - for elem in ('LOAD_GLOBAL',): - self.assertNotIn(elem, asm) - for elem in ('LOAD_CONST', '('+name+')'): - self.assertIn(elem, asm) + for func, elem in ((f, None), (g, True), (h, False)): + self.assertNotInBytecode(func, 'LOAD_GLOBAL') + self.assertInBytecode(func, 'LOAD_CONST', elem) def f(): 'Adding a docstring made this test fail in Py2.5.0' return None - self.assertIn('LOAD_CONST', disassemble(f)) - self.assertNotIn('LOAD_GLOBAL', disassemble(f)) + self.assertNotInBytecode(f, 'LOAD_GLOBAL') + self.assertInBytecode(f, 'LOAD_CONST', None) def test_while_one(self): # Skip over: LOAD_CONST trueconst POP_JUMP_IF_FALSE xx @@ -74,11 +55,10 @@ class TestTranforms(unittest.TestCase): while 1: pass return list - asm = disassemble(f) for elem in ('LOAD_CONST', 'POP_JUMP_IF_FALSE'): - self.assertNotIn(elem, asm) + self.assertNotInBytecode(f, elem) for elem in ('JUMP_ABSOLUTE',): - self.assertIn(elem, asm) + self.assertInBytecode(f, elem) def test_pack_unpack(self): for line, elem in ( @@ -86,28 +66,30 @@ class TestTranforms(unittest.TestCase): ('a, b = a, b', 'ROT_TWO',), ('a, b, c = a, b, c', 'ROT_THREE',), ): - asm = dis_single(line) - self.assertIn(elem, asm) - self.assertNotIn('BUILD_TUPLE', asm) - self.assertNotIn('UNPACK_TUPLE', asm) + code = compile(line,'','single') + self.assertInBytecode(code, elem) + self.assertNotInBytecode(code, 'BUILD_TUPLE') + self.assertNotInBytecode(code, 'UNPACK_TUPLE') def test_folding_of_tuples_of_constants(self): for line, elem in ( - ('a = 1,2,3', '((1, 2, 3))'), - ('("a","b","c")', "(('a', 'b', 'c'))"), - ('a,b,c = 1,2,3', '((1, 2, 3))'), - ('(None, 1, None)', '((None, 1, None))'), - ('((1, 2), 3, 4)', '(((1, 2), 3, 4))'), + ('a = 1,2,3', (1, 2, 3)), + ('("a","b","c")', ('a', 'b', 'c')), + ('a,b,c = 1,2,3', (1, 2, 3)), + ('(None, 1, None)', (None, 1, None)), + ('((1, 2), 3, 4)', ((1, 2), 3, 4)), ): - asm = dis_single(line) - self.assertIn(elem, asm) - self.assertNotIn('BUILD_TUPLE', asm) + code = compile(line,'','single') + self.assertInBytecode(code, 'LOAD_CONST', elem) + self.assertNotInBytecode(code, 'BUILD_TUPLE') # Long tuples should be folded too. - asm = dis_single(repr(tuple(range(10000)))) + code = compile(repr(tuple(range(10000))),'','single') + self.assertNotInBytecode(code, 'BUILD_TUPLE') # One LOAD_CONST for the tuple, one for the None return value - self.assertEqual(asm.count('LOAD_CONST'), 2) - self.assertNotIn('BUILD_TUPLE', asm) + load_consts = [instr for instr in dis.get_instructions(code) + if instr.opname == 'LOAD_CONST'] + self.assertEqual(len(load_consts), 2) # Bug 1053819: Tuple of constants misidentified when presented with: # . . . opcode_with_arg 100 unary_opcode BUILD_TUPLE 1 . . . @@ -129,14 +111,14 @@ class TestTranforms(unittest.TestCase): def test_folding_of_lists_of_constants(self): for line, elem in ( # in/not in constants with BUILD_LIST should be folded to a tuple: - ('a in [1,2,3]', '(1, 2, 3)'), - ('a not in ["a","b","c"]', "(('a', 'b', 'c'))"), - ('a in [None, 1, None]', '((None, 1, None))'), - ('a not in [(1, 2), 3, 4]', '(((1, 2), 3, 4))'), + ('a in [1,2,3]', (1, 2, 3)), + ('a not in ["a","b","c"]', ('a', 'b', 'c')), + ('a in [None, 1, None]', (None, 1, None)), + ('a not in [(1, 2), 3, 4]', ((1, 2), 3, 4)), ): - asm = dis_single(line) - self.assertIn(elem, asm) - self.assertNotIn('BUILD_LIST', asm) + code = compile(line, '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', elem) + self.assertNotInBytecode(code, 'BUILD_LIST') def test_folding_of_sets_of_constants(self): for line, elem in ( @@ -147,18 +129,9 @@ class TestTranforms(unittest.TestCase): ('a not in {(1, 2), 3, 4}', frozenset({(1, 2), 3, 4})), ('a in {1, 2, 3, 3, 2, 1}', frozenset({1, 2, 3})), ): - asm = dis_single(line) - self.assertNotIn('BUILD_SET', asm) - - # Verify that the frozenset 'elem' is in the disassembly - # The ordering of the elements in repr( frozenset ) isn't - # guaranteed, so we jump through some hoops to ensure that we have - # the frozenset we expect: - self.assertIn('frozenset', asm) - # Extract the frozenset literal from the disassembly: - m = re.match(r'.*(frozenset\({.*}\)).*', asm, re.DOTALL) - self.assertTrue(m) - self.assertEqual(eval(m.group(1)), elem) + code = compile(line, '', 'single') + self.assertNotInBytecode(code, 'BUILD_SET') + self.assertInBytecode(code, 'LOAD_CONST', elem) # Ensure that the resulting code actually works: def f(a): @@ -176,98 +149,103 @@ class TestTranforms(unittest.TestCase): def test_folding_of_binops_on_constants(self): for line, elem in ( - ('a = 2+3+4', '(9)'), # chained fold - ('"@"*4', "('@@@@')"), # check string ops - ('a="abc" + "def"', "('abcdef')"), # check string ops - ('a = 3**4', '(81)'), # binary power - ('a = 3*4', '(12)'), # binary multiply - ('a = 13//4', '(3)'), # binary floor divide - ('a = 14%4', '(2)'), # binary modulo - ('a = 2+3', '(5)'), # binary add - ('a = 13-4', '(9)'), # binary subtract - ('a = (12,13)[1]', '(13)'), # binary subscr - ('a = 13 << 2', '(52)'), # binary lshift - ('a = 13 >> 2', '(3)'), # binary rshift - ('a = 13 & 7', '(5)'), # binary and - ('a = 13 ^ 7', '(10)'), # binary xor - ('a = 13 | 7', '(15)'), # binary or + ('a = 2+3+4', 9), # chained fold + ('"@"*4', '@@@@'), # check string ops + ('a="abc" + "def"', 'abcdef'), # check string ops + ('a = 3**4', 81), # binary power + ('a = 3*4', 12), # binary multiply + ('a = 13//4', 3), # binary floor divide + ('a = 14%4', 2), # binary modulo + ('a = 2+3', 5), # binary add + ('a = 13-4', 9), # binary subtract + ('a = (12,13)[1]', 13), # binary subscr + ('a = 13 << 2', 52), # binary lshift + ('a = 13 >> 2', 3), # binary rshift + ('a = 13 & 7', 5), # binary and + ('a = 13 ^ 7', 10), # binary xor + ('a = 13 | 7', 15), # binary or ): - asm = dis_single(line) - self.assertIn(elem, asm, asm) - self.assertNotIn('BINARY_', asm) + code = compile(line, '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', elem) + for instr in dis.get_instructions(code): + self.assertFalse(instr.opname.startswith('BINARY_')) # Verify that unfoldables are skipped - asm = dis_single('a=2+"b"') - self.assertIn('(2)', asm) - self.assertIn("('b')", asm) + code = compile('a=2+"b"', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', 2) + self.assertInBytecode(code, 'LOAD_CONST', 'b') # Verify that large sequences do not result from folding - asm = dis_single('a="x"*1000') - self.assertIn('(1000)', asm) + code = compile('a="x"*1000', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', 1000) def test_binary_subscr_on_unicode(self): # valid code get optimized - asm = dis_single('"foo"[0]') - self.assertIn("('f')", asm) - self.assertNotIn('BINARY_SUBSCR', asm) - asm = dis_single('"\u0061\uffff"[1]') - self.assertIn("('\\uffff')", asm) - self.assertNotIn('BINARY_SUBSCR', asm) - asm = dis_single('"\U00012345abcdef"[3]') - self.assertIn("('c')", asm) - self.assertNotIn('BINARY_SUBSCR', asm) + code = compile('"foo"[0]', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', 'f') + self.assertNotInBytecode(code, 'BINARY_SUBSCR') + code = compile('"\u0061\uffff"[1]', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', '\uffff') + self.assertNotInBytecode(code,'BINARY_SUBSCR') + + # With PEP 393, non-BMP char get optimized + code = compile('"\U00012345"[0]', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', '\U00012345') + self.assertNotInBytecode(code, 'BINARY_SUBSCR') # invalid code doesn't get optimized # out of range - asm = dis_single('"fuu"[10]') - self.assertIn('BINARY_SUBSCR', asm) + code = compile('"fuu"[10]', '', 'single') + self.assertInBytecode(code, 'BINARY_SUBSCR') def test_folding_of_unaryops_on_constants(self): for line, elem in ( - ('-0.5', '(-0.5)'), # unary negative - ('-0.0', '(-0.0)'), # -0.0 - ('-(1.0-1.0)','(-0.0)'), # -0.0 after folding - ('-0', '(0)'), # -0 - ('~-2', '(1)'), # unary invert - ('+1', '(1)'), # unary positive + ('-0.5', -0.5), # unary negative + ('-0.0', -0.0), # -0.0 + ('-(1.0-1.0)', -0.0), # -0.0 after folding + ('-0', 0), # -0 + ('~-2', 1), # unary invert + ('+1', 1), # unary positive ): - asm = dis_single(line) - self.assertIn(elem, asm, asm) - self.assertNotIn('UNARY_', asm) + code = compile(line, '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', elem) + for instr in dis.get_instructions(code): + self.assertFalse(instr.opname.startswith('UNARY_')) # Check that -0.0 works after marshaling def negzero(): return -(1.0-1.0) - self.assertNotIn('UNARY_', disassemble(negzero)) - self.assertTrue(copysign(1.0, negzero()) < 0) + for instr in dis.get_instructions(code): + self.assertFalse(instr.opname.startswith('UNARY_')) # Verify that unfoldables are skipped - for line, elem in ( - ('-"abc"', "('abc')"), # unary negative - ('~"abc"', "('abc')"), # unary invert + for line, elem, opname in ( + ('-"abc"', 'abc', 'UNARY_NEGATIVE'), + ('~"abc"', 'abc', 'UNARY_INVERT'), ): - asm = dis_single(line) - self.assertIn(elem, asm, asm) - self.assertIn('UNARY_', asm) + code = compile(line, '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', elem) + self.assertInBytecode(code, opname) def test_elim_extra_return(self): # RETURN LOAD_CONST None RETURN --> RETURN def f(x): return x - asm = disassemble(f) - self.assertNotIn('LOAD_CONST', asm) - self.assertNotIn('(None)', asm) - self.assertEqual(asm.split().count('RETURN_VALUE'), 1) + self.assertNotInBytecode(f, 'LOAD_CONST', None) + returns = [instr for instr in dis.get_instructions(f) + if instr.opname == 'RETURN_VALUE'] + self.assertEqual(len(returns), 1) def test_elim_jump_to_return(self): # JUMP_FORWARD to RETURN --> RETURN def f(cond, true_value, false_value): return true_value if cond else false_value - asm = disassemble(f) - self.assertNotIn('JUMP_FORWARD', asm) - self.assertNotIn('JUMP_ABSOLUTE', asm) - self.assertEqual(asm.split().count('RETURN_VALUE'), 2) + self.assertNotInBytecode(f, 'JUMP_FORWARD') + self.assertNotInBytecode(f, 'JUMP_ABSOLUTE') + returns = [instr for instr in dis.get_instructions(f) + if instr.opname == 'RETURN_VALUE'] + self.assertEqual(len(returns), 2) def test_elim_jump_after_return1(self): # Eliminate dead code: jumps immediately after returns can't be reached @@ -280,48 +258,53 @@ class TestTranforms(unittest.TestCase): if cond1: return 4 return 5 return 6 - asm = disassemble(f) - self.assertNotIn('JUMP_FORWARD', asm) - self.assertNotIn('JUMP_ABSOLUTE', asm) - self.assertEqual(asm.split().count('RETURN_VALUE'), 6) + self.assertNotInBytecode(f, 'JUMP_FORWARD') + self.assertNotInBytecode(f, 'JUMP_ABSOLUTE') + returns = [instr for instr in dis.get_instructions(f) + if instr.opname == 'RETURN_VALUE'] + self.assertEqual(len(returns), 6) def test_elim_jump_after_return2(self): # Eliminate dead code: jumps immediately after returns can't be reached def f(cond1, cond2): while 1: if cond1: return 4 - asm = disassemble(f) - self.assertNotIn('JUMP_FORWARD', asm) + self.assertNotInBytecode(f, 'JUMP_FORWARD') # There should be one jump for the while loop. - self.assertEqual(asm.split().count('JUMP_ABSOLUTE'), 1) - self.assertEqual(asm.split().count('RETURN_VALUE'), 2) + returns = [instr for instr in dis.get_instructions(f) + if instr.opname == 'JUMP_ABSOLUTE'] + self.assertEqual(len(returns), 1) + returns = [instr for instr in dis.get_instructions(f) + if instr.opname == 'RETURN_VALUE'] + self.assertEqual(len(returns), 2) def test_make_function_doesnt_bail(self): def f(): def g()->1+1: pass return g - asm = disassemble(f) - self.assertNotIn('BINARY_ADD', asm) + self.assertNotInBytecode(f, 'BINARY_ADD') def test_constant_folding(self): # Issue #11244: aggressive constant folding. exprs = [ - "3 * -5", - "-3 * 5", - "2 * (3 * 4)", - "(2 * 3) * 4", - "(-1, 2, 3)", - "(1, -2, 3)", - "(1, 2, -3)", - "(1, 2, -3) * 6", - "lambda x: x in {(3 * -5) + (-1 - 6), (1, -2, 3) * 2, None}", + '3 * -5', + '-3 * 5', + '2 * (3 * 4)', + '(2 * 3) * 4', + '(-1, 2, 3)', + '(1, -2, 3)', + '(1, 2, -3)', + '(1, 2, -3) * 6', + 'lambda x: x in {(3 * -5) + (-1 - 6), (1, -2, 3) * 2, None}', ] for e in exprs: - asm = dis_single(e) - self.assertNotIn('UNARY_', asm, e) - self.assertNotIn('BINARY_', asm, e) - self.assertNotIn('BUILD_', asm, e) + code = compile(e, '', 'single') + for instr in dis.get_instructions(code): + self.assertFalse(instr.opname.startswith('UNARY_')) + self.assertFalse(instr.opname.startswith('BINARY_')) + self.assertFalse(instr.opname.startswith('BUILD_')) + class TestBuglets(unittest.TestCase): @@ -343,7 +326,7 @@ def test_main(verbose=None): support.run_unittest(*test_classes) # verify reference counting - if verbose and hasattr(sys, "gettotalrefcount"): + if verbose and hasattr(sys, 'gettotalrefcount'): import gc counts = [None] * 5 for i in range(len(counts)): diff --git a/Lib/test/test_pep277.py b/Lib/test/test_pep277.py index 4b16cbb383..9bae6dcad7 100644 --- a/Lib/test/test_pep277.py +++ b/Lib/test/test_pep277.py @@ -99,10 +99,6 @@ class UnicodeFileTests(unittest.TestCase): with self.assertRaises(expected_exception) as c: fn(filename) exc_filename = c.exception.filename - # listdir may append a wildcard to the filename - if fn is os.listdir and sys.platform == 'win32': - exc_filename, _, wildcard = exc_filename.rpartition(os.sep) - self.assertEqual(wildcard, '*.*') if check_filename: self.assertEqual(exc_filename, filename, "Function '%s(%a) failed " "with bad filename in the exception: %a" % diff --git a/Lib/test/test_pep352.py b/Lib/test/test_pep352.py index 558cdb56d2..7c98c460b9 100644 --- a/Lib/test/test_pep352.py +++ b/Lib/test/test_pep352.py @@ -1,7 +1,6 @@ import unittest import builtins import warnings -from test.support import run_unittest import os from platform import system as platform_system @@ -180,8 +179,6 @@ class UsageTests(unittest.TestCase): # Catching a string is bad. self.catch_fails("spam") -def test_main(): - run_unittest(ExceptionClassTests, UsageTests) if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_pkgimport.py b/Lib/test/test_pkgimport.py index a8426b5c9a..370b2aae28 100644 --- a/Lib/test/test_pkgimport.py +++ b/Lib/test/test_pkgimport.py @@ -6,7 +6,7 @@ import random import tempfile import unittest -from imp import cache_from_source +from importlib.util import cache_from_source from test.support import run_unittest, create_empty_file class TestImport(unittest.TestCase): diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index fd0661450a..1f488534d7 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -1,12 +1,12 @@ from test.support import run_unittest, unload, check_warnings import unittest import sys -import imp import importlib import pkgutil import os import os.path import tempfile +import types import shutil import zipfile @@ -105,7 +105,7 @@ class PkgutilPEP302Tests(unittest.TestCase): class MyTestLoader(object): def load_module(self, fullname): # Create an empty module - mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) + mod = sys.modules.setdefault(fullname, types.ModuleType(fullname)) mod.__file__ = "<%s>" % self.__class__.__name__ mod.__loader__ = self # Make it a package diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py index f98a280e9a..a1e5c3dbf8 100644 --- a/Lib/test/test_poll.py +++ b/Lib/test/test_poll.py @@ -1,6 +1,7 @@ # Test case for the os.poll() function import os +import subprocess import random import select import _testcapi @@ -15,7 +16,7 @@ from test.support import TESTFN, run_unittest, reap_threads try: select.poll except AttributeError: - raise unittest.SkipTest("select.poll not defined -- skipping test_poll") + raise unittest.SkipTest("select.poll not defined") def find_ready_matching(ready, flag): @@ -76,13 +77,11 @@ class PollTests(unittest.TestCase): self.assertEqual(bufs, [MSG] * NUM_PIPES) - def poll_unit_tests(self): + def test_poll_unit_tests(self): # returns NVAL for invalid file descriptor - FD = 42 - try: - os.close(FD) - except OSError: - pass + FD, w = os.pipe() + os.close(FD) + os.close(w) p = select.poll() p.register(FD) r = p.poll() @@ -125,7 +124,9 @@ class PollTests(unittest.TestCase): def test_poll2(self): cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done' - p = os.popen(cmd, 'r') + proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + bufsize=0) + p = proc.stdout pollster = select.poll() pollster.register( p, select.POLLIN ) for tout in (0, 1000, 2000, 4000, 8000, 16000) + (-1,)*10: @@ -135,7 +136,7 @@ class PollTests(unittest.TestCase): fd, flags = fdlist[0] if flags & select.POLLHUP: line = p.readline() - if line != "": + if line != b"": self.fail('error: pipe seems to be closed, but still returns data') continue @@ -143,6 +144,7 @@ class PollTests(unittest.TestCase): line = p.readline() if not line: break + self.assertEqual(line, b'testing...\n') continue else: self.fail('Unexpected return value from select.poll: %s' % fdlist) diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index c0929a06b3..935848bc5f 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -18,6 +18,13 @@ threading = test_support.import_module('threading') HOST = test_support.HOST PORT = 0 +SUPPORTS_SSL = False +if hasattr(poplib, 'POP3_SSL'): + import ssl + + SUPPORTS_SSL = True + CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "keycert.pem") + # the dummy data returned by server when LIST and RETR commands are issued LIST_RESP = b'1 1\r\n2 2\r\n3 3\r\n4 4\r\n5 5\r\n.\r\n' RETR_RESP = b"""From: postmaster@python.org\ @@ -33,11 +40,15 @@ line3\r\n\ class DummyPOP3Handler(asynchat.async_chat): + CAPAS = {'UIDL': [], 'IMPLEMENTATION': ['python-testlib-pop-server']} + def __init__(self, conn): asynchat.async_chat.__init__(self, conn) self.set_terminator(b"\r\n") self.in_buffer = [] self.push('+OK dummy pop3 server ready. <timestamp>') + self.tls_active = False + self.tls_starting = False def collect_incoming_data(self, data): self.in_buffer.append(data) @@ -112,6 +123,65 @@ class DummyPOP3Handler(asynchat.async_chat): self.push('+OK closing.') self.close_when_done() + def _get_capas(self): + _capas = dict(self.CAPAS) + if not self.tls_active and SUPPORTS_SSL: + _capas['STLS'] = [] + return _capas + + def cmd_capa(self, arg): + self.push('+OK Capability list follows') + if self._get_capas(): + for cap, params in self._get_capas().items(): + _ln = [cap] + if params: + _ln.extend(params) + self.push(' '.join(_ln)) + self.push('.') + + if SUPPORTS_SSL: + + def cmd_stls(self, arg): + if self.tls_active is False: + self.push('+OK Begin TLS negotiation') + tls_sock = ssl.wrap_socket(self.socket, certfile=CERTFILE, + server_side=True, + do_handshake_on_connect=False, + suppress_ragged_eofs=False) + self.del_channel() + self.set_socket(tls_sock) + self.tls_active = True + self.tls_starting = True + self.in_buffer = [] + self._do_tls_handshake() + else: + self.push('-ERR Command not permitted when TLS active') + + def _do_tls_handshake(self): + try: + self.socket.do_handshake() + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return + elif err.args[0] == ssl.SSL_ERROR_EOF: + return self.handle_close() + raise + except OSError as err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self.tls_active = True + self.tls_starting = False + + def handle_read(self): + if self.tls_starting: + self._do_tls_handshake() + else: + try: + asynchat.async_chat.handle_read(self) + except ssl.SSLEOFError: + self.handle_close() class DummyPOP3Server(asyncore.dispatcher, threading.Thread): @@ -232,19 +302,35 @@ class TestPOP3Class(TestCase): self.client.uidl() self.client.uidl('foo') + def test_capa(self): + capa = self.client.capa() + self.assertTrue('IMPLEMENTATION' in capa.keys()) + def test_quit(self): resp = self.client.quit() self.assertTrue(resp) self.assertIsNone(self.client.sock) self.assertIsNone(self.client.file) + if SUPPORTS_SSL: -SUPPORTS_SSL = False -if hasattr(poplib, 'POP3_SSL'): - import ssl + def test_stls_capa(self): + capa = self.client.capa() + self.assertTrue('STLS' in capa.keys()) - SUPPORTS_SSL = True - CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "keycert.pem") + def test_stls(self): + expected = b'+OK Begin TLS negotiation' + resp = self.client.stls() + self.assertEqual(resp, expected) + + def test_stls_context(self): + expected = b'+OK Begin TLS negotiation' + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + resp = self.client.stls(context=ctx) + self.assertEqual(resp, expected) + + +if SUPPORTS_SSL: class DummyPOP3_SSLHandler(DummyPOP3Handler): @@ -256,34 +342,13 @@ if hasattr(poplib, 'POP3_SSL'): self.del_channel() self.set_socket(ssl_socket) # Must try handshake before calling push() - self._ssl_accepting = True - self._do_ssl_handshake() + self.tls_active = True + self.tls_starting = True + self._do_tls_handshake() self.set_terminator(b"\r\n") self.in_buffer = [] self.push('+OK dummy pop3 server ready. <timestamp>') - def _do_ssl_handshake(self): - try: - self.socket.do_handshake() - except ssl.SSLError as err: - if err.args[0] in (ssl.SSL_ERROR_WANT_READ, - ssl.SSL_ERROR_WANT_WRITE): - return - elif err.args[0] == ssl.SSL_ERROR_EOF: - return self.handle_close() - raise - except socket.error as err: - if err.args[0] == errno.ECONNABORTED: - return self.handle_close() - else: - self._ssl_accepting = False - - def handle_read(self): - if self._ssl_accepting: - self._do_ssl_handshake() - else: - DummyPOP3Handler.handle_read(self) - class TestPOP3_SSLClass(TestPOP3Class): # repeat previous tests by using poplib.POP3_SSL @@ -314,6 +379,39 @@ if hasattr(poplib, 'POP3_SSL'): self.assertIs(self.client.sock.context, ctx) self.assertTrue(self.client.noop().startswith(b'+OK')) + def test_stls(self): + self.assertRaises(poplib.error_proto, self.client.stls) + + test_stls_context = test_stls + + def test_stls_capa(self): + capa = self.client.capa() + self.assertFalse('STLS' in capa.keys()) + + + class TestPOP3_TLSClass(TestPOP3Class): + # repeat previous tests by using poplib.POP3.stls() + + def setUp(self): + self.server = DummyPOP3Server((HOST, PORT)) + self.server.start() + self.client = poplib.POP3(self.server.host, self.server.port, timeout=3) + self.client.stls() + + def tearDown(self): + if self.client.file is not None and self.client.sock is not None: + self.client.quit() + self.server.stop() + + def test_stls(self): + self.assertRaises(poplib.error_proto, self.client.stls) + + test_stls_context = test_stls + + def test_stls_capa(self): + capa = self.client.capa() + self.assertFalse(b'STLS' in capa.keys()) + class TestTimeouts(TestCase): @@ -373,6 +471,7 @@ def test_main(): tests = [TestPOP3Class, TestTimeouts] if SUPPORTS_SSL: tests.append(TestPOP3_SSLClass) + tests.append(TestPOP3_TLSClass) thread_info = test_support.threading_setup() try: test_support.run_unittest(*tests) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 4d9e1f599b..6cd3393a63 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -563,8 +563,10 @@ class PosixTester(unittest.TestCase): r, w = os.pipe2(os.O_CLOEXEC|os.O_NONBLOCK) self.addCleanup(os.close, r) self.addCleanup(os.close, w) - self.assertTrue(fcntl.fcntl(r, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) - self.assertTrue(fcntl.fcntl(w, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) + self.assertFalse(os.get_inheritable(r)) + self.assertFalse(os.get_inheritable(w)) + self.assertTrue(fcntl.fcntl(r, fcntl.F_GETFL) & os.O_NONBLOCK) + self.assertTrue(fcntl.fcntl(w, fcntl.F_GETFL) & os.O_NONBLOCK) # try reading from an empty pipe: this should fail, not block self.assertRaises(OSError, os.read, r, 1) # try a write big enough to fill-up the pipe: this should either @@ -607,7 +609,7 @@ class PosixTester(unittest.TestCase): self.assertEqual(st.st_flags | stat.UF_IMMUTABLE, new_st.st_flags) try: fd = open(target_file, 'w+') - except IOError as e: + except OSError as e: self.assertEqual(e.errno, errno.EPERM) finally: posix.chflags(target_file, st.st_flags) @@ -664,41 +666,39 @@ class PosixTester(unittest.TestCase): self.assertEqual(type(v), item_type) def test_getcwd_long_pathnames(self): - if hasattr(posix, 'getcwd'): - dirname = 'getcwd-test-directory-0123456789abcdef-01234567890abcdef' - curdir = os.getcwd() - base_path = os.path.abspath(support.TESTFN) + '.getcwd' + dirname = 'getcwd-test-directory-0123456789abcdef-01234567890abcdef' + curdir = os.getcwd() + base_path = os.path.abspath(support.TESTFN) + '.getcwd' - try: - os.mkdir(base_path) - os.chdir(base_path) - except: -# Just returning nothing instead of the SkipTest exception, -# because the test results in Error in that case. -# Is that ok? -# raise unittest.SkipTest("cannot create directory for testing") - return - - def _create_and_do_getcwd(dirname, current_path_length = 0): - try: - os.mkdir(dirname) - except: - raise unittest.SkipTest("mkdir cannot create directory sufficiently deep for getcwd test") - - os.chdir(dirname) - try: - os.getcwd() - if current_path_length < 1027: - _create_and_do_getcwd(dirname, current_path_length + len(dirname) + 1) - finally: - os.chdir('..') - os.rmdir(dirname) - - _create_and_do_getcwd(dirname) + try: + os.mkdir(base_path) + os.chdir(base_path) + except: + # Just returning nothing instead of the SkipTest exception, because + # the test results in Error in that case. Is that ok? + # raise unittest.SkipTest("cannot create directory for testing") + return - finally: - os.chdir(curdir) - support.rmtree(base_path) + def _create_and_do_getcwd(dirname, current_path_length = 0): + try: + os.mkdir(dirname) + except: + raise unittest.SkipTest("mkdir cannot create directory sufficiently deep for getcwd test") + + os.chdir(dirname) + try: + os.getcwd() + if current_path_length < 1027: + _create_and_do_getcwd(dirname, current_path_length + len(dirname) + 1) + finally: + os.chdir('..') + os.rmdir(dirname) + + _create_and_do_getcwd(dirname) + + finally: + os.chdir(curdir) + support.rmtree(base_path) @unittest.skipUnless(hasattr(posix, 'getgrouplist'), "test needs posix.getgrouplist()") @unittest.skipUnless(hasattr(pwd, 'getpwuid'), "test needs pwd.getpwuid()") diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 0e7d866485..412849cff3 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -186,63 +186,6 @@ class PosixPathTest(unittest.TestCase): if not f.close(): f.close() - @staticmethod - def _create_file(filename): - with open(filename, 'wb') as f: - f.write(b'foo') - - def test_samefile(self): - test_fn = support.TESTFN + "1" - self._create_file(test_fn) - self.assertTrue(posixpath.samefile(test_fn, test_fn)) - self.assertRaises(TypeError, posixpath.samefile) - - @unittest.skipIf( - sys.platform.startswith('win'), - "posixpath.samefile does not work on links in Windows") - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") - def test_samefile_on_links(self): - test_fn1 = support.TESTFN + "1" - test_fn2 = support.TESTFN + "2" - self._create_file(test_fn1) - - os.symlink(test_fn1, test_fn2) - self.assertTrue(posixpath.samefile(test_fn1, test_fn2)) - os.remove(test_fn2) - - self._create_file(test_fn2) - self.assertFalse(posixpath.samefile(test_fn1, test_fn2)) - - - def test_samestat(self): - test_fn = support.TESTFN + "1" - self._create_file(test_fn) - test_fns = [test_fn]*2 - stats = map(os.stat, test_fns) - self.assertTrue(posixpath.samestat(*stats)) - - @unittest.skipIf( - sys.platform.startswith('win'), - "posixpath.samestat does not work on links in Windows") - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") - def test_samestat_on_links(self): - test_fn1 = support.TESTFN + "1" - test_fn2 = support.TESTFN + "2" - self._create_file(test_fn1) - test_fns = (test_fn1, test_fn2) - os.symlink(*test_fns) - stats = map(os.stat, test_fns) - self.assertTrue(posixpath.samestat(*stats)) - os.remove(test_fn2) - - self._create_file(test_fn2) - stats = map(os.stat, test_fns) - self.assertFalse(posixpath.samestat(*stats)) - - self.assertRaises(TypeError, posixpath.samestat) - def test_ismount(self): self.assertIs(posixpath.ismount("/"), True) with warnings.catch_warnings(): @@ -595,11 +538,6 @@ class PosixPathTest(unittest.TestCase): finally: os.getcwdb = real_getcwdb - def test_sameopenfile(self): - fname = support.TESTFN + "1" - with open(fname, "wb") as a, open(fname, "wb") as b: - self.assertTrue(posixpath.sameopenfile(a.fileno(), b.fileno())) - class PosixCommonTest(test_genericpath.CommonTest, unittest.TestCase): pathmodule = posixpath diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index a85298e26f..3d364c4595 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import pprint import test.support import unittest @@ -530,6 +532,54 @@ frozenset2({0, self.assertEqual(pprint.pformat(dict.fromkeys(keys, 0)), '{%r: 0, %r: 0}' % tuple(sorted(keys, key=id))) + def test_str_wrap(self): + # pprint tries to wrap strings intelligently + fox = 'the quick brown fox jumped over a lazy dog' + self.assertEqual(pprint.pformat(fox, width=20), """\ +'the quick brown ' +'fox jumped over ' +'a lazy dog'""") + self.assertEqual(pprint.pformat({'a': 1, 'b': fox, 'c': 2}, + width=26), """\ +{'a': 1, + 'b': 'the quick brown ' + 'fox jumped over ' + 'a lazy dog', + 'c': 2}""") + # With some special characters + # - \n always triggers a new line in the pprint + # - \t and \n are escaped + # - non-ASCII is allowed + # - an apostrophe doesn't disrupt the pprint + special = "Portons dix bons \"whiskys\"\nà l'avocat goujat\t qui fumait au zoo" + self.assertEqual(pprint.pformat(special, width=20), """\ +'Portons dix bons ' +'"whiskys"\\n' +"à l'avocat " +'goujat\\t qui ' +'fumait au zoo'""") + # An unwrappable string is formatted as its repr + unwrappable = "x" * 100 + self.assertEqual(pprint.pformat(unwrappable, width=80), repr(unwrappable)) + self.assertEqual(pprint.pformat(''), "''") + # Check that the pprint is a usable repr + special *= 10 + for width in range(3, 40): + formatted = pprint.pformat(special, width=width) + self.assertEqual(eval("(" + formatted + ")"), special) + + def test_compact(self): + o = ([list(range(i * i)) for i in range(5)] + + [list(range(i)) for i in range(6)]) + expected = """\ +[[], [0], [0, 1, 2, 3], + [0, 1, 2, 3, 4, 5, 6, 7, 8], + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15], + [], [0], [0, 1], [0, 1, 2], [0, 1, 2, 3], + [0, 1, 2, 3, 4]]""" + self.assertEqual(pprint.pformat(o, width=48, compact=True), expected) + class DottedPrettyPrinter(pprint.PrettyPrinter): diff --git a/Lib/test/test_print.py b/Lib/test/test_print.py index 9d6dbea46b..7eea349115 100644 --- a/Lib/test/test_print.py +++ b/Lib/test/test_print.py @@ -1,62 +1,55 @@ -"""Test correct operation of the print function. -""" - -# In 2.6, this gives us the behavior we want. In 3.0, it has -# no function, but it still must parse correctly. -from __future__ import print_function - import unittest -from test import support +from io import StringIO -try: - # 3.x - from io import StringIO -except ImportError: - # 2.x - from StringIO import StringIO +from test import support NotDefined = object() # A dispatch table all 8 combinations of providing -# sep, end, and file +# sep, end, and file. # I use this machinery so that I'm not just passing default -# values to print, I'm either passing or not passing in the -# arguments +# values to print, I'm either passing or not passing in the +# arguments. dispatch = { (False, False, False): - lambda args, sep, end, file: print(*args), + lambda args, sep, end, file: print(*args), (False, False, True): - lambda args, sep, end, file: print(file=file, *args), + lambda args, sep, end, file: print(file=file, *args), (False, True, False): - lambda args, sep, end, file: print(end=end, *args), + lambda args, sep, end, file: print(end=end, *args), (False, True, True): - lambda args, sep, end, file: print(end=end, file=file, *args), + lambda args, sep, end, file: print(end=end, file=file, *args), (True, False, False): - lambda args, sep, end, file: print(sep=sep, *args), + lambda args, sep, end, file: print(sep=sep, *args), (True, False, True): - lambda args, sep, end, file: print(sep=sep, file=file, *args), + lambda args, sep, end, file: print(sep=sep, file=file, *args), (True, True, False): - lambda args, sep, end, file: print(sep=sep, end=end, *args), + lambda args, sep, end, file: print(sep=sep, end=end, *args), (True, True, True): - lambda args, sep, end, file: print(sep=sep, end=end, file=file, *args), - } + lambda args, sep, end, file: print(sep=sep, end=end, file=file, *args), +} + # Class used to test __str__ and print class ClassWith__str__: def __init__(self, x): self.x = x + def __str__(self): return self.x + class TestPrint(unittest.TestCase): + """Test correct operation of the print function.""" + def check(self, expected, args, - sep=NotDefined, end=NotDefined, file=NotDefined): + sep=NotDefined, end=NotDefined, file=NotDefined): # Capture sys.stdout in a StringIO. Call print with args, - # and with sep, end, and file, if they're defined. Result - # must match expected. + # and with sep, end, and file, if they're defined. Result + # must match expected. - # Look up the actual function to call, based on if sep, end, and file - # are defined + # Look up the actual function to call, based on if sep, end, + # and file are defined. fn = dispatch[(sep is not NotDefined, end is not NotDefined, file is not NotDefined)] @@ -69,7 +62,7 @@ class TestPrint(unittest.TestCase): def test_print(self): def x(expected, args, sep=NotDefined, end=NotDefined): # Run the test 2 ways: not using file, and using - # file directed to a StringIO + # file directed to a StringIO. self.check(expected, args, sep=sep, end=end) @@ -101,11 +94,6 @@ class TestPrint(unittest.TestCase): x('*\n', (ClassWith__str__('*'),)) x('abc 1\n', (ClassWith__str__('abc'), 1)) -# # 2.x unicode tests -# x(u'1 2\n', ('1', u'2')) -# x(u'u\1234\n', (u'u\1234',)) -# x(u' abc 1\n', (' ', ClassWith__str__(u'abc'), 1)) - # errors self.assertRaises(TypeError, print, '', sep=3) self.assertRaises(TypeError, print, '', end=3) @@ -113,12 +101,14 @@ class TestPrint(unittest.TestCase): def test_print_flush(self): # operation of the flush flag - class filelike(): + class filelike: def __init__(self): self.written = '' self.flushed = 0 + def write(self, str): self.written += str + def flush(self): self.flushed += 1 @@ -130,15 +120,13 @@ class TestPrint(unittest.TestCase): self.assertEqual(f.flushed, 2) # ensure exceptions from flush are passed through - class noflush(): + class noflush: def write(self, str): pass + def flush(self): raise RuntimeError self.assertRaises(RuntimeError, print, 1, file=noflush(), flush=True) -def test_main(): - support.run_unittest(TestPrint) - if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_profile.py b/Lib/test/test_profile.py index cd7ec58e23..1fc3c42669 100644 --- a/Lib/test/test_profile.py +++ b/Lib/test/test_profile.py @@ -3,9 +3,11 @@ import sys import pstats import unittest +import os from difflib import unified_diff from io import StringIO -from test.support import run_unittest +from test.support import TESTFN, run_unittest, unlink +from contextlib import contextmanager import profile from test.profilee import testfunc, timer @@ -14,9 +16,13 @@ from test.profilee import testfunc, timer class ProfileTest(unittest.TestCase): profilerclass = profile.Profile + profilermodule = profile methodnames = ['print_stats', 'print_callers', 'print_callees'] expected_max_output = ':0(max)' + def tearDown(self): + unlink(TESTFN) + def get_expected_output(self): return _ProfileOutput @@ -74,6 +80,19 @@ class ProfileTest(unittest.TestCase): self.assertIn(self.expected_max_output, res, "Profiling {0!r} didn't report max:\n{1}".format(stmt, res)) + def test_run(self): + with silent(): + self.profilermodule.run("int('1')") + self.profilermodule.run("int('1')", filename=TESTFN) + self.assertTrue(os.path.exists(TESTFN)) + + def test_runctx(self): + with silent(): + self.profilermodule.runctx("testfunc()", globals(), locals()) + self.profilermodule.runctx("testfunc()", globals(), locals(), + filename=TESTFN) + self.assertTrue(os.path.exists(TESTFN)) + def regenerate_expected_output(filename, cls): filename = filename.rstrip('co') @@ -95,6 +114,14 @@ def regenerate_expected_output(filename, cls): method, results[i+1])) f.write('\nif __name__ == "__main__":\n main()\n') +@contextmanager +def silent(): + stdout = sys.stdout + try: + sys.stdout = StringIO() + yield + finally: + sys.stdout = stdout def test_main(): run_unittest(ProfileTest) diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index 29297f8841..8916861f5b 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -187,7 +187,7 @@ class PtyTest(unittest.TestCase): ##debug("Reading from master_fd now that the child has exited") ##try: ## s1 = os.read(master_fd, 1024) - ##except os.error: + ##except OSError: ## pass ##else: ## raise TestFailed("Read from master_fd did not raise exception") diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index f3c1a6a44b..2ad9c3a000 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -1,7 +1,8 @@ -import imp +import importlib.util import os import py_compile import shutil +import stat import tempfile import unittest @@ -13,7 +14,7 @@ class PyCompileTests(unittest.TestCase): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') self.pyc_path = self.source_path + 'c' - self.cache_path = imp.cache_from_source(self.source_path) + self.cache_path = importlib.util.cache_from_source(self.source_path) self.cwd_drive = os.path.splitdrive(os.getcwd())[0] # In these tests we compute relative paths. When using Windows, the # current working directory path and the 'self.source_path' might be @@ -35,6 +36,26 @@ class PyCompileTests(unittest.TestCase): self.assertTrue(os.path.exists(self.pyc_path)) self.assertFalse(os.path.exists(self.cache_path)) + def test_do_not_overwrite_symlinks(self): + # In the face of a cfile argument being a symlink, bail out. + # Issue #17222 + try: + os.symlink(self.pyc_path + '.actual', self.pyc_path) + except (NotImplementedError, OSError): + self.skipTest('need to be able to create a symlink for a file') + else: + assert os.path.islink(self.pyc_path) + with self.assertRaises(FileExistsError): + py_compile.compile(self.source_path, self.pyc_path) + + @unittest.skipIf(not os.path.exists(os.devnull) or os.path.isfile(os.devnull), + 'requires os.devnull and for it to be a non-regular file') + def test_do_not_overwrite_nonregular_files(self): + # In the face of a cfile argument being a non-regular file, bail out. + # Issue #17222 + with self.assertRaises(FileExistsError): + py_compile.compile(self.source_path, os.devnull) + def test_cache_path(self): py_compile.compile(self.source_path) self.assertTrue(os.path.exists(self.cache_path)) @@ -54,8 +75,20 @@ class PyCompileTests(unittest.TestCase): self.assertTrue(os.path.exists(self.pyc_path)) self.assertFalse(os.path.exists(self.cache_path)) -def test_main(): - support.run_unittest(PyCompileTests) + @unittest.skipIf(os.name == 'nt', + 'cannot control directory permissions on Windows') + def test_exceptions_propagate(self): + # Make sure that exceptions raised thanks to issues with writing + # bytecode. + # http://bugs.python.org/issue17244 + mode = os.stat(self.directory) + os.chmod(self.directory, stat.S_IREAD) + try: + with self.assertRaises(IOError): + py_compile.compile(self.source_path, self.pyc_path) + finally: + os.chmod(self.directory, mode.st_mode) + if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index e83989e2d8..88aff898d4 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -142,7 +142,7 @@ class PyclbrTest(TestCase): self.checkModule('pyclbr') self.checkModule('ast') self.checkModule('doctest', ignore=("TestResults", "_SpoofOut", - "DocTestCase")) + "DocTestCase", '_DocTestSuite')) self.checkModule('difflib', ignore=("Match",)) def test_decorators(self): @@ -158,7 +158,7 @@ class PyclbrTest(TestCase): cm('random', ignore=('Random',)) # from _random import Random as CoreGenerator cm('cgi', ignore=('log',)) # set with = in module cm('pickle') - cm('aifc', ignore=('openfp',)) # set with = in module + cm('aifc', ignore=('openfp', '_aifc_params')) # set with = in module cm('sre_parse', ignore=('dump',)) # from sre_constants import * cm('pdb') cm('pydoc') diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index cdcc707d61..399f818c32 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -29,10 +29,6 @@ try: except ImportError: threading = None -# Just in case sys.modules["test"] has the optional attribute __loader__. -if hasattr(pydoc_mod, "__loader__"): - del pydoc_mod.__loader__ - if test.support.HAVE_DOCSTRINGS: expected_data_docstrings = ( 'dictionary for instance variables (if defined)', @@ -421,6 +417,31 @@ class PydocDocTest(unittest.TestCase): synopsis = pydoc.synopsis(TESTFN, {}) self.assertEqual(synopsis, 'line 1: h\xe9') + def test_splitdoc_with_description(self): + example_string = "I Am A Doc\n\n\nHere is my description" + self.assertEqual(pydoc.splitdoc(example_string), + ('I Am A Doc', '\nHere is my description')) + + def test_is_object_or_method(self): + doc = pydoc.Doc() + # Bound Method + self.assertTrue(pydoc._is_some_method(doc.fail)) + # Method Descriptor + self.assertTrue(pydoc._is_some_method(int.__add__)) + # String + self.assertFalse(pydoc._is_some_method("I am not a method")) + + def test_is_package_when_not_package(self): + with test.support.temp_cwd() as test_dir: + self.assertFalse(pydoc.ispackage(test_dir)) + + def test_is_package_when_is_package(self): + with test.support.temp_cwd() as test_dir: + init_path = os.path.join(test_dir, '__init__.py') + open(init_path, 'w').close() + self.assertTrue(pydoc.ispackage(test_dir)) + os.remove(init_path) + def test_allmethods(self): # issue 17476: allmethods was no longer returning unbound methods. # This test is a bit fragile in the face of changes to object and type, diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index facddb1ebd..49a3f7bb00 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 import unittest +import unittest.mock import random import time import pickle import warnings +from functools import partial from math import log, exp, pi, fsum, sin from test import support @@ -46,6 +48,48 @@ class TestBasicOps: self.assertRaises(TypeError, self.gen.seed, 1, 2, 3, 4) self.assertRaises(TypeError, type(self.gen), []) + @unittest.mock.patch('random._urandom') # os.urandom + def test_seed_when_randomness_source_not_found(self, urandom_mock): + # Random.seed() uses time.time() when an operating system specific + # randomness source is not found. To test this on machines were it + # exists, run the above test, test_seedargs(), again after mocking + # os.urandom() so that it raises the exception expected when the + # randomness source is not available. + urandom_mock.side_effect = NotImplementedError + self.test_seedargs() + + def test_shuffle(self): + shuffle = self.gen.shuffle + lst = [] + shuffle(lst) + self.assertEqual(lst, []) + lst = [37] + shuffle(lst) + self.assertEqual(lst, [37]) + seqs = [list(range(n)) for n in range(10)] + shuffled_seqs = [list(range(n)) for n in range(10)] + for shuffled_seq in shuffled_seqs: + shuffle(shuffled_seq) + for (seq, shuffled_seq) in zip(seqs, shuffled_seqs): + self.assertEqual(len(seq), len(shuffled_seq)) + self.assertEqual(set(seq), set(shuffled_seq)) + # The above tests all would pass if the shuffle was a + # no-op. The following non-deterministic test covers that. It + # asserts that the shuffled sequence of 1000 distinct elements + # must be different from the original one. Although there is + # mathematically a non-zero probability that this could + # actually happen in a genuinely random shuffle, it is + # completely negligible, given that the number of possible + # permutations of 1000 objects is 1000! (factorial of 1000), + # which is considerably larger than the number of atoms in the + # universe... + lst = list(range(1000)) + shuffled_lst = list(range(1000)) + shuffle(shuffled_lst) + self.assertTrue(lst != shuffled_lst) + shuffle(lst) + self.assertTrue(lst != shuffled_lst) + def test_choice(self): choice = self.gen.choice with self.assertRaises(IndexError): @@ -65,6 +109,8 @@ class TestBasicOps: self.assertEqual(len(uniq), k) self.assertTrue(uniq <= set(population)) self.assertEqual(self.gen.sample([], 0), []) # test edge case N==k==0 + # Exception raised if size of sample exceeds that of population + self.assertRaises(ValueError, self.gen.sample, population, N+1) def test_sample_distribution(self): # For the entire allowable range of 0 <= k <= N, validate that @@ -205,6 +251,25 @@ class SystemRandom_TestBasicOps(TestBasicOps, unittest.TestCase): self.assertEqual(set(range(start,stop)), set([self.gen.randrange(start,stop) for i in range(100)])) + def test_randrange_nonunit_step(self): + rint = self.gen.randrange(0, 10, 2) + self.assertIn(rint, (0, 2, 4, 6, 8)) + rint = self.gen.randrange(0, 2, 2) + self.assertEqual(rint, 0) + + def test_randrange_errors(self): + raises = partial(self.assertRaises, ValueError, self.gen.randrange) + # Empty range + raises(3, 3) + raises(-721) + raises(0, 100, -12) + # Non-integer start/stop + raises(3.14159) + raises(0, 2.71828) + # Zero and non-integer step + raises(0, 42, 0) + raises(0, 42, 3.14159) + def test_genrandbits(self): # Verify ranges for k in range(1, 1000): @@ -274,6 +339,16 @@ class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase): # Last element s/b an int also self.assertRaises(TypeError, self.gen.setstate, (2, (0,)*624+('a',), None)) + # Little trick to make "tuple(x % (2**32) for x in internalstate)" + # raise ValueError. I cannot think of a simple way to achieve this, so + # I am opting for using a generator as the middle argument of setstate + # which attempts to cast a NaN to integer. + state_values = self.gen.getstate()[1] + state_values = list(state_values) + state_values[-1] = float('nan') + state = (int(x) for x in state_values) + self.assertRaises(TypeError, self.gen.setstate, (2, state, None)) + def test_referenceImplementation(self): # Compare the python implementation with results from the original # code. Create 2000 53-bit precision random floats. Compare only @@ -413,6 +488,38 @@ class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase): self.assertEqual(k, numbits) # note the stronger assertion self.assertTrue(2**k > n > 2**(k-1)) # note the stronger assertion + @unittest.mock.patch('random.Random.random') + def test_randbelow_overriden_random(self, random_mock): + # Random._randbelow() can only use random() when the built-in one + # has been overridden but no new getrandbits() method was supplied. + random_mock.side_effect = random.SystemRandom().random + maxsize = 1<<random.BPF + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + # Population range too large (n >= maxsize) + self.gen._randbelow(maxsize+1, maxsize = maxsize) + self.gen._randbelow(5640, maxsize = maxsize) + + # This might be going too far to test a single line, but because of our + # noble aim of achieving 100% test coverage we need to write a case in + # which the following line in Random._randbelow() gets executed: + # + # rem = maxsize % n + # limit = (maxsize - rem) / maxsize + # r = random() + # while r >= limit: + # r = random() # <== *This line* <==< + # + # Therefore, to guarantee that the while loop is executed at least + # once, we need to mock random() so that it returns a number greater + # than 'limit' the first time it gets called. + + n = 42 + epsilon = 0.01 + limit = (maxsize - (maxsize % n)) / maxsize + random_mock.side_effect = [limit + epsilon, limit - epsilon] + self.gen._randbelow(n, maxsize = maxsize) + def test_randrange_bug_1590891(self): start = 1000000000000 stop = -100000000000000000000 @@ -530,6 +637,106 @@ class TestDistributions(unittest.TestCase): random.vonmisesvariate(0, 1e15) random.vonmisesvariate(0, 1e100) + def test_gammavariate_errors(self): + # Both alpha and beta must be > 0.0 + self.assertRaises(ValueError, random.gammavariate, -1, 3) + self.assertRaises(ValueError, random.gammavariate, 0, 2) + self.assertRaises(ValueError, random.gammavariate, 2, 0) + self.assertRaises(ValueError, random.gammavariate, 1, -3) + + @unittest.mock.patch('random.Random.random') + def test_gammavariate_full_code_coverage(self, random_mock): + # There are three different possibilities in the current implementation + # of random.gammavariate(), depending on the value of 'alpha'. What we + # are going to do here is to fix the values returned by random() to + # generate test cases that provide 100% line coverage of the method. + + # #1: alpha > 1.0: we want the first random number to be outside the + # [1e-7, .9999999] range, so that the continue statement executes + # once. The values of u1 and u2 will be 0.5 and 0.3, respectively. + random_mock.side_effect = [1e-8, 0.5, 0.3] + returned_value = random.gammavariate(1.1, 2.3) + self.assertAlmostEqual(returned_value, 2.53) + + # #2: alpha == 1: first random number less than 1e-7 to that the body + # of the while loop executes once. Then random.random() returns 0.45, + # which causes while to stop looping and the algorithm to terminate. + random_mock.side_effect = [1e-8, 0.45] + returned_value = random.gammavariate(1.0, 3.14) + self.assertAlmostEqual(returned_value, 2.507314166123803) + + # #3: 0 < alpha < 1. This is the most complex region of code to cover, + # as there are multiple if-else statements. Let's take a look at the + # source code, and determine the values that we need accordingly: + # + # while 1: + # u = random() + # b = (_e + alpha)/_e + # p = b*u + # if p <= 1.0: # <=== (A) + # x = p ** (1.0/alpha) + # else: # <=== (B) + # x = -_log((b-p)/alpha) + # u1 = random() + # if p > 1.0: # <=== (C) + # if u1 <= x ** (alpha - 1.0): # <=== (D) + # break + # elif u1 <= _exp(-x): # <=== (E) + # break + # return x * beta + # + # First, we want (A) to be True. For that we need that: + # b*random() <= 1.0 + # r1 = random() <= 1.0 / b + # + # We now get to the second if-else branch, and here, since p <= 1.0, + # (C) is False and we take the elif branch, (E). For it to be True, + # so that the break is executed, we need that: + # r2 = random() <= _exp(-x) + # r2 <= _exp(-(p ** (1.0/alpha))) + # r2 <= _exp(-((b*r1) ** (1.0/alpha))) + + _e = random._e + _exp = random._exp + _log = random._log + alpha = 0.35 + beta = 1.45 + b = (_e + alpha)/_e + epsilon = 0.01 + + r1 = 0.8859296441566 # 1.0 / b + r2 = 0.3678794411714 # _exp(-((b*r1) ** (1.0/alpha))) + + # These four "random" values result in the following trace: + # (A) True, (E) False --> [next iteration of while] + # (A) True, (E) True --> [while loop breaks] + random_mock.side_effect = [r1, r2 + epsilon, r1, r2] + returned_value = random.gammavariate(alpha, beta) + self.assertAlmostEqual(returned_value, 1.4499999999997544) + + # Let's now make (A) be False. If this is the case, when we get to the + # second if-else 'p' is greater than 1, so (C) evaluates to True. We + # now encounter a second if statement, (D), which in order to execute + # must satisfy the following condition: + # r2 <= x ** (alpha - 1.0) + # r2 <= (-_log((b-p)/alpha)) ** (alpha - 1.0) + # r2 <= (-_log((b-(b*r1))/alpha)) ** (alpha - 1.0) + r1 = 0.8959296441566 # (1.0 / b) + epsilon -- so that (A) is False + r2 = 0.9445400408898141 + + # And these four values result in the following trace: + # (B) and (C) True, (D) False --> [next iteration of while] + # (B) and (C) True, (D) True [while loop breaks] + random_mock.side_effect = [r1, r2 + epsilon, r1, r2] + returned_value = random.gammavariate(alpha, beta) + self.assertAlmostEqual(returned_value, 1.5830349561760781) + + @unittest.mock.patch('random.Random.gammavariate') + def test_betavariate_return_zero(self, gammavariate_mock): + # betavariate() returns zero when the Gamma distribution + # that it uses internally returns this same value. + gammavariate_mock.return_value = 0.0 + self.assertEqual(0.0, random.betavariate(2.71828, 3.14159)) class TestModule(unittest.TestCase): def testMagicConstants(self): diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 2a13bfeabd..f088387c33 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -313,7 +313,7 @@ class RangeTest(unittest.TestCase): self.assertRaises(TypeError, range, IN()) # Test use of user-defined classes in slice indices. - self.assertEqual(list(range(10)[:I(5)]), list(range(5))) + self.assertEqual(range(10)[:I(5)], range(5)) with self.assertRaises(RuntimeError): range(0, 10)[:IX()] diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py new file mode 100644 index 0000000000..a398a4f836 --- /dev/null +++ b/Lib/test/test_regrtest.py @@ -0,0 +1,275 @@ +""" +Tests of regrtest.py. +""" + +import argparse +import faulthandler +import getopt +import os.path +import unittest +from test import regrtest, support + +class ParseArgsTestCase(unittest.TestCase): + + """Test regrtest's argument parsing.""" + + def checkError(self, args, msg): + with support.captured_stderr() as err, self.assertRaises(SystemExit): + regrtest._parse_args(args) + self.assertIn(msg, err.getvalue()) + + def test_help(self): + for opt in '-h', '--help': + with self.subTest(opt=opt): + with support.captured_stdout() as out, \ + self.assertRaises(SystemExit): + regrtest._parse_args([opt]) + self.assertIn('Run Python regression tests.', out.getvalue()) + + @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'), + "faulthandler.dump_traceback_later() required") + def test_timeout(self): + ns = regrtest._parse_args(['--timeout', '4.2']) + self.assertEqual(ns.timeout, 4.2) + self.checkError(['--timeout'], 'expected one argument') + self.checkError(['--timeout', 'foo'], 'invalid float value') + + def test_wait(self): + ns = regrtest._parse_args(['--wait']) + self.assertTrue(ns.wait) + + def test_slaveargs(self): + ns = regrtest._parse_args(['--slaveargs', '[[], {}]']) + self.assertEqual(ns.slaveargs, '[[], {}]') + self.checkError(['--slaveargs'], 'expected one argument') + + def test_start(self): + for opt in '-S', '--start': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt, 'foo']) + self.assertEqual(ns.start, 'foo') + self.checkError([opt], 'expected one argument') + + def test_verbose(self): + ns = regrtest._parse_args(['-v']) + self.assertEqual(ns.verbose, 1) + ns = regrtest._parse_args(['-vvv']) + self.assertEqual(ns.verbose, 3) + ns = regrtest._parse_args(['--verbose']) + self.assertEqual(ns.verbose, 1) + ns = regrtest._parse_args(['--verbose'] * 3) + self.assertEqual(ns.verbose, 3) + ns = regrtest._parse_args([]) + self.assertEqual(ns.verbose, 0) + + def test_verbose2(self): + for opt in '-w', '--verbose2': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt]) + self.assertTrue(ns.verbose2) + + def test_verbose3(self): + for opt in '-W', '--verbose3': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt]) + self.assertTrue(ns.verbose3) + + def test_quiet(self): + for opt in '-q', '--quiet': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt]) + self.assertTrue(ns.quiet) + self.assertEqual(ns.verbose, 0) + + def test_slow(self): + for opt in '-o', '--slow': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt]) + self.assertTrue(ns.print_slow) + + def test_header(self): + ns = regrtest._parse_args(['--header']) + self.assertTrue(ns.header) + + def test_randomize(self): + for opt in '-r', '--randomize': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt]) + self.assertTrue(ns.randomize) + + def test_randseed(self): + ns = regrtest._parse_args(['--randseed', '12345']) + self.assertEqual(ns.random_seed, 12345) + self.assertTrue(ns.randomize) + self.checkError(['--randseed'], 'expected one argument') + self.checkError(['--randseed', 'foo'], 'invalid int value') + + def test_fromfile(self): + for opt in '-f', '--fromfile': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt, 'foo']) + self.assertEqual(ns.fromfile, 'foo') + self.checkError([opt], 'expected one argument') + self.checkError([opt, 'foo', '-s'], "don't go together") + + def test_exclude(self): + for opt in '-x', '--exclude': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt]) + self.assertTrue(ns.exclude) + + def test_single(self): + for opt in '-s', '--single': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt]) + self.assertTrue(ns.single) + self.checkError([opt, '-f', 'foo'], "don't go together") + + def test_match(self): + for opt in '-m', '--match': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt, 'pattern']) + self.assertEqual(ns.match_tests, 'pattern') + self.checkError([opt], 'expected one argument') + + def test_failfast(self): + for opt in '-G', '--failfast': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt, '-v']) + self.assertTrue(ns.failfast) + ns = regrtest._parse_args([opt, '-W']) + self.assertTrue(ns.failfast) + self.checkError([opt], '-G/--failfast needs either -v or -W') + + def test_use(self): + for opt in '-u', '--use': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt, 'gui,network']) + self.assertEqual(ns.use_resources, ['gui', 'network']) + ns = regrtest._parse_args([opt, 'gui,none,network']) + self.assertEqual(ns.use_resources, ['network']) + expected = list(regrtest.RESOURCE_NAMES) + expected.remove('gui') + ns = regrtest._parse_args([opt, 'all,-gui']) + self.assertEqual(ns.use_resources, expected) + self.checkError([opt], 'expected one argument') + self.checkError([opt, 'foo'], 'invalid resource') + + def test_memlimit(self): + for opt in '-M', '--memlimit': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt, '4G']) + self.assertEqual(ns.memlimit, '4G') + self.checkError([opt], 'expected one argument') + + def test_testdir(self): + ns = regrtest._parse_args(['--testdir', 'foo']) + self.assertEqual(ns.testdir, os.path.join(support.SAVEDCWD, 'foo')) + self.checkError(['--testdir'], 'expected one argument') + + def test_runleaks(self): + for opt in '-L', '--runleaks': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt]) + self.assertTrue(ns.runleaks) + + def test_huntrleaks(self): + for opt in '-R', '--huntrleaks': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt, ':']) + self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt')) + ns = regrtest._parse_args([opt, '6:']) + self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt')) + ns = regrtest._parse_args([opt, ':3']) + self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt')) + ns = regrtest._parse_args([opt, '6:3:leaks.log']) + self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log')) + self.checkError([opt], 'expected one argument') + self.checkError([opt, '6'], + 'needs 2 or 3 colon-separated arguments') + self.checkError([opt, 'foo:'], 'invalid huntrleaks value') + self.checkError([opt, '6:foo'], 'invalid huntrleaks value') + + def test_multiprocess(self): + for opt in '-j', '--multiprocess': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt, '2']) + self.assertEqual(ns.use_mp, 2) + self.checkError([opt], 'expected one argument') + self.checkError([opt, 'foo'], 'invalid int value') + self.checkError([opt, '2', '-T'], "don't go together") + self.checkError([opt, '2', '-l'], "don't go together") + self.checkError([opt, '2', '-M', '4G'], "don't go together") + + def test_coverage(self): + for opt in '-T', '--coverage': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt]) + self.assertTrue(ns.trace) + + def test_coverdir(self): + for opt in '-D', '--coverdir': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt, 'foo']) + self.assertEqual(ns.coverdir, + os.path.join(support.SAVEDCWD, 'foo')) + self.checkError([opt], 'expected one argument') + + def test_nocoverdir(self): + for opt in '-N', '--nocoverdir': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt]) + self.assertIsNone(ns.coverdir) + + def test_threshold(self): + for opt in '-t', '--threshold': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt, '1000']) + self.assertEqual(ns.threshold, 1000) + self.checkError([opt], 'expected one argument') + self.checkError([opt, 'foo'], 'invalid int value') + + def test_nowindows(self): + for opt in '-n', '--nowindows': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt]) + self.assertTrue(ns.nowindows) + + def test_forever(self): + for opt in '-F', '--forever': + with self.subTest(opt=opt): + ns = regrtest._parse_args([opt]) + self.assertTrue(ns.forever) + + + def test_unrecognized_argument(self): + self.checkError(['--xxx'], 'usage:') + + def test_long_option__partial(self): + ns = regrtest._parse_args(['--qui']) + self.assertTrue(ns.quiet) + self.assertEqual(ns.verbose, 0) + + def test_two_options(self): + ns = regrtest._parse_args(['--quiet', '--exclude']) + self.assertTrue(ns.quiet) + self.assertEqual(ns.verbose, 0) + self.assertTrue(ns.exclude) + + def test_option_with_empty_string_value(self): + ns = regrtest._parse_args(['--start', '']) + self.assertEqual(ns.start, '') + + def test_arg(self): + ns = regrtest._parse_args(['foo']) + self.assertEqual(ns.args, ['foo']) + + def test_option_and_arg(self): + ns = regrtest._parse_args(['--quiet', 'foo']) + self.assertTrue(ns.quiet) + self.assertEqual(ns.verbose, 0) + self.assertEqual(ns.args, ['foo']) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index 589ecdd4da..104e3b5705 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -3,11 +3,11 @@ Nick Mathewson """ -import imp import sys import os import shutil import importlib +import importlib.util import unittest from test.support import run_unittest, create_empty_file, verbose @@ -241,7 +241,8 @@ class LongReprTest(unittest.TestCase): source_path_len += 2 * (len(self.longname) + 1) # a path separator + `module_name` + ".py" source_path_len += len(module_name) + 1 + len(".py") - cached_path_len = source_path_len + len(imp.cache_from_source("x.py")) - len("x.py") + cached_path_len = (source_path_len + + len(importlib.util.cache_from_source("x.py")) - len("x.py")) if os.name == 'nt' and cached_path_len >= 258: # Under Windows, the max path len is 260 including C's terminating # NUL character. diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py index 0cf61cb17a..1bf2a5a6ce 100644 --- a/Lib/test/test_resource.py +++ b/Lib/test/test_resource.py @@ -61,7 +61,7 @@ class ResourceTest(unittest.TestCase): for i in range(5): time.sleep(.1) f.flush() - except IOError: + except OSError: if not limit_set: raise if limit_set: diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 2ddba3413a..16e2e12090 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -575,12 +575,5 @@ s = "non-ASCII: h\xe9" self.assertEqual(result['s'], "non-ASCII: h\xe9") -def test_main(): - run_unittest( - ExecutionLayerTestCase, - RunModuleTestCase, - RunPathTestCase - ) - if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_sched.py b/Lib/test/test_sched.py index 070886d1ea..c6abf3d89a 100644 --- a/Lib/test/test_sched.py +++ b/Lib/test/test_sched.py @@ -197,8 +197,5 @@ class TestCase(unittest.TestCase): self.assertEqual(l, []) -def test_main(): - support.run_unittest(TestCase) - if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 26ce0424f8..b325545f32 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -715,6 +715,19 @@ class ScopeTests(unittest.TestCase): def b(): global a + def testClassNamespaceOverridesClosure(self): + # See #17853. + x = 42 + class X: + locals()["x"] = 43 + y = x + self.assertEqual(X.y, 43) + class X: + locals()["x"] = 43 + del x + self.assertFalse(hasattr(X, "x")) + self.assertEqual(x, 42) + @cpython_only def testCellLeak(self): # Issue 17927. @@ -743,10 +756,6 @@ class ScopeTests(unittest.TestCase): del tester self.assertIsNone(ref()) - def test__Class__Global(self): - s = "class X:\n global __class__\n def f(self): super()" - self.assertRaises(SyntaxError, exec, s) - def test_main(): run_unittest(ScopeTests) diff --git a/Lib/test/test_select.py b/Lib/test/test_select.py index ddb9a0f67e..8f9a1c9d88 100644 --- a/Lib/test/test_select.py +++ b/Lib/test/test_select.py @@ -5,7 +5,7 @@ import sys import unittest from test import support -@unittest.skipIf(sys.platform[:3] in ('win', 'os2', 'riscos'), +@unittest.skipIf((sys.platform[:3]=='win'), "can't easily test on this system") class SelectTestCase(unittest.TestCase): @@ -32,7 +32,7 @@ class SelectTestCase(unittest.TestCase): fp.close() try: select.select([fd], [], [], 0) - except select.error as err: + except OSError as err: self.assertEqual(err.errno, errno.EBADF) else: self.fail("exception not raised") diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py new file mode 100644 index 0000000000..6ce4d8a216 --- /dev/null +++ b/Lib/test/test_selectors.py @@ -0,0 +1,391 @@ +import errno +import random +import selectors +import signal +import socket +from test import support +from time import sleep +import unittest +try: + from time import monotonic as time +except ImportError: + from time import time as time +try: + import resource +except ImportError: + resource = None + + +if hasattr(socket, 'socketpair'): + socketpair = socket.socketpair +else: + def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): + with socket.socket(family, type, proto) as l: + l.bind((support.HOST, 0)) + l.listen(3) + c = socket.socket(family, type, proto) + try: + c.connect(l.getsockname()) + caddr = c.getsockname() + while True: + a, addr = l.accept() + # check that we've got the correct client + if addr == caddr: + return c, a + a.close() + except OSError: + c.close() + raise + + +def find_ready_matching(ready, flag): + match = [] + for key, events in ready: + if events & flag: + match.append(key.fileobj) + return match + + +class BaseSelectorTestCase(unittest.TestCase): + + def test_register(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertIsInstance(key, selectors.SelectorKey) + self.assertEqual(key.fileobj, rd) + self.assertEqual(key.fd, rd.fileno()) + self.assertEqual(key.events, selectors.EVENT_READ) + self.assertEqual(key.data, "data") + + # register an unknown event + self.assertRaises(ValueError, s.register, 0, 999999) + + # register an invalid FD + self.assertRaises(ValueError, s.register, -10, selectors.EVENT_READ) + + # register twice + self.assertRaises(KeyError, s.register, rd, selectors.EVENT_READ) + + # register the same FD, but with a different object + self.assertRaises(KeyError, s.register, rd.fileno(), + selectors.EVENT_READ) + + def test_unregister(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + s.register(rd, selectors.EVENT_READ) + s.unregister(rd) + + # unregister an unknown file obj + self.assertRaises(KeyError, s.unregister, 999999) + + # unregister twice + self.assertRaises(KeyError, s.unregister, rd) + + def test_modify(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + key = s.register(rd, selectors.EVENT_READ) + + # modify events + key2 = s.modify(rd, selectors.EVENT_WRITE) + self.assertNotEqual(key.events, key2.events) + self.assertEqual(key2, s.get_key(rd)) + + s.unregister(rd) + + # modify data + d1 = object() + d2 = object() + + key = s.register(rd, selectors.EVENT_READ, d1) + key2 = s.modify(rd, selectors.EVENT_READ, d2) + self.assertEqual(key.events, key2.events) + self.assertNotEqual(key.data, key2.data) + self.assertEqual(key2, s.get_key(rd)) + self.assertEqual(key2.data, d2) + + # modify unknown file obj + self.assertRaises(KeyError, s.modify, 999999, selectors.EVENT_READ) + + def test_close(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + + s.close() + self.assertRaises(KeyError, s.get_key, rd) + self.assertRaises(KeyError, s.get_key, wr) + + def test_get_key(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertEqual(key, s.get_key(rd)) + + # unknown file obj + self.assertRaises(KeyError, s.get_key, 999999) + + def test_select(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + s.register(rd, selectors.EVENT_READ) + wr_key = s.register(wr, selectors.EVENT_WRITE) + + result = s.select() + for key, events in result: + self.assertTrue(isinstance(key, selectors.SelectorKey)) + self.assertTrue(events) + self.assertFalse(events & ~(selectors.EVENT_READ | + selectors.EVENT_WRITE)) + + self.assertEqual([(wr_key, selectors.EVENT_WRITE)], result) + + def test_context_manager(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + with s as sel: + sel.register(rd, selectors.EVENT_READ) + sel.register(wr, selectors.EVENT_WRITE) + + self.assertRaises(KeyError, s.get_key, rd) + self.assertRaises(KeyError, s.get_key, wr) + + def test_fileno(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + if hasattr(s, 'fileno'): + fd = s.fileno() + self.assertTrue(isinstance(fd, int)) + self.assertGreaterEqual(fd, 0) + + def test_selector(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + NUM_SOCKETS = 12 + MSG = b" This is a test." + MSG_LEN = len(MSG) + readers = [] + writers = [] + r2w = {} + w2r = {} + + for i in range(NUM_SOCKETS): + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + readers.append(rd) + writers.append(wr) + r2w[rd] = wr + w2r[wr] = rd + + bufs = [] + + while writers: + ready = s.select() + ready_writers = find_ready_matching(ready, selectors.EVENT_WRITE) + if not ready_writers: + self.fail("no sockets ready for writing") + wr = random.choice(ready_writers) + wr.send(MSG) + + for i in range(10): + ready = s.select() + ready_readers = find_ready_matching(ready, + selectors.EVENT_READ) + if ready_readers: + break + # there might be a delay between the write to the write end and + # the read end is reported ready + sleep(0.1) + else: + self.fail("no sockets ready for reading") + self.assertEqual([w2r[wr]], ready_readers) + rd = ready_readers[0] + buf = rd.recv(MSG_LEN) + self.assertEqual(len(buf), MSG_LEN) + bufs.append(buf) + s.unregister(r2w[rd]) + s.unregister(rd) + writers.remove(r2w[rd]) + + self.assertEqual(bufs, [MSG] * NUM_SOCKETS) + + def test_timeout(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + s.register(wr, selectors.EVENT_WRITE) + t = time() + self.assertEqual(1, len(s.select(0))) + self.assertEqual(1, len(s.select(-1))) + self.assertTrue(time() - t < 0.5) + + s.unregister(wr) + s.register(rd, selectors.EVENT_READ) + t = time() + self.assertFalse(s.select(0)) + self.assertFalse(s.select(-1)) + self.assertTrue(time() - t < 0.5) + + t = time() + self.assertFalse(s.select(1)) + self.assertTrue(0.5 < time() - t < 1.5) + + @unittest.skipUnless(hasattr(signal, "alarm"), + "signal.alarm() required for this test") + def test_interrupted_retry(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + orig_alrm_handler = signal.signal(signal.SIGALRM, lambda *args: None) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + self.addCleanup(signal.alarm, 0) + + signal.alarm(1) + + s.register(rd, selectors.EVENT_READ) + t = time() + self.assertFalse(s.select(2)) + self.assertLess(time() - t, 2.5) + + +class ScalableSelectorMixIn: + + # see issue #18963 for why it's skipped on older OS X versions + @support.requires_mac_ver(10, 5) + @unittest.skipUnless(resource, "Test needs resource module") + def test_above_fd_setsize(self): + # A scalable implementation should have no problem with more than + # FD_SETSIZE file descriptors. Since we don't know the value, we just + # try to set the soft RLIMIT_NOFILE to the hard RLIMIT_NOFILE ceiling. + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard)) + self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, + (soft, hard)) + NUM_FDS = hard + except (OSError, ValueError): + NUM_FDS = soft + + # guard for already allocated FDs (stdin, stdout...) + NUM_FDS -= 32 + + s = self.SELECTOR() + self.addCleanup(s.close) + + for i in range(NUM_FDS // 2): + try: + rd, wr = socketpair() + except OSError: + # too many FDs, skip - note that we should only catch EMFILE + # here, but apparently *BSD and Solaris can fail upon connect() + # or bind() with EADDRNOTAVAIL, so let's be safe + self.skipTest("FD limit reached") + + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + try: + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + except OSError as e: + if e.errno == errno.ENOSPC: + # this can be raised by epoll if we go over + # fs.epoll.max_user_watches sysctl + self.skipTest("FD limit reached") + raise + + self.assertEqual(NUM_FDS // 2, len(s.select())) + + +class DefaultSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = selectors.DefaultSelector + + +class SelectSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = selectors.SelectSelector + + +@unittest.skipUnless(hasattr(selectors, 'PollSelector'), + "Test needs selectors.PollSelector") +class PollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'PollSelector', None) + + +@unittest.skipUnless(hasattr(selectors, 'EpollSelector'), + "Test needs selectors.EpollSelector") +class EpollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'EpollSelector', None) + + +@unittest.skipUnless(hasattr(selectors, 'KqueueSelector'), + "Test needs selectors.KqueueSelector)") +class KqueueSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'KqueueSelector', None) + + +def test_main(): + tests = [DefaultSelectorTestCase, SelectSelectorTestCase, + PollSelectorTestCase, EpollSelectorTestCase, + KqueueSelectorTestCase] + support.run_unittest(*tests) + support.reap_children() + + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index d02e42759a..13cefec1a9 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -848,8 +848,6 @@ class TestBasicOps: for v in self.set: self.assertIn(v, self.values) setiter = iter(self.set) - # note: __length_hint__ is an internal undocumented API, - # don't rely on it in your own programs self.assertEqual(setiter.__length_hint__(), len(self.set)) def test_pickling(self): diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 13c126566d..bd51d868fe 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -148,6 +148,19 @@ class TestCase(unittest.TestCase): p2 = d[encodedkey] self.assertNotEqual(p1, p2) # Write creates new object in store + def test_with(self): + d1 = {} + with shelve.Shelf(d1, protocol=2, writeback=False) as s: + s['key1'] = [1,2,3,4] + self.assertEqual(s['key1'], [1,2,3,4]) + self.assertEqual(len(s), 1) + self.assertRaises(ValueError, len, s) + try: + s['key1'] + except ValueError: + pass + else: + self.fail('Closed shelf should not find a key') from test import mapping_tests diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 9af7da71cb..0fbe078dbf 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -18,7 +18,8 @@ from shutil import (_make_tarball, _make_zipfile, make_archive, register_archive_format, unregister_archive_format, get_archive_formats, Error, unpack_archive, register_unpack_format, RegistryError, - unregister_unpack_format, get_unpack_formats) + unregister_unpack_format, get_unpack_formats, + SameFileError) import tarfile import warnings @@ -742,14 +743,14 @@ class TestShutil(unittest.TestCase): os.chmod(restrictive_subdir, 0o600) shutil.copytree(src_dir, dst_dir) - self.assertEquals(os.stat(src_dir).st_mode, os.stat(dst_dir).st_mode) - self.assertEquals(os.stat(os.path.join(src_dir, 'permissive.txt')).st_mode, + self.assertEqual(os.stat(src_dir).st_mode, os.stat(dst_dir).st_mode) + self.assertEqual(os.stat(os.path.join(src_dir, 'permissive.txt')).st_mode, os.stat(os.path.join(dst_dir, 'permissive.txt')).st_mode) - self.assertEquals(os.stat(os.path.join(src_dir, 'restrictive.txt')).st_mode, + self.assertEqual(os.stat(os.path.join(src_dir, 'restrictive.txt')).st_mode, os.stat(os.path.join(dst_dir, 'restrictive.txt')).st_mode) restrictive_subdir_dst = os.path.join(dst_dir, os.path.split(restrictive_subdir)[1]) - self.assertEquals(os.stat(restrictive_subdir).st_mode, + self.assertEqual(os.stat(restrictive_subdir).st_mode, os.stat(restrictive_subdir_dst).st_mode) @unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') @@ -765,7 +766,7 @@ class TestShutil(unittest.TestCase): with open(src, 'w') as f: f.write('cheddar') os.link(src, dst) - self.assertRaises(shutil.Error, shutil.copyfile, src, dst) + self.assertRaises(shutil.SameFileError, shutil.copyfile, src, dst) with open(src, 'r') as f: self.assertEqual(f.read(), 'cheddar') os.remove(dst) @@ -785,7 +786,7 @@ class TestShutil(unittest.TestCase): # to TESTFN/TESTFN/cheese, while it should point at # TESTFN/cheese. os.symlink('cheese', dst) - self.assertRaises(shutil.Error, shutil.copyfile, src, dst) + self.assertRaises(shutil.SameFileError, shutil.copyfile, src, dst) with open(src, 'r') as f: self.assertEqual(f.read(), 'cheddar') os.remove(dst) @@ -1292,6 +1293,16 @@ class TestShutil(unittest.TestCase): self.assertTrue(os.path.exists(rv)) self.assertEqual(read_file(src_file), read_file(dst_file)) + def test_copyfile_same_file(self): + # copyfile() should raise SameFileError if the source and destination + # are the same. + src_dir = self.mkdtemp() + src_file = os.path.join(src_dir, 'foo') + write_file(src_file, 'foo') + self.assertRaises(SameFileError, shutil.copyfile, src_file, src_file) + # But Error should work too, to stay backward compatible. + self.assertRaises(Error, shutil.copyfile, src_file, src_file) + def test_copytree_return_value(self): # copytree returns its destination path. src_dir = self.mkdtemp() @@ -1587,7 +1598,7 @@ class TestCopyFile(unittest.TestCase): self._exited_with = exc_type, exc_val, exc_tb if self._raise_in_exit: self._raised = True - raise IOError("Cannot close") + raise OSError("Cannot close") return self._suppress_at_exit def tearDown(self): @@ -1601,12 +1612,12 @@ class TestCopyFile(unittest.TestCase): def test_w_source_open_fails(self): def _open(filename, mode='r'): if filename == 'srcfile': - raise IOError('Cannot open "srcfile"') + raise OSError('Cannot open "srcfile"') assert 0 # shouldn't reach here. self._set_shutil_open(_open) - self.assertRaises(IOError, shutil.copyfile, 'srcfile', 'destfile') + self.assertRaises(OSError, shutil.copyfile, 'srcfile', 'destfile') def test_w_dest_open_fails(self): @@ -1616,14 +1627,14 @@ class TestCopyFile(unittest.TestCase): if filename == 'srcfile': return srcfile if filename == 'destfile': - raise IOError('Cannot open "destfile"') + raise OSError('Cannot open "destfile"') assert 0 # shouldn't reach here. self._set_shutil_open(_open) shutil.copyfile('srcfile', 'destfile') self.assertTrue(srcfile._entered) - self.assertTrue(srcfile._exited_with[0] is IOError) + self.assertTrue(srcfile._exited_with[0] is OSError) self.assertEqual(srcfile._exited_with[1].args, ('Cannot open "destfile"',)) @@ -1645,7 +1656,7 @@ class TestCopyFile(unittest.TestCase): self.assertTrue(srcfile._entered) self.assertTrue(destfile._entered) self.assertTrue(destfile._raised) - self.assertTrue(srcfile._exited_with[0] is IOError) + self.assertTrue(srcfile._exited_with[0] is OSError) self.assertEqual(srcfile._exited_with[1].args, ('Cannot close',)) @@ -1663,7 +1674,7 @@ class TestCopyFile(unittest.TestCase): self._set_shutil_open(_open) - self.assertRaises(IOError, + self.assertRaises(OSError, shutil.copyfile, 'srcfile', 'destfile') self.assertTrue(srcfile._entered) self.assertTrue(destfile._entered) @@ -1734,9 +1745,5 @@ class TermsizeTests(unittest.TestCase): self.assertEqual(expected, actual) -def test_main(): - support.run_unittest(TestShutil, TestMove, TestCopyFile, - TermsizeTests, TestWhich) - if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 9b4ba5016e..3530d8a14e 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -15,9 +15,6 @@ try: except ImportError: threading = None -if sys.platform in ('os2', 'riscos'): - raise unittest.SkipTest("Can't test signal on %s" % sys.platform) - class HandlerBCalled(Exception): pass @@ -36,7 +33,7 @@ def exit_subprocess(): def ignoring_eintr(__func, *args, **kwargs): try: return __func(*args, **kwargs) - except EnvironmentError as e: + except OSError as e: if e.errno != errno.EINTR: raise return None @@ -278,6 +275,57 @@ class WakeupSignalTests(unittest.TestCase): assert_python_ok('-c', code) + def test_wakeup_write_error(self): + # Issue #16105: write() errors in the C signal handler should not + # pass silently. + # Use a subprocess to have only one thread. + code = """if 1: + import errno + import fcntl + import os + import signal + import sys + import time + from test.support import captured_stderr + + def handler(signum, frame): + 1/0 + + signal.signal(signal.SIGALRM, handler) + r, w = os.pipe() + flags = fcntl.fcntl(r, fcntl.F_GETFL, 0) + fcntl.fcntl(r, fcntl.F_SETFL, flags | os.O_NONBLOCK) + + # Set wakeup_fd a read-only file descriptor to trigger the error + signal.set_wakeup_fd(r) + try: + with captured_stderr() as err: + signal.alarm(1) + time.sleep(5.0) + except ZeroDivisionError: + # An ignored exception should have been printed out on stderr + err = err.getvalue() + if ('Exception ignored when trying to write to the signal wakeup fd' + not in err): + raise AssertionError(err) + if ('OSError: [Errno %d]' % errno.EBADF) not in err: + raise AssertionError(err) + else: + raise AssertionError("ZeroDivisionError not raised") + """ + r, w = os.pipe() + try: + os.write(r, b'x') + except OSError: + pass + else: + self.skipTest("OS doesn't report write() error on the read end of a pipe") + finally: + os.close(r) + os.close(w) + + assert_python_ok('-c', code) + def test_wakeup_fd_early(self): self.check_wakeup("""def test(): import select @@ -315,10 +363,10 @@ class WakeupSignalTests(unittest.TestCase): # We attempt to get a signal during the select call try: select.select([read], [], [], TIMEOUT_FULL) - except select.error: + except OSError: pass else: - raise Exception("select.error not raised") + raise Exception("OSError not raised") after_time = time.time() dt = after_time - before_time if dt >= TIMEOUT_HALF: diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 16081a1e1e..3ada0a671c 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -225,11 +225,7 @@ class HelperFunctionsTests(unittest.TestCase): site.PREFIXES = ['xoxo'] dirs = site.getsitepackages() - if sys.platform in ('os2emx', 'riscos'): - self.assertEqual(len(dirs), 1) - wanted = os.path.join('xoxo', 'Lib', 'site-packages') - self.assertEqual(dirs[0], wanted) - elif (sys.platform == "darwin" and + if (sys.platform == "darwin" and sysconfig.get_config_var("PYTHONFRAMEWORK")): # OS X framework builds site.PREFIXES = ['Python.framework'] @@ -424,5 +420,37 @@ class ImportSideEffectTests(unittest.TestCase): self.assertEqual(code, 200, msg="Can't find " + url) +class StartupImportTests(unittest.TestCase): + + def test_startup_imports(self): + # This tests checks which modules are loaded by Python when it + # initially starts upon startup. + popen = subprocess.Popen([sys.executable, '-I', '-v', '-c', + 'import sys; print(set(sys.modules))'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = popen.communicate() + stdout = stdout.decode('utf-8') + stderr = stderr.decode('utf-8') + modules = eval(stdout) + + self.assertIn('site', modules) + + # http://bugs.python.org/issue19205 + re_mods = {'re', '_sre', 'sre_compile', 'sre_constants', 'sre_parse'} + # _osx_support uses the re module in many placs + if sys.platform != 'darwin': + self.assertFalse(modules.intersection(re_mods), stderr) + # http://bugs.python.org/issue9548 + self.assertNotIn('locale', modules, stderr) + # http://bugs.python.org/issue19209 + self.assertNotIn('copyreg', modules, stderr) + # http://bugs.python.org/issue19218> + collection_mods = {'_collections', 'collections', 'functools', + 'heapq', 'itertools', 'keyword', 'operator', + 'reprlib', 'types', 'weakref'} + self.assertFalse(modules.intersection(re_mods), stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_slice.py b/Lib/test/test_slice.py index 2df9271da7..9203d5eb19 100644 --- a/Lib/test/test_slice.py +++ b/Lib/test/test_slice.py @@ -4,8 +4,70 @@ import unittest from test import support from pickle import loads, dumps +import itertools +import operator import sys + +def evaluate_slice_index(arg): + """ + Helper function to convert a slice argument to an integer, and raise + TypeError with a suitable message on failure. + + """ + if hasattr(arg, '__index__'): + return operator.index(arg) + else: + raise TypeError( + "slice indices must be integers or " + "None or have an __index__ method") + +def slice_indices(slice, length): + """ + Reference implementation for the slice.indices method. + + """ + # Compute step and length as integers. + length = operator.index(length) + step = 1 if slice.step is None else evaluate_slice_index(slice.step) + + # Raise ValueError for negative length or zero step. + if length < 0: + raise ValueError("length should not be negative") + if step == 0: + raise ValueError("slice step cannot be zero") + + # Find lower and upper bounds for start and stop. + lower = -1 if step < 0 else 0 + upper = length - 1 if step < 0 else length + + # Compute start. + if slice.start is None: + start = upper if step < 0 else lower + else: + start = evaluate_slice_index(slice.start) + start = max(start + length, lower) if start < 0 else min(start, upper) + + # Compute stop. + if slice.stop is None: + stop = lower if step < 0 else upper + else: + stop = evaluate_slice_index(slice.stop) + stop = max(stop + length, lower) if stop < 0 else min(stop, upper) + + return start, stop, step + + +# Class providing an __index__ method. Used for testing slice.indices. + +class MyIndexable(object): + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + + class SliceTest(unittest.TestCase): def test_constructor(self): @@ -75,6 +137,22 @@ class SliceTest(unittest.TestCase): s = slice(obj) self.assertTrue(s.stop is obj) + def check_indices(self, slice, length): + try: + actual = slice.indices(length) + except ValueError: + actual = "valueerror" + try: + expected = slice_indices(slice, length) + except ValueError: + expected = "valueerror" + self.assertEqual(actual, expected) + + if length >= 0 and slice.step != 0: + actual = range(*slice.indices(length)) + expected = range(length)[slice] + self.assertEqual(actual, expected) + def test_indices(self): self.assertEqual(slice(None ).indices(10), (0, 10, 1)) self.assertEqual(slice(None, None, 2).indices(10), (0, 10, 2)) @@ -108,7 +186,41 @@ class SliceTest(unittest.TestCase): self.assertEqual(list(range(10))[::sys.maxsize - 1], [0]) - self.assertRaises(OverflowError, slice(None).indices, 1<<100) + # Check a variety of start, stop, step and length values, including + # values exceeding sys.maxsize (see issue #14794). + vals = [None, -2**100, -2**30, -53, -7, -1, 0, 1, 7, 53, 2**30, 2**100] + lengths = [0, 1, 7, 53, 2**30, 2**100] + for slice_args in itertools.product(vals, repeat=3): + s = slice(*slice_args) + for length in lengths: + self.check_indices(s, length) + self.check_indices(slice(0, 10, 1), -3) + + # Negative length should raise ValueError + with self.assertRaises(ValueError): + slice(None).indices(-1) + + # Zero step should raise ValueError + with self.assertRaises(ValueError): + slice(0, 10, 0).indices(5) + + # Using a start, stop or step or length that can't be interpreted as an + # integer should give a TypeError ... + with self.assertRaises(TypeError): + slice(0.0, 10, 1).indices(5) + with self.assertRaises(TypeError): + slice(0, 10.0, 1).indices(5) + with self.assertRaises(TypeError): + slice(0, 10, 1.0).indices(5) + with self.assertRaises(TypeError): + slice(0, 10, 1).indices(5.0) + + # ... but it should be fine to use a custom class that provides index. + self.assertEqual(slice(0, 10, 1).indices(5), (0, 5, 1)) + self.assertEqual(slice(MyIndexable(0), 10, 1).indices(5), (0, 5, 1)) + self.assertEqual(slice(0, MyIndexable(10), 1).indices(5), (0, 5, 1)) + self.assertEqual(slice(0, 10, MyIndexable(1)).indices(5), (0, 5, 1)) + self.assertEqual(slice(0, 10, 1).indices(MyIndexable(5)), (0, 5, 1)) def test_setslice_without_getslice(self): tmp = [] diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 8d1dbbfc43..a501f40b56 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -222,7 +222,7 @@ class DebuggingServerTests(unittest.TestCase): self.assertEqual(smtp.source_address, ('127.0.0.1', port)) self.assertEqual(smtp.local_hostname, 'localhost') smtp.quit() - except IOError as e: + except OSError as e: if e.errno == errno.EADDRINUSE: self.skipTest("couldn't bind to port %d" % port) raise @@ -524,12 +524,6 @@ class DebuggingServerTests(unittest.TestCase): class NonConnectingTests(unittest.TestCase): - def setUp(self): - smtplib.socket = mock_socket - - def tearDown(self): - smtplib.socket = socket - def testNotConnected(self): # Test various operations on an unconnected SMTP object that # should raise exceptions (at present the attempt in SMTP.send @@ -541,10 +535,10 @@ class NonConnectingTests(unittest.TestCase): smtp.send, 'test msg') def testNonnumericPort(self): - # check that non-numeric port raises socket.error - self.assertRaises(mock_socket.error, smtplib.SMTP, + # check that non-numeric port raises OSError + self.assertRaises(OSError, smtplib.SMTP, "localhost", "bogus") - self.assertRaises(mock_socket.error, smtplib.SMTP, + self.assertRaises(OSError, smtplib.SMTP, "localhost:bogus") diff --git a/Lib/test/test_sndhdr.py b/Lib/test/test_sndhdr.py index 10046887d7..5e0abe0b36 100644 --- a/Lib/test/test_sndhdr.py +++ b/Lib/test/test_sndhdr.py @@ -12,7 +12,7 @@ class TestFormats(unittest.TestCase): ('sndhdr.hcom', ('hcom', 22050.0, 1, -1, 8)), ('sndhdr.sndt', ('sndt', 44100, 1, 5, 8)), ('sndhdr.voc', ('voc', 0, 1, -1, 8)), - ('sndhdr.wav', ('wav', 44100, 2, -1, 16)), + ('sndhdr.wav', ('wav', 44100, 2, 5, 16)), ): filename = findfile(filename, subdir="sndhdrdata") what = sndhdr.what(filename) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index b95e1abafe..e995c99df1 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -2,7 +2,6 @@ import unittest from test import support -from unittest.case import _ExpectedFailure import errno import io @@ -24,13 +23,13 @@ import math import pickle import struct try: - import fcntl -except ImportError: - fcntl = False -try: import multiprocessing except ImportError: multiprocessing = False +try: + import fcntl +except ImportError: + fcntl = None HOST = support.HOST MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return @@ -46,7 +45,7 @@ def _have_socket_can(): """Check whether CAN sockets are supported on this host.""" try: s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) - except (AttributeError, socket.error, OSError): + except (AttributeError, OSError): return False else: s.close() @@ -121,12 +120,42 @@ class SocketCANTest(unittest.TestCase): interface = 'vcan0' bufsize = 128 + """The CAN frame structure is defined in <linux/can.h>: + + struct can_frame { + canid_t can_id; /* 32 bit CAN_ID + EFF/RTR/ERR flags */ + __u8 can_dlc; /* data length code: 0 .. 8 */ + __u8 data[8] __attribute__((aligned(8))); + }; + """ + can_frame_fmt = "=IB3x8s" + can_frame_size = struct.calcsize(can_frame_fmt) + + """The Broadcast Management Command frame structure is defined + in <linux/can/bcm.h>: + + struct bcm_msg_head { + __u32 opcode; + __u32 flags; + __u32 count; + struct timeval ival1, ival2; + canid_t can_id; + __u32 nframes; + struct can_frame frames[0]; + } + + `bcm_msg_head` must be 8 bytes aligned because of the `frames` member (see + `struct can_frame` definition). Must use native not standard types for packing. + """ + bcm_cmd_msg_fmt = "@3I4l2I" + bcm_cmd_msg_fmt += "x" * (struct.calcsize(bcm_cmd_msg_fmt) % 8) + def setUp(self): self.s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) self.addCleanup(self.s.close) try: self.s.bind((self.interface,)) - except socket.error: + except OSError: self.skipTest('network interface `%s` does not exist' % self.interface) @@ -242,9 +271,6 @@ class ThreadableTest: raise TypeError("test_func must be a callable function") try: test_func() - except _ExpectedFailure: - # We deliberately ignore expected failures - pass except BaseException as e: self.queue.put(e) finally: @@ -295,7 +321,7 @@ class ThreadedCANSocketTest(SocketCANTest, ThreadableTest): self.cli = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) try: self.cli.bind((self.interface,)) - except socket.error: + except OSError: # skipTest should not be called here, and will be called in the # server instead pass @@ -543,11 +569,7 @@ class SCTPStreamBase(InetTestBase): class Inet6TestBase(InetTestBase): """Base class for IPv6 socket tests.""" - # Don't use "localhost" here - it may not have an IPv6 address - # assigned to it by default (e.g. in /etc/hosts), and if someone - # has assigned it an IPv4-mapped address, then it's unlikely to - # work with the full IPv6 API. - host = "::1" + host = support.HOSTv6 class UDP6TestBase(Inet6TestBase): """Base class for UDP-over-IPv6 tests.""" @@ -608,7 +630,7 @@ def requireSocket(*args): for obj in args] try: s = socket.socket(*callargs) - except socket.error as e: + except OSError as e: # XXX: check errno? err = str(e) else: @@ -626,8 +648,17 @@ class GeneralModuleTests(unittest.TestCase): def test_repr(self): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.addCleanup(s.close) - self.assertTrue(repr(s).startswith("<socket.socket object")) + with s: + self.assertIn('fd=%i' % s.fileno(), repr(s)) + self.assertIn('family=%i' % socket.AF_INET, repr(s)) + self.assertIn('type=%i' % socket.SOCK_STREAM, repr(s)) + self.assertIn('proto=0', repr(s)) + self.assertNotIn('raddr', repr(s)) + s.bind(('127.0.0.1', 0)) + self.assertIn('laddr', repr(s)) + self.assertIn(str(s.getsockname()), repr(s)) + self.assertIn('[closed]', repr(s)) + self.assertNotIn('laddr', repr(s)) def test_weakref(self): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -645,11 +676,11 @@ class GeneralModuleTests(unittest.TestCase): def testSocketError(self): # Testing socket module exceptions msg = "Error raising socket exception (%s)." - with self.assertRaises(socket.error, msg=msg % 'socket.error'): - raise socket.error - with self.assertRaises(socket.error, msg=msg % 'socket.herror'): + with self.assertRaises(OSError, msg=msg % 'OSError'): + raise OSError + with self.assertRaises(OSError, msg=msg % 'socket.herror'): raise socket.herror - with self.assertRaises(socket.error, msg=msg % 'socket.gaierror'): + with self.assertRaises(OSError, msg=msg % 'socket.gaierror'): raise socket.gaierror def testSendtoErrors(self): @@ -712,13 +743,13 @@ class GeneralModuleTests(unittest.TestCase): hostname = socket.gethostname() try: ip = socket.gethostbyname(hostname) - except socket.error: + except OSError: # Probably name lookup wasn't set up right; skip this test return self.assertTrue(ip.find('.') >= 0, "Error resolving host to ip.") try: hname, aliases, ipaddrs = socket.gethostbyaddr(ip) - except socket.error: + except OSError: # Probably a similar problem as above; skip this test return all_host_names = [hostname, hname] + aliases @@ -726,13 +757,27 @@ class GeneralModuleTests(unittest.TestCase): if not fqhn in all_host_names: self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) + def test_host_resolution(self): + for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', + '1:1:1:1:1:1:1:1:1']: + self.assertRaises(OSError, socket.gethostbyname, addr) + self.assertRaises(OSError, socket.gethostbyaddr, addr) + + for addr in [support.HOST, '10.0.0.1', '255.255.255.255']: + self.assertEqual(socket.gethostbyname(addr), addr) + + # we don't test support.HOSTv6 because there's a chance it doesn't have + # a matching name entry (e.g. 'ip6-localhost') + for host in [support.HOST]: + self.assertIn(host, socket.gethostbyaddr(host)[2]) + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") def test_sethostname(self): oldhn = socket.gethostname() try: socket.sethostname('new') - except socket.error as e: + except OSError as e: if e.errno == errno.EPERM: self.skipTest("test should be run as root") else: @@ -766,8 +811,8 @@ class GeneralModuleTests(unittest.TestCase): 'socket.if_nameindex() not available.') def testInvalidInterfaceNameIndex(self): # test nonexistent interface index/name - self.assertRaises(socket.error, socket.if_indextoname, 0) - self.assertRaises(socket.error, socket.if_nametoindex, '_DEADBEEF') + self.assertRaises(OSError, socket.if_indextoname, 0) + self.assertRaises(OSError, socket.if_nametoindex, '_DEADBEEF') # test with invalid values self.assertRaises(TypeError, socket.if_nametoindex, 0) self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') @@ -788,7 +833,7 @@ class GeneralModuleTests(unittest.TestCase): try: # On some versions, this crashes the interpreter. socket.getnameinfo(('x', 0, 0, 0), 0) - except socket.error: + except OSError: pass def testNtoH(self): @@ -835,17 +880,17 @@ class GeneralModuleTests(unittest.TestCase): try: port = socket.getservbyname(service, 'tcp') break - except socket.error: + except OSError: pass else: - raise socket.error + raise OSError # Try same call with optional protocol omitted port2 = socket.getservbyname(service) eq(port, port2) # Try udp, but don't barf if it doesn't exist try: udpport = socket.getservbyname(service, 'udp') - except socket.error: + except OSError: udpport = None else: eq(udpport, port) @@ -901,7 +946,7 @@ class GeneralModuleTests(unittest.TestCase): g = lambda a: inet_pton(AF_INET, a) assertInvalid = lambda func,a: self.assertRaises( - (socket.error, ValueError), func, a + (OSError, ValueError), func, a ) self.assertEqual(b'\x00\x00\x00\x00', f('0.0.0.0')) @@ -936,7 +981,7 @@ class GeneralModuleTests(unittest.TestCase): return f = lambda a: inet_pton(AF_INET6, a) assertInvalid = lambda a: self.assertRaises( - (socket.error, ValueError), f, a + (OSError, ValueError), f, a ) self.assertEqual(b'\x00' * 16, f('::')) @@ -985,7 +1030,7 @@ class GeneralModuleTests(unittest.TestCase): from socket import inet_ntoa as f, inet_ntop, AF_INET g = lambda a: inet_ntop(AF_INET, a) assertInvalid = lambda func,a: self.assertRaises( - (socket.error, ValueError), func, a + (OSError, ValueError), func, a ) self.assertEqual('1.0.1.0', f(b'\x01\x00\x01\x00')) @@ -1014,7 +1059,7 @@ class GeneralModuleTests(unittest.TestCase): return f = lambda a: inet_ntop(AF_INET6, a) assertInvalid = lambda a: self.assertRaises( - (socket.error, ValueError), f, a + (OSError, ValueError), f, a ) self.assertEqual('::', f(b'\x00' * 16)) @@ -1042,7 +1087,7 @@ class GeneralModuleTests(unittest.TestCase): # At least for eCos. This is required for the S/390 to pass. try: my_ip_addr = socket.gethostbyname(socket.gethostname()) - except socket.error: + except OSError: # Probably name lookup wasn't set up right; skip this test return self.assertIn(name[0], ("0.0.0.0", my_ip_addr), '%s invalid' % name[0]) @@ -1069,13 +1114,19 @@ class GeneralModuleTests(unittest.TestCase): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(1) sock.close() - self.assertRaises(socket.error, sock.send, b"spam") + self.assertRaises(OSError, sock.send, b"spam") def testNewAttributes(self): # testing .family, .type and .protocol + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.assertEqual(sock.family, socket.AF_INET) - self.assertEqual(sock.type, socket.SOCK_STREAM) + if hasattr(socket, 'SOCK_CLOEXEC'): + self.assertIn(sock.type, + (socket.SOCK_STREAM | socket.SOCK_CLOEXEC, + socket.SOCK_STREAM)) + else: + self.assertEqual(sock.type, socket.SOCK_STREAM) self.assertEqual(sock.proto, 0) sock.close() @@ -1128,9 +1179,12 @@ class GeneralModuleTests(unittest.TestCase): socket.getaddrinfo(HOST, 80) socket.getaddrinfo(HOST, None) # test family and socktype filters - infos = socket.getaddrinfo(HOST, None, socket.AF_INET) - for family, _, _, _, _ in infos: + infos = socket.getaddrinfo(HOST, 80, socket.AF_INET, socket.SOCK_STREAM) + for family, type, _, _, _ in infos: self.assertEqual(family, socket.AF_INET) + self.assertEqual(str(family), 'AddressFamily.AF_INET') + self.assertEqual(type, socket.SOCK_STREAM) + self.assertEqual(str(type), 'SocketType.SOCK_STREAM') infos = socket.getaddrinfo(HOST, None, 0, socket.SOCK_STREAM) for _, socktype, _, _, _ in infos: self.assertEqual(socktype, socket.SOCK_STREAM) @@ -1172,7 +1226,7 @@ class GeneralModuleTests(unittest.TestCase): def test_getnameinfo(self): # only IP addresses are allowed - self.assertRaises(socket.error, socket.getnameinfo, ('mail.python.org',0), 0) + self.assertRaises(OSError, socket.getnameinfo, ('mail.python.org',0), 0) @unittest.skipUnless(support.is_resource_enabled('network'), 'network is not enabled') @@ -1284,10 +1338,31 @@ class GeneralModuleTests(unittest.TestCase): @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') def test_flowinfo(self): self.assertRaises(OverflowError, socket.getnameinfo, - ('::1',0, 0xffffffff), 0) + (support.HOSTv6, 0, 0xffffffff), 0) with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: - self.assertRaises(OverflowError, s.bind, ('::1', 0, -10)) - + self.assertRaises(OverflowError, s.bind, (support.HOSTv6, 0, -10)) + + def test_str_for_enums(self): + # Make sure that the AF_* and SOCK_* constants have enum-like string + # reprs. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + self.assertEqual(str(s.family), 'AddressFamily.AF_INET') + self.assertEqual(str(s.type), 'SocketType.SOCK_STREAM') + + @unittest.skipIf(os.name == 'nt', 'Will not work on Windows') + def test_uknown_socket_family_repr(self): + # Test that when created with a family that's not one of the known + # AF_*/SOCK_* constants, socket.family just returns the number. + # + # To do this we fool socket.socket into believing it already has an + # open fd because on this path it doesn't actually verify the family and + # type and populates the socket object. + # + # On Windows this trick won't work, so the test is skipped. + fd, _ = tempfile.mkstemp() + with socket.socket(family=42424, type=13331, fileno=fd) as s: + self.assertEqual(s.family, 42424) + self.assertEqual(s.type, 13331) @unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') class BasicCANTest(unittest.TestCase): @@ -1297,10 +1372,35 @@ class BasicCANTest(unittest.TestCase): socket.PF_CAN socket.CAN_RAW + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testBCMConstants(self): + socket.CAN_BCM + + # opcodes + socket.CAN_BCM_TX_SETUP # create (cyclic) transmission task + socket.CAN_BCM_TX_DELETE # remove (cyclic) transmission task + socket.CAN_BCM_TX_READ # read properties of (cyclic) transmission task + socket.CAN_BCM_TX_SEND # send one CAN frame + socket.CAN_BCM_RX_SETUP # create RX content filter subscription + socket.CAN_BCM_RX_DELETE # remove RX content filter subscription + socket.CAN_BCM_RX_READ # read properties of RX content filter subscription + socket.CAN_BCM_TX_STATUS # reply to TX_READ request + socket.CAN_BCM_TX_EXPIRED # notification on performed transmissions (count=0) + socket.CAN_BCM_RX_STATUS # reply to RX_READ request + socket.CAN_BCM_RX_TIMEOUT # cyclic message is absent + socket.CAN_BCM_RX_CHANGED # updated CAN frame (detected content change) + def testCreateSocket(self): with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: pass + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testCreateBCMSocket(self): + with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) as s: + pass + def testBindAny(self): with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: s.bind(('', )) @@ -1308,7 +1408,7 @@ class BasicCANTest(unittest.TestCase): def testTooLongInterfaceName(self): # most systems limit IFNAMSIZ to 16, take 1024 to be sure with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: - self.assertRaisesRegex(socket.error, 'interface name too long', + self.assertRaisesRegex(OSError, 'interface name too long', s.bind, ('x' * 1024,)) @unittest.skipUnless(hasattr(socket, "CAN_RAW_LOOPBACK"), @@ -1333,19 +1433,8 @@ class BasicCANTest(unittest.TestCase): @unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') -@unittest.skipUnless(thread, 'Threading required for this test.') class CANTest(ThreadedCANSocketTest): - """The CAN frame structure is defined in <linux/can.h>: - - struct can_frame { - canid_t can_id; /* 32 bit CAN_ID + EFF/RTR/ERR flags */ - __u8 can_dlc; /* data length code: 0 .. 8 */ - __u8 data[8] __attribute__((aligned(8))); - }; - """ - can_frame_fmt = "=IB3x8s" - def __init__(self, methodName='runTest'): ThreadedCANSocketTest.__init__(self, methodName=methodName) @@ -1394,6 +1483,46 @@ class CANTest(ThreadedCANSocketTest): self.cf2 = self.build_can_frame(0x12, b'\x99\x22\x33') self.cli.send(self.cf2) + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def _testBCM(self): + cf, addr = self.cli.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + can_id, can_dlc, data = self.dissect_can_frame(cf) + self.assertEqual(self.can_id, can_id) + self.assertEqual(self.data, data) + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testBCM(self): + bcm = socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) + self.addCleanup(bcm.close) + bcm.connect((self.interface,)) + self.can_id = 0x123 + self.data = bytes([0xc0, 0xff, 0xee]) + self.cf = self.build_can_frame(self.can_id, self.data) + opcode = socket.CAN_BCM_TX_SEND + flags = 0 + count = 0 + ival1_seconds = ival1_usec = ival2_seconds = ival2_usec = 0 + bcm_can_id = 0x0222 + nframes = 1 + assert len(self.cf) == 16 + header = struct.pack(self.bcm_cmd_msg_fmt, + opcode, + flags, + count, + ival1_seconds, + ival1_usec, + ival2_seconds, + ival2_usec, + bcm_can_id, + nframes, + ) + header_plus_frame = header + self.cf + bytes_sent = bcm.send(header_plus_frame) + self.assertEqual(bytes_sent, len(header_plus_frame)) + @unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.') class BasicRDSTest(unittest.TestCase): @@ -1608,7 +1737,7 @@ class BasicTCPTest(SocketConnectedTest): self.assertEqual(f, fileno) # cli_conn cannot be used anymore... self.assertTrue(self.cli_conn._closed) - self.assertRaises(socket.error, self.cli_conn.recv, 1024) + self.assertRaises(OSError, self.cli_conn.recv, 1024) self.cli_conn.close() # ...but we can create another socket using the (still open) # file descriptor @@ -1977,7 +2106,7 @@ class SendmsgTests(SendrecvmsgServerTimeoutBase): def _testSendmsgExcessCmsgReject(self): if not hasattr(socket, "CMSG_SPACE"): # Can only send one item - with self.assertRaises(socket.error) as cm: + with self.assertRaises(OSError) as cm: self.sendmsgToServer([MSG], [(0, 0, b""), (0, 0, b"")]) self.assertIsNone(cm.exception.errno) self.sendToServer(b"done") @@ -1988,7 +2117,7 @@ class SendmsgTests(SendrecvmsgServerTimeoutBase): def _testSendmsgAfterClose(self): self.cli_sock.close() - self.assertRaises(socket.error, self.sendmsgToServer, [MSG]) + self.assertRaises(OSError, self.sendmsgToServer, [MSG]) class SendmsgStreamTests(SendmsgTests): @@ -2032,7 +2161,7 @@ class SendmsgStreamTests(SendmsgTests): @testSendmsgDontWait.client_skip def _testSendmsgDontWait(self): try: - with self.assertRaises(socket.error) as cm: + with self.assertRaises(OSError) as cm: while True: self.sendmsgToServer([b"a"*512], [], socket.MSG_DONTWAIT) self.assertIn(cm.exception.errno, @@ -2052,9 +2181,9 @@ class SendmsgConnectionlessTests(SendmsgTests): pass def _testSendmsgNoDestAddr(self): - self.assertRaises(socket.error, self.cli_sock.sendmsg, + self.assertRaises(OSError, self.cli_sock.sendmsg, [MSG]) - self.assertRaises(socket.error, self.cli_sock.sendmsg, + self.assertRaises(OSError, self.cli_sock.sendmsg, [MSG], [], 0, None) @@ -2140,7 +2269,7 @@ class RecvmsgGenericTests(SendrecvmsgBase): def testRecvmsgAfterClose(self): # Check that recvmsg[_into]() fails on a closed socket. self.serv_sock.close() - self.assertRaises(socket.error, self.doRecvmsg, self.serv_sock, 1024) + self.assertRaises(OSError, self.doRecvmsg, self.serv_sock, 1024) def _testRecvmsgAfterClose(self): pass @@ -2586,7 +2715,7 @@ class SCMRightsTest(SendrecvmsgServerTimeoutBase): # call fails, just send msg with no ancillary data. try: nbytes = self.sendmsgToServer([msg], ancdata) - except socket.error as e: + except OSError as e: # Check that it was the system call that failed self.assertIsInstance(e.errno, int) nbytes = self.sendmsgToServer([msg]) @@ -2964,7 +3093,7 @@ class RFC3542AncillaryTest(SendrecvmsgServerTimeoutBase): array.array("i", [self.traffic_class]).tobytes() + b"\x00"), (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, array.array("i", [self.hop_limit]))]) - except socket.error as e: + except OSError as e: self.assertIsInstance(e.errno, int) nbytes = self.sendmsgToServer( [MSG], @@ -3422,10 +3551,10 @@ class InterruptedRecvTimeoutTest(InterruptedTimeoutBase, UDPTestBase): self.serv.settimeout(self.timeout) def checkInterruptedRecv(self, func, *args, **kwargs): - # Check that func(*args, **kwargs) raises socket.error with an + # Check that func(*args, **kwargs) raises OSError with an # errno of EINTR when interrupted by a signal. self.setAlarm(self.alarm_time) - with self.assertRaises(socket.error) as cm: + with self.assertRaises(OSError) as cm: func(*args, **kwargs) self.assertNotIsInstance(cm.exception, socket.timeout) self.assertEqual(cm.exception.errno, errno.EINTR) @@ -3482,9 +3611,9 @@ class InterruptedSendTimeoutTest(InterruptedTimeoutBase, def checkInterruptedSend(self, func, *args, **kwargs): # Check that func(*args, **kwargs), run in a loop, raises - # socket.error with an errno of EINTR when interrupted by a + # OSError with an errno of EINTR when interrupted by a # signal. - with self.assertRaises(socket.error) as cm: + with self.assertRaises(OSError) as cm: while True: self.setAlarm(self.alarm_time) func(*args, **kwargs) @@ -3581,7 +3710,7 @@ class NonBlockingTCPTests(ThreadedTCPSocketTest): start = time.time() try: self.serv.accept() - except socket.error: + except OSError: pass end = time.time() self.assertTrue((end - start) < 1.0, "Error setting non-blocking mode.") @@ -3606,7 +3735,7 @@ class NonBlockingTCPTests(ThreadedTCPSocketTest): start = time.time() try: self.serv.accept() - except socket.error: + except OSError: pass end = time.time() self.assertTrue((end - start) < 1.0, "Error creating with non-blocking mode.") @@ -3636,7 +3765,7 @@ class NonBlockingTCPTests(ThreadedTCPSocketTest): self.serv.setblocking(0) try: conn, addr = self.serv.accept() - except socket.error: + except OSError: pass else: self.fail("Error trying to do non-blocking accept.") @@ -3666,7 +3795,7 @@ class NonBlockingTCPTests(ThreadedTCPSocketTest): conn.setblocking(0) try: msg = conn.recv(len(MSG)) - except socket.error: + except OSError: pass else: self.fail("Error trying to do non-blocking recv.") @@ -3749,7 +3878,7 @@ class FileObjectClassTestCase(SocketConnectedTest): # First read raises a timeout self.assertRaises(socket.timeout, self.read_file.read, 1) # Second read is disallowed - with self.assertRaises(IOError) as ctx: + with self.assertRaises(OSError) as ctx: self.read_file.read(1) self.assertIn("cannot read from timed out object", str(ctx.exception)) @@ -3841,7 +3970,7 @@ class FileObjectClassTestCase(SocketConnectedTest): self.read_file.close() self.assertRaises(ValueError, self.read_file.fileno) self.cli_conn.close() - self.assertRaises(socket.error, self.cli_conn.getsockname) + self.assertRaises(OSError, self.cli_conn.getsockname) def _testRealClose(self): pass @@ -3878,7 +4007,7 @@ class FileObjectInterruptedTestCase(unittest.TestCase): @staticmethod def _raise_eintr(): - raise socket.error(errno.EINTR, "interrupted") + raise OSError(errno.EINTR, "interrupted") def _textiowrap_mock_socket(self, mock, buffering=-1): raw = socket.SocketIO(mock, "r") @@ -3990,7 +4119,7 @@ class UnbufferedFileObjectClassTestCase(FileObjectClassTestCase): self.assertEqual(msg, self.read_msg) # ...until the file is itself closed self.read_file.close() - self.assertRaises(socket.error, self.cli_conn.recv, 1024) + self.assertRaises(OSError, self.cli_conn.recv, 1024) def _testMakefileClose(self): self.write_file.write(self.write_msg) @@ -4139,7 +4268,7 @@ class NetworkConnectionNoServer(unittest.TestCase): port = support.find_unused_port() cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.addCleanup(cli.close) - with self.assertRaises(socket.error) as cm: + with self.assertRaises(OSError) as cm: cli.connect((HOST, port)) self.assertEqual(cm.exception.errno, errno.ECONNREFUSED) @@ -4147,7 +4276,7 @@ class NetworkConnectionNoServer(unittest.TestCase): # Issue #9792: errors raised by create_connection() should have # a proper errno attribute. port = support.find_unused_port() - with self.assertRaises(socket.error) as cm: + with self.assertRaises(OSError) as cm: socket.create_connection((HOST, port)) # Issue #16257: create_connection() calls getaddrinfo() against @@ -4295,7 +4424,7 @@ class TCPTimeoutTest(SocketTCPTest): foo = self.serv.accept() except socket.timeout: self.fail("caught timeout instead of error (TCP)") - except socket.error: + except OSError: ok = True except: self.fail("caught unexpected exception (TCP)") @@ -4352,7 +4481,7 @@ class UDPTimeoutTest(SocketUDPTest): foo = self.serv.recv(1024) except socket.timeout: self.fail("caught timeout instead of error (UDP)") - except socket.error: + except OSError: ok = True except: self.fail("caught unexpected exception (UDP)") @@ -4362,10 +4491,10 @@ class UDPTimeoutTest(SocketUDPTest): class TestExceptions(unittest.TestCase): def testExceptionTree(self): - self.assertTrue(issubclass(socket.error, Exception)) - self.assertTrue(issubclass(socket.herror, socket.error)) - self.assertTrue(issubclass(socket.gaierror, socket.error)) - self.assertTrue(issubclass(socket.timeout, socket.error)) + self.assertTrue(issubclass(OSError, Exception)) + self.assertTrue(issubclass(socket.herror, OSError)) + self.assertTrue(issubclass(socket.gaierror, OSError)) + self.assertTrue(issubclass(socket.timeout, OSError)) class TestLinuxAbstractNamespace(unittest.TestCase): @@ -4391,7 +4520,7 @@ class TestLinuxAbstractNamespace(unittest.TestCase): def testNameOverflow(self): address = "\x00" + "h" * self.UNIX_PATH_MAX with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: - self.assertRaises(socket.error, s.bind, address) + self.assertRaises(OSError, s.bind, address) def testStrName(self): # Check that an abstract name can be passed as a string. @@ -4630,7 +4759,7 @@ class ContextManagersTest(ThreadedTCPSocketTest): self.assertTrue(sock._closed) # exception inside with block with socket.socket() as sock: - self.assertRaises(socket.error, sock.sendall, b'foo') + self.assertRaises(OSError, sock.sendall, b'foo') self.assertTrue(sock._closed) def testCreateConnectionBase(self): @@ -4658,19 +4787,76 @@ class ContextManagersTest(ThreadedTCPSocketTest): with socket.create_connection(address) as sock: sock.close() self.assertTrue(sock._closed) - self.assertRaises(socket.error, sock.sendall, b'foo') + self.assertRaises(OSError, sock.sendall, b'foo') -@unittest.skipUnless(hasattr(socket, "SOCK_CLOEXEC"), - "SOCK_CLOEXEC not defined") -@unittest.skipUnless(fcntl, "module fcntl not available") -class CloexecConstantTest(unittest.TestCase): +class InheritanceTest(unittest.TestCase): + @unittest.skipUnless(hasattr(socket, "SOCK_CLOEXEC"), + "SOCK_CLOEXEC not defined") @support.requires_linux_version(2, 6, 28) def test_SOCK_CLOEXEC(self): with socket.socket(socket.AF_INET, socket.SOCK_STREAM | socket.SOCK_CLOEXEC) as s: self.assertTrue(s.type & socket.SOCK_CLOEXEC) - self.assertTrue(fcntl.fcntl(s, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) + self.assertFalse(s.get_inheritable()) + + def test_default_inheritable(self): + sock = socket.socket() + with sock: + self.assertEqual(sock.get_inheritable(), False) + + def test_dup(self): + sock = socket.socket() + with sock: + newsock = sock.dup() + sock.close() + with newsock: + self.assertEqual(newsock.get_inheritable(), False) + + def test_set_inheritable(self): + sock = socket.socket() + with sock: + sock.set_inheritable(True) + self.assertEqual(sock.get_inheritable(), True) + + sock.set_inheritable(False) + self.assertEqual(sock.get_inheritable(), False) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_get_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(sock.get_inheritable(), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(sock.get_inheritable(), True) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_set_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + sock.set_inheritable(True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + + + @unittest.skipUnless(hasattr(socket, "socketpair"), + "need socket.socketpair()") + def test_socketpair(self): + s1, s2 = socket.socketpair() + self.addCleanup(s1.close) + self.addCleanup(s2.close) + self.assertEqual(s1.get_inheritable(), False) + self.assertEqual(s2.get_inheritable(), False) @unittest.skipUnless(hasattr(socket, "SOCK_NONBLOCK"), @@ -4839,7 +5025,7 @@ def test_main(): NetworkConnectionAttributesTest, NetworkConnectionBehaviourTest, ContextManagersTest, - CloexecConstantTest, + InheritanceTest, NonblockConstantTest ]) if hasattr(socket, "socketpair"): diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 464057e19f..02c6c1e4fd 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -2,8 +2,8 @@ Test suite for socketserver. """ +import _imp as imp import contextlib -import imp import os import select import signal @@ -27,7 +27,7 @@ TEST_STR = b"hello world\n" HOST = test.support.HOST HAVE_UNIX_SOCKETS = hasattr(socket, "AF_UNIX") -HAVE_FORKING = hasattr(os, "fork") and os.name != "os2" +HAVE_FORKING = hasattr(os, "fork") def signal_alarm(n): """Call signal.alarm when it exists (i.e. not on Windows).""" @@ -82,7 +82,7 @@ class SocketServerTest(unittest.TestCase): for fn in self.test_files: try: os.remove(fn) - except os.error: + except OSError: pass self.test_files[:] = [] @@ -93,21 +93,7 @@ class SocketServerTest(unittest.TestCase): # XXX: We need a way to tell AF_UNIX to pick its own name # like AF_INET provides port==0. dir = None - if os.name == 'os2': - dir = '\socket' fn = tempfile.mktemp(prefix='unix_socket.', dir=dir) - if os.name == 'os2': - # AF_UNIX socket names on OS/2 require a specific prefix - # which can't include a drive letter and must also use - # backslashes as directory separators - if fn[1] == ':': - fn = fn[2:] - if fn[0] in (os.sep, os.altsep): - fn = fn[1:] - if os.sep == '/': - fn = fn.replace(os.sep, os.altsep) - else: - fn = fn.replace(os.altsep, os.sep) self.test_files.append(fn) return fn diff --git a/Lib/test/test_pep263.py b/Lib/test/test_source_encoding.py index 324ae38619..cd9d2b374c 100644 --- a/Lib/test/test_pep263.py +++ b/Lib/test/test_source_encoding.py @@ -1,9 +1,12 @@ # -*- coding: koi8-r -*- import unittest -from test import support +from test.support import TESTFN, unlink, unload +import importlib +import os +import sys -class PEP263Test(unittest.TestCase): +class SourceEncodingTest(unittest.TestCase): def test_pep263(self): self.assertEqual( @@ -72,9 +75,61 @@ class PEP263Test(unittest.TestCase): with self.assertRaisesRegex(SyntaxError, 'BOM'): compile(b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec') + def test_bad_coding(self): + module_name = 'bad_coding' + self.verify_bad_module(module_name) -def test_main(): - support.run_unittest(PEP263Test) + def test_bad_coding2(self): + module_name = 'bad_coding2' + self.verify_bad_module(module_name) -if __name__=="__main__": - test_main() + def verify_bad_module(self, module_name): + self.assertRaises(SyntaxError, __import__, 'test.' + module_name) + + path = os.path.dirname(__file__) + filename = os.path.join(path, module_name + '.py') + with open(filename, "rb") as fp: + bytes = fp.read() + self.assertRaises(SyntaxError, compile, bytes, filename, 'exec') + + def test_exec_valid_coding(self): + d = {} + exec(b'# coding: cp949\na = "\xaa\xa7"\n', d) + self.assertEqual(d['a'], '\u3047') + + def test_file_parse(self): + # issue1134: all encodings outside latin-1 and utf-8 fail on + # multiline strings and long lines (>512 columns) + unload(TESTFN) + filename = TESTFN + ".py" + f = open(filename, "w", encoding="cp1252") + sys.path.insert(0, os.curdir) + try: + with f: + f.write("# -*- coding: cp1252 -*-\n") + f.write("'''A short string\n") + f.write("'''\n") + f.write("'A very long string %s'\n" % ("X" * 1000)) + + importlib.invalidate_caches() + __import__(TESTFN) + finally: + del sys.path[0] + unlink(filename) + unlink(filename + "c") + unlink(filename + "o") + unload(TESTFN) + + def test_error_from_string(self): + # See http://bugs.python.org/issue6289 + input = "# coding: ascii\n\N{SNOWMAN}".encode('utf-8') + with self.assertRaises(SyntaxError) as c: + compile(input, "<string>", "exec") + expected = "'ascii' codec can't decode byte 0xe2 in position 16: " \ + "ordinal not in range(128)" + self.assertTrue(c.exception.args[0].startswith(expected), + msg=c.exception.args[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index fdd2cf79e8..2605e68cce 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -6,6 +6,7 @@ from test import support import socket import select import time +import datetime import gc import os import errno @@ -17,16 +18,11 @@ import asyncore import weakref import platform import functools +from unittest import mock ssl = support.import_module("ssl") -PROTOCOLS = [ - ssl.PROTOCOL_SSLv3, - ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1 -] -if hasattr(ssl, 'PROTOCOL_SSLv2'): - PROTOCOLS.append(ssl.PROTOCOL_SSLv2) - +PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) HOST = support.HOST data_file = lambda name: os.path.join(os.path.dirname(__file__), name) @@ -48,6 +44,11 @@ KEY_PASSWORD = "somepass" CAPATH = data_file("capath") BYTES_CAPATH = os.fsencode(CAPATH) +# Two keys and certs signed by the same CA (for SNI tests) +SIGNED_CERTFILE = data_file("keycert3.pem") +SIGNED_CERTFILE2 = data_file("keycert4.pem") +SIGNING_CA = data_file("pycacert.pem") + SVN_PYTHON_ORG_ROOT_CERT = data_file("https_svn_python_org_root.pem") EMPTYCERT = data_file("nullcert.pem") @@ -60,6 +61,7 @@ NULLBYTECERT = data_file("nullbytecert.pem") DHFILE = data_file("dh512.pem") BYTES_DHFILE = os.fsencode(DHFILE) + def handle_error(prefix): exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) if support.verbose: @@ -73,6 +75,19 @@ def no_sslv2_implies_sslv3_hello(): # 0.9.7h or higher return ssl.OPENSSL_VERSION_INFO >= (0, 9, 7, 8, 15) +def asn1time(cert_time): + # Some versions of OpenSSL ignore seconds, see #18207 + # 0.9.8.i + if ssl._OPENSSL_API_VERSION == (0, 9, 8, 9, 15): + fmt = "%b %d %H:%M:%S %Y GMT" + dt = datetime.datetime.strptime(cert_time, fmt) + dt = dt.replace(second=0) + cert_time = dt.strftime(fmt) + # %d adds leading zero but ASN1_TIME_print() uses leading space + if cert_time[4] == "0": + cert_time = cert_time[:4] + " " + cert_time[5:] + + return cert_time # Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2 def skip_if_broken_ubuntu_ssl(func): @@ -90,14 +105,12 @@ def skip_if_broken_ubuntu_ssl(func): else: return func +needs_sni = unittest.skipUnless(ssl.HAS_SNI, "SNI support needed for this test") + class BasicSocketTests(unittest.TestCase): def test_constants(self): - #ssl.PROTOCOL_SSLv2 - ssl.PROTOCOL_SSLv23 - ssl.PROTOCOL_SSLv3 - ssl.PROTOCOL_TLSv1 ssl.CERT_NONE ssl.CERT_OPTIONAL ssl.CERT_REQUIRED @@ -175,8 +188,9 @@ class BasicSocketTests(unittest.TestCase): (('organizationName', 'Python Software Foundation'),), (('commonName', 'localhost'),)) ) - self.assertEqual(p['notAfter'], 'Oct 5 23:01:56 2020 GMT') - self.assertEqual(p['notBefore'], 'Oct 8 23:01:56 2010 GMT') + # Note the next three asserts will fail if the keys are regenerated + self.assertEqual(p['notAfter'], asn1time('Oct 5 23:01:56 2020 GMT')) + self.assertEqual(p['notBefore'], asn1time('Oct 8 23:01:56 2010 GMT')) self.assertEqual(p['serialNumber'], 'D7C7381919AFC24E') self.assertEqual(p['subject'], ((('countryName', 'XY'),), @@ -276,15 +290,15 @@ class BasicSocketTests(unittest.TestCase): def test_wrapped_unconnected(self): # Methods on an unconnected SSLSocket propagate the original - # socket.error raise by the underlying socket object. + # OSError raise by the underlying socket object. s = socket.socket(socket.AF_INET) with ssl.wrap_socket(s) as ss: - self.assertRaises(socket.error, ss.recv, 1) - self.assertRaises(socket.error, ss.recv_into, bytearray(b'x')) - self.assertRaises(socket.error, ss.recvfrom, 1) - self.assertRaises(socket.error, ss.recvfrom_into, bytearray(b'x'), 1) - self.assertRaises(socket.error, ss.send, b'x') - self.assertRaises(socket.error, ss.sendto, b'x', ('0.0.0.0', 0)) + self.assertRaises(OSError, ss.recv, 1) + self.assertRaises(OSError, ss.recv_into, bytearray(b'x')) + self.assertRaises(OSError, ss.recvfrom, 1) + self.assertRaises(OSError, ss.recvfrom_into, bytearray(b'x'), 1) + self.assertRaises(OSError, ss.send, b'x') + self.assertRaises(OSError, ss.sendto, b'x', ('0.0.0.0', 0)) def test_timeout(self): # Issue #8524: when creating an SSL socket, the timeout of the @@ -309,15 +323,15 @@ class BasicSocketTests(unittest.TestCase): with ssl.wrap_socket(sock, server_side=True, certfile=CERTFILE) as s: self.assertRaisesRegex(ValueError, "can't connect in server-side mode", s.connect, (HOST, 8080)) - with self.assertRaises(IOError) as cm: + with self.assertRaises(OSError) as cm: with socket.socket() as sock: ssl.wrap_socket(sock, certfile=WRONGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) - with self.assertRaises(IOError) as cm: + with self.assertRaises(OSError) as cm: with socket.socket() as sock: ssl.wrap_socket(sock, certfile=CERTFILE, keyfile=WRONGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) - with self.assertRaises(IOError) as cm: + with self.assertRaises(OSError) as cm: with socket.socket() as sock: ssl.wrap_socket(sock, certfile=WRONGCERT, keyfile=WRONGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) @@ -463,15 +477,48 @@ class BasicSocketTests(unittest.TestCase): support.gc_collect() self.assertIn(r, str(cm.warning.args[0])) + def test_get_default_verify_paths(self): + paths = ssl.get_default_verify_paths() + self.assertEqual(len(paths), 6) + self.assertIsInstance(paths, ssl.DefaultVerifyPaths) + + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + paths = ssl.get_default_verify_paths() + self.assertEqual(paths.cafile, CERTFILE) + self.assertEqual(paths.capath, CAPATH) + + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_cert_store(self): + self.assertEqual(ssl.X509_ASN_ENCODING, 1) + self.assertEqual(ssl.PKCS_7_ASN_ENCODING, 0x00010000) + + self.assertEqual(ssl.enum_cert_store("CA"), + ssl.enum_cert_store("CA", "certificate")) + ssl.enum_cert_store("CA", "crl") + self.assertEqual(ssl.enum_cert_store("ROOT"), + ssl.enum_cert_store("ROOT", "certificate")) + ssl.enum_cert_store("ROOT", "crl") + + self.assertRaises(TypeError, ssl.enum_cert_store) + self.assertRaises(WindowsError, ssl.enum_cert_store, "") + self.assertRaises(ValueError, ssl.enum_cert_store, "CA", "wrong") + + ca = ssl.enum_cert_store("CA") + self.assertIsInstance(ca, list) + self.assertIsInstance(ca[0], tuple) + self.assertEqual(len(ca[0]), 2) + self.assertIsInstance(ca[0][0], bytes) + self.assertIsInstance(ca[0][1], int) + class ContextTests(unittest.TestCase): @skip_if_broken_ubuntu_ssl def test_constructor(self): - if hasattr(ssl, 'PROTOCOL_SSLv2'): - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv2) - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv3) - ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + for protocol in PROTOCOLS: + ssl.SSLContext(protocol) self.assertRaises(TypeError, ssl.SSLContext) self.assertRaises(ValueError, ssl.SSLContext, -1) self.assertRaises(ValueError, ssl.SSLContext, 42) @@ -531,7 +578,7 @@ class ContextTests(unittest.TestCase): ctx.load_cert_chain(CERTFILE) ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE) self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE) - with self.assertRaises(IOError) as cm: + with self.assertRaises(OSError) as cm: ctx.load_cert_chain(WRONGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): @@ -616,7 +663,7 @@ class ContextTests(unittest.TestCase): ctx.load_verify_locations(cafile=BYTES_CERTFILE, capath=None) self.assertRaises(TypeError, ctx.load_verify_locations) self.assertRaises(TypeError, ctx.load_verify_locations, None, None) - with self.assertRaises(IOError) as cm: + with self.assertRaises(OSError) as cm: ctx.load_verify_locations(WRONGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): @@ -674,6 +721,75 @@ class ContextTests(unittest.TestCase): self.assertRaises(ValueError, ctx.set_ecdh_curve, "foo") self.assertRaises(ValueError, ctx.set_ecdh_curve, b"foo") + @needs_sni + def test_sni_callback(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + + # set_servername_callback expects a callable, or None + self.assertRaises(TypeError, ctx.set_servername_callback) + self.assertRaises(TypeError, ctx.set_servername_callback, 4) + self.assertRaises(TypeError, ctx.set_servername_callback, "") + self.assertRaises(TypeError, ctx.set_servername_callback, ctx) + + def dummycallback(sock, servername, ctx): + pass + ctx.set_servername_callback(None) + ctx.set_servername_callback(dummycallback) + + @needs_sni + def test_sni_callback_refcycle(self): + # Reference cycles through the servername callback are detected + # and cleared. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + def dummycallback(sock, servername, ctx, cycle=ctx): + pass + ctx.set_servername_callback(dummycallback) + wr = weakref.ref(ctx) + del ctx, dummycallback + gc.collect() + self.assertIs(wr(), None) + + def test_cert_store_stats(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_cert_chain(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 1}) + ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 1, 'crl': 0, 'x509': 2}) + + def test_get_ca_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.get_ca_certs(), []) + # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.get_ca_certs(), []) + # but SVN_PYTHON_ORG_ROOT_CERT is a CA cert + ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT) + self.assertEqual(ctx.get_ca_certs(), + [{'issuer': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'notAfter': asn1time('Mar 29 12:29:49 2033 GMT'), + 'notBefore': asn1time('Mar 30 12:29:49 2003 GMT'), + 'serialNumber': '00', + 'subject': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'version': 3}]) + + with open(SVN_PYTHON_ORG_ROOT_CERT) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + self.assertEqual(ctx.get_ca_certs(True), [der]) + class SSLErrorTests(unittest.TestCase): @@ -989,6 +1105,22 @@ class NetworkedTests(unittest.TestCase): finally: s.close() + def test_get_ca_certs_capath(self): + # capath certs are loaded on request + with support.transient_internet("svn.python.org"): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + self.assertEqual(ctx.get_ca_certs(), []) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect(("svn.python.org", 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + self.assertEqual(len(ctx.get_ca_certs()), 1) + try: import threading @@ -1116,7 +1248,7 @@ else: sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n" % (msg, ctype, msg.lower(), ctype)) self.write(msg.lower()) - except socket.error: + except OSError: if self.server.chatty: handle_error("Test server failure:\n") self.close() @@ -1226,7 +1358,7 @@ else: return self.handle_close() except ssl.SSLError: raise - except socket.error as err: + except OSError as err: if err.args[0] == errno.ECONNABORTED: return self.handle_close() else: @@ -1330,19 +1462,19 @@ else: except ssl.SSLError as x: if support.verbose: sys.stdout.write("\nSSLError is %s\n" % x.args[1]) - except socket.error as x: + except OSError as x: if support.verbose: - sys.stdout.write("\nsocket.error is %s\n" % x.args[1]) - except IOError as x: + sys.stdout.write("\nOSError is %s\n" % x.args[1]) + except OSError as x: if x.errno != errno.ENOENT: raise if support.verbose: - sys.stdout.write("\IOError is %s\n" % str(x)) + sys.stdout.write("\OSError is %s\n" % str(x)) else: raise AssertionError("Use of invalid cert should have failed!") def server_params_test(client_context, server_context, indata=b"FOO\n", - chatty=True, connectionchatty=False): + chatty=True, connectionchatty=False, sni_name=None): """ Launch a server, connect a client to it and try various reads and writes. @@ -1352,7 +1484,8 @@ else: chatty=chatty, connectionchatty=False) with server: - with client_context.wrap_socket(socket.socket()) as s: + with client_context.wrap_socket(socket.socket(), + server_hostname=sni_name) as s: s.connect((HOST, server.port)) for arg in [indata, bytearray(indata), memoryview(indata)]: if connectionchatty: @@ -1376,6 +1509,7 @@ else: stats.update({ 'compression': s.compression(), 'cipher': s.cipher(), + 'peercert': s.getpeercert(), 'client_npn_protocol': s.selected_npn_protocol() }) s.close() @@ -1401,12 +1535,15 @@ else: client_context.options = ssl.OP_ALL | client_options server_context = ssl.SSLContext(server_protocol) server_context.options = ssl.OP_ALL | server_options + + # NOTE: we must enable "ALL" ciphers on the client, otherwise an + # SSLv23 client will send an SSLv3 hello (rather than SSLv2) + # starting from OpenSSL 1.0.0 (see issue #8322). + if client_context.protocol == ssl.PROTOCOL_SSLv23: + client_context.set_ciphers("ALL") + for ctx in (client_context, server_context): ctx.verify_mode = certsreqs - # NOTE: we must enable "ALL" ciphers, otherwise an SSLv23 client - # will send an SSLv3 hello (rather than SSLv2) starting from - # OpenSSL 1.0.0 (see issue #8322). - ctx.set_ciphers("ALL") ctx.load_cert_chain(CERTFILE) ctx.load_verify_locations(CERTFILE) try: @@ -1417,7 +1554,7 @@ else: except ssl.SSLError: if expect_success: raise - except socket.error as e: + except OSError as e: if expect_success or e.errno != errno.ECONNRESET: raise else: @@ -1436,10 +1573,11 @@ else: if support.verbose: sys.stdout.write("\n") for protocol in PROTOCOLS: - context = ssl.SSLContext(protocol) - context.load_cert_chain(CERTFILE) - server_params_test(context, context, - chatty=True, connectionchatty=True) + with self.subTest(protocol=ssl._PROTOCOL_NAMES[protocol]): + context = ssl.SSLContext(protocol) + context.load_cert_chain(CERTFILE) + server_params_test(context, context, + chatty=True, connectionchatty=True) def test_getpeercert(self): if support.verbose: @@ -1450,8 +1588,14 @@ else: context.load_cert_chain(CERTFILE) server = ThreadedEchoServer(context=context, chatty=False) with server: - s = context.wrap_socket(socket.socket()) + s = context.wrap_socket(socket.socket(), + do_handshake_on_connect=False) s.connect((HOST, server.port)) + # getpeercert() raise ValueError while the handshake isn't + # done. + with self.assertRaises(ValueError): + s.getpeercert() + s.do_handshake() cert = s.getpeercert() self.assertTrue(cert, "Can't get peer certificate.") cipher = s.cipher() @@ -1491,7 +1635,7 @@ else: "badkey.pem")) def test_rude_shutdown(self): - """A brutal shutdown of an SSL server should raise an IOError + """A brutal shutdown of an SSL server should raise an OSError in the client when attempting handshake. """ listener_ready = threading.Event() @@ -1519,7 +1663,7 @@ else: listener_gone.wait() try: ssl_sock = ssl.wrap_socket(c) - except IOError: + except OSError: pass else: self.fail('connecting to closed SSL socket should have failed') @@ -1562,7 +1706,7 @@ else: if hasattr(ssl, 'PROTOCOL_SSLv2'): try: try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv2, True) - except (ssl.SSLError, socket.error) as x: + except OSError as x: # this fails on some older versions of OpenSSL (0.9.7l, for instance) if support.verbose: sys.stdout.write( @@ -1622,6 +1766,49 @@ else: try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False, client_options=ssl.OP_NO_TLSv1) + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_1"), + "TLS version 1.1 not supported.") + def test_protocol_tlsv1_1(self): + """Connecting to a TLSv1.1 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, True) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1_1) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_1, True) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False) + + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_2"), + "TLS version 1.2 not supported.") + def test_protocol_tlsv1_2(self): + """Connecting to a TLSv1.2 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, True, + server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2, + client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv2, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1_2) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_2, True) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_2, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False) + def test_starttls(self): """Switching from clear text to encrypted and back again.""" msgs = (b"msg 1", b"MSG 2", b"STARTTLS", b"MSG 3", b"msg 4", b"ENDTLS", b"msg 5", b"msg 6") @@ -1682,7 +1869,7 @@ else: def test_socketserver(self): """Using a SocketServer to create and manage SSL connections.""" - server = make_https_server(self, CERTFILE) + server = make_https_server(self, certfile=CERTFILE) # try to connect if support.verbose: sys.stdout.write('\n') @@ -1938,6 +2125,20 @@ else: self.assertIsInstance(remote, ssl.SSLSocket) self.assertEqual(peer, client_addr) + def test_getpeercert_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.getpeercert() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_do_handshake_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.do_handshake() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + def test_default_ciphers(self): context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) try: @@ -1949,7 +2150,7 @@ else: ssl_version=ssl.PROTOCOL_SSLv23, chatty=False) as server: with context.wrap_socket(socket.socket()) as s: - with self.assertRaises((OSError, ssl.SSLError)): + with self.assertRaises(OSError): s.connect((HOST, server.port)) self.assertIn("no shared cipher", str(server.conn_errors[0])) @@ -2082,6 +2283,124 @@ else: if len(stats['server_npn_protocols']) else 'nothing' self.assertEqual(server_result, expected, msg % (server_result, "server")) + def sni_contexts(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + other_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + other_context.load_cert_chain(SIGNED_CERTFILE2) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.verify_mode = ssl.CERT_REQUIRED + client_context.load_verify_locations(SIGNING_CA) + return server_context, other_context, client_context + + def check_common_name(self, stats, name): + cert = stats['peercert'] + self.assertIn((('commonName', name),), cert['subject']) + + @needs_sni + def test_sni_callback(self): + calls = [] + server_context, other_context, client_context = self.sni_contexts() + + def servername_cb(ssl_sock, server_name, initial_context): + calls.append((server_name, initial_context)) + if server_name is not None: + ssl_sock.context = other_context + server_context.set_servername_callback(servername_cb) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='supermessage') + # The hostname was fetched properly, and the certificate was + # changed for the connection. + self.assertEqual(calls, [("supermessage", server_context)]) + # CERTFILE4 was selected + self.check_common_name(stats, 'fakehostname') + + calls = [] + # The callback is called with server_name=None + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name=None) + self.assertEqual(calls, [(None, server_context)]) + self.check_common_name(stats, 'localhost') + + # Check disabling the callback + calls = [] + server_context.set_servername_callback(None) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='notfunny') + # Certificate didn't change + self.check_common_name(stats, 'localhost') + self.assertEqual(calls, []) + + @needs_sni + def test_sni_callback_alert(self): + # Returning a TLS alert is reflected to the connecting client + server_context, other_context, client_context = self.sni_contexts() + + def cb_returning_alert(ssl_sock, server_name, initial_context): + return ssl.ALERT_DESCRIPTION_ACCESS_DENIED + server_context.set_servername_callback(cb_returning_alert) + + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_ACCESS_DENIED') + + @needs_sni + def test_sni_callback_raising(self): + # Raising fails the connection with a TLS handshake failure alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_raising(ssl_sock, server_name, initial_context): + 1/0 + server_context.set_servername_callback(cb_raising) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'SSLV3_ALERT_HANDSHAKE_FAILURE') + self.assertIn("ZeroDivisionError", stderr.getvalue()) + + @needs_sni + def test_sni_callback_wrong_return_type(self): + # Returning the wrong return type terminates the TLS connection + # with an internal error alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_wrong_return_type(ssl_sock, server_name, initial_context): + return "foo" + server_context.set_servername_callback(cb_wrong_return_type) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR') + self.assertIn("TypeError", stderr.getvalue()) + + def test_read_write_after_close_raises_valuerror(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + + with server: + s = context.wrap_socket(socket.socket()) + s.connect((HOST, server.port)) + s.close() + + self.assertRaises(ValueError, s.read, 1024) + self.assertRaises(ValueError, s.write, b'hello') + def test_main(verbose=False): if support.verbose: @@ -2101,10 +2420,16 @@ def test_main(verbose=False): (ssl.OPENSSL_VERSION, ssl.OPENSSL_VERSION_INFO)) print(" under %s" % plat) print(" HAS_SNI = %r" % ssl.HAS_SNI) + print(" OP_ALL = 0x%8x" % ssl.OP_ALL) + try: + print(" OP_NO_TLSv1_1 = 0x%8x" % ssl.OP_NO_TLSv1_1) + except AttributeError: + pass for filename in [ CERTFILE, SVN_PYTHON_ORG_ROOT_CERT, BYTES_CERTFILE, ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY, + SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA, BADCERT, BADKEY, EMPTYCERT]: if not os.path.exists(filename): raise support.TestFailed("Can't read certificate file %r" % filename) diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index eb3f07a63d..af6ced4204 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -1,9 +1,13 @@ import unittest import os -from test.support import TESTFN, run_unittest, import_fresh_module -import stat +from test.support import TESTFN, import_fresh_module + +c_stat = import_fresh_module('stat', fresh=['_stat']) +py_stat = import_fresh_module('stat', blocked=['_stat']) + +class TestFilemode: + statmod = None -class TestFilemode(unittest.TestCase): file_flags = {'SF_APPEND', 'SF_ARCHIVED', 'SF_IMMUTABLE', 'SF_NOUNLINK', 'SF_SNAPSHOT', 'UF_APPEND', 'UF_COMPRESSED', 'UF_HIDDEN', 'UF_IMMUTABLE', 'UF_NODUMP', 'UF_NOUNLINK', 'UF_OPAQUE'} @@ -63,17 +67,17 @@ class TestFilemode(unittest.TestCase): st_mode = os.lstat(fname).st_mode else: st_mode = os.stat(fname).st_mode - modestr = stat.filemode(st_mode) + modestr = self.statmod.filemode(st_mode) return st_mode, modestr def assertS_IS(self, name, mode): # test format, lstrip is for S_IFIFO - fmt = getattr(stat, "S_IF" + name.lstrip("F")) - self.assertEqual(stat.S_IFMT(mode), fmt) + fmt = getattr(self.statmod, "S_IF" + name.lstrip("F")) + self.assertEqual(self.statmod.S_IFMT(mode), fmt) # test that just one function returns true testname = "S_IS" + name for funcname in self.format_funcs: - func = getattr(stat, funcname, None) + func = getattr(self.statmod, funcname, None) if func is None: if funcname == testname: raise ValueError(funcname) @@ -91,35 +95,35 @@ class TestFilemode(unittest.TestCase): st_mode, modestr = self.get_mode() self.assertEqual(modestr, '-rwx------') self.assertS_IS("REG", st_mode) - self.assertEqual(stat.S_IMODE(st_mode), - stat.S_IRWXU) + self.assertEqual(self.statmod.S_IMODE(st_mode), + self.statmod.S_IRWXU) os.chmod(TESTFN, 0o070) st_mode, modestr = self.get_mode() self.assertEqual(modestr, '----rwx---') self.assertS_IS("REG", st_mode) - self.assertEqual(stat.S_IMODE(st_mode), - stat.S_IRWXG) + self.assertEqual(self.statmod.S_IMODE(st_mode), + self.statmod.S_IRWXG) os.chmod(TESTFN, 0o007) st_mode, modestr = self.get_mode() self.assertEqual(modestr, '-------rwx') self.assertS_IS("REG", st_mode) - self.assertEqual(stat.S_IMODE(st_mode), - stat.S_IRWXO) + self.assertEqual(self.statmod.S_IMODE(st_mode), + self.statmod.S_IRWXO) os.chmod(TESTFN, 0o444) st_mode, modestr = self.get_mode() self.assertS_IS("REG", st_mode) self.assertEqual(modestr, '-r--r--r--') - self.assertEqual(stat.S_IMODE(st_mode), 0o444) + self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444) else: os.chmod(TESTFN, 0o700) st_mode, modestr = self.get_mode() self.assertEqual(modestr[:3], '-rw') self.assertS_IS("REG", st_mode) - self.assertEqual(stat.S_IFMT(st_mode), - stat.S_IFREG) + self.assertEqual(self.statmod.S_IFMT(st_mode), + self.statmod.S_IFREG) def test_directory(self): os.mkdir(TESTFN) @@ -165,25 +169,34 @@ class TestFilemode(unittest.TestCase): def test_module_attributes(self): for key, value in self.stat_struct.items(): - modvalue = getattr(stat, key) + modvalue = getattr(self.statmod, key) self.assertEqual(value, modvalue, key) for key, value in self.permission_bits.items(): - modvalue = getattr(stat, key) + modvalue = getattr(self.statmod, key) self.assertEqual(value, modvalue, key) for key in self.file_flags: - modvalue = getattr(stat, key) + modvalue = getattr(self.statmod, key) self.assertIsInstance(modvalue, int) for key in self.formats: - modvalue = getattr(stat, key) + modvalue = getattr(self.statmod, key) self.assertIsInstance(modvalue, int) for key in self.format_funcs: - func = getattr(stat, key) + func = getattr(self.statmod, key) self.assertTrue(callable(func)) self.assertEqual(func(0), 0) -def test_main(): - run_unittest(TestFilemode) +class TestFilemodeCStat(TestFilemode, unittest.TestCase): + statmod = c_stat + + formats = TestFilemode.formats | {'S_IFDOOR', 'S_IFPORT', 'S_IFWHT'} + format_funcs = TestFilemode.format_funcs | {'S_ISDOOR', 'S_ISPORT', + 'S_ISWHT'} + + +class TestFilemodePyStat(TestFilemode, unittest.TestCase): + statmod = py_stat + if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_strftime.py b/Lib/test/test_strftime.py index 14057eb34a..6510c36698 100644 --- a/Lib/test/test_strftime.py +++ b/Lib/test/test_strftime.py @@ -176,8 +176,6 @@ class StrftimeTest(unittest.TestCase): (e[0], e[2])) print(" Expected %s, but got %s" % (e[1], result)) -def test_main(): - support.run_unittest(StrftimeTest) if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 50cae052f3..0107eebca6 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -1,4 +1,6 @@ +from collections import abc import array +import operator import unittest import struct import sys @@ -6,7 +8,6 @@ import sys from test import support ISBIGENDIAN = sys.byteorder == "big" -IS32BIT = sys.maxsize == 0x7fffffff integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'n', 'N' byteorders = '', '@', '=', '<', '>', '!' @@ -489,7 +490,7 @@ class StructTest(unittest.TestCase): def test_bool(self): class ExplodingBool(object): def __bool__(self): - raise IOError + raise OSError for prefix in tuple("<>!=")+('',): false = (), [], [], '', 0 true = [1], 'test', 5, -1, 0xffffffff+1, 0xffffffff/2 @@ -520,10 +521,10 @@ class StructTest(unittest.TestCase): try: struct.pack(prefix + '?', ExplodingBool()) - except IOError: + except OSError: pass else: - self.fail("Expected IOError: struct.pack(%r, " + self.fail("Expected OSError: struct.pack(%r, " "ExplodingBool())" % (prefix + '?')) for c in [b'\x01', b'\x7f', b'\xff', b'\x0f', b'\xf0']: @@ -536,10 +537,6 @@ class StructTest(unittest.TestCase): hugecount2 = '{}b{}H'.format(sys.maxsize//2, sys.maxsize//2) self.assertRaises(struct.error, struct.calcsize, hugecount2) - if IS32BIT: - def test_crasher(self): - self.assertRaises(MemoryError, struct.pack, "357913941b", "a") - def test_trailing_counter(self): store = array.array('b', b' '*100) @@ -576,7 +573,7 @@ class StructTest(unittest.TestCase): # The size of 'PyStructObject' totalsize = support.calcobjsize('2n3P') # The size taken up by the 'formatcode' dynamic array - totalsize += struct.calcsize('P2n0P') * (number_of_codes + 1) + totalsize += struct.calcsize('P3n0P') * (number_of_codes + 1) support.check_sizeof(self, struct.Struct(format_str), totalsize) @support.cpython_only @@ -587,14 +584,84 @@ class StructTest(unittest.TestCase): self.check_sizeof('B' * 1234, 1234) self.check_sizeof('fd', 2) self.check_sizeof('xxxxxxxxxxxxxx', 0) - self.check_sizeof('100H', 100) + self.check_sizeof('100H', 1) self.check_sizeof('187s', 1) self.check_sizeof('20p', 1) self.check_sizeof('0s', 1) self.check_sizeof('0c', 0) + +class UnpackIteratorTest(unittest.TestCase): + """ + Tests for iterative unpacking (struct.Struct.iter_unpack). + """ + + def test_construct(self): + def _check_iterator(it): + self.assertIsInstance(it, abc.Iterator) + self.assertIsInstance(it, abc.Iterable) + s = struct.Struct('>ibcp') + it = s.iter_unpack(b"") + _check_iterator(it) + it = s.iter_unpack(b"1234567") + _check_iterator(it) + # Wrong bytes length + with self.assertRaises(struct.error): + s.iter_unpack(b"123456") + with self.assertRaises(struct.error): + s.iter_unpack(b"12345678") + # Zero-length struct + s = struct.Struct('>') + with self.assertRaises(struct.error): + s.iter_unpack(b"") + with self.assertRaises(struct.error): + s.iter_unpack(b"12") + + def test_iterate(self): + s = struct.Struct('>IB') + b = bytes(range(1, 16)) + it = s.iter_unpack(b) + self.assertEqual(next(it), (0x01020304, 5)) + self.assertEqual(next(it), (0x06070809, 10)) + self.assertEqual(next(it), (0x0b0c0d0e, 15)) + self.assertRaises(StopIteration, next, it) + self.assertRaises(StopIteration, next, it) + + def test_arbitrary_buffer(self): + s = struct.Struct('>IB') + b = bytes(range(1, 11)) + it = s.iter_unpack(memoryview(b)) + self.assertEqual(next(it), (0x01020304, 5)) + self.assertEqual(next(it), (0x06070809, 10)) + self.assertRaises(StopIteration, next, it) + self.assertRaises(StopIteration, next, it) + + def test_length_hint(self): + lh = operator.length_hint + s = struct.Struct('>IB') + b = bytes(range(1, 16)) + it = s.iter_unpack(b) + self.assertEqual(lh(it), 3) + next(it) + self.assertEqual(lh(it), 2) + next(it) + self.assertEqual(lh(it), 1) + next(it) + self.assertEqual(lh(it), 0) + self.assertRaises(StopIteration, next, it) + self.assertEqual(lh(it), 0) + + def test_module_func(self): + # Sanity check for the global struct.iter_unpack() + it = struct.iter_unpack('>IB', bytes(range(1, 11))) + self.assertEqual(next(it), (0x01020304, 5)) + self.assertEqual(next(it), (0x06070809, 10)) + self.assertRaises(StopIteration, next, it) + self.assertRaises(StopIteration, next, it) + + def test_main(): - support.run_unittest(StructTest) + support.run_unittest(__name__) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index a89e9556c1..353d0eadcc 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -38,7 +38,7 @@ class StructSeqTest(unittest.TestCase): # os.stat() gives a complicated struct sequence. st = os.stat(__file__) rep = repr(st) - self.assertTrue(rep.startswith(os.name + ".stat_result")) + self.assertTrue(rep.startswith("os.stat_result")) self.assertIn("st_mode=", rep) self.assertIn("st_ino=", rep) self.assertIn("st_dev=", rep) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 4c9f29bdb8..840577dba1 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -19,10 +19,6 @@ import gc import textwrap try: - import resource -except ImportError: - resource = None -try: import threading except ImportError: threading = None @@ -162,8 +158,28 @@ class ProcessTestCase(BaseTestCase): stderr=subprocess.STDOUT) self.assertIn(b'BDFL', output) + def test_check_output_stdin_arg(self): + # check_output() can be called with stdin set to a file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + output = subprocess.check_output( + [sys.executable, "-c", + "import sys; sys.stdout.write(sys.stdin.read().upper())"], + stdin=tf) + self.assertIn(b'PEAR', output) + + def test_check_output_input_arg(self): + # check_output() can be called with input set to a string + output = subprocess.check_output( + [sys.executable, "-c", + "import sys; sys.stdout.write(sys.stdin.read().upper())"], + input=b'pear') + self.assertIn(b'PEAR', output) + def test_check_output_stdout_arg(self): - # check_output() function stderr redirected to stdout + # check_output() refuses to accept 'stdout' argument with self.assertRaises(ValueError) as c: output = subprocess.check_output( [sys.executable, "-c", "print('will not be run')"], @@ -171,6 +187,20 @@ class ProcessTestCase(BaseTestCase): self.fail("Expected ValueError when stdout arg supplied.") self.assertIn('stdout', c.exception.args[0]) + def test_check_output_stdin_with_input_arg(self): + # check_output() refuses to accept 'stdin' with 'input' + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + with self.assertRaises(ValueError) as c: + output = subprocess.check_output( + [sys.executable, "-c", "print('will not be run')"], + stdin=tf, input=b'hare') + self.fail("Expected ValueError when stdin and input args supplied.") + self.assertIn('stdin', c.exception.args[0]) + self.assertIn('input', c.exception.args[0]) + def test_check_output_timeout(self): # check_output() function with timeout arg with self.assertRaises(subprocess.TimeoutExpired) as c: @@ -853,8 +883,9 @@ class ProcessTestCase(BaseTestCase): # # UTF-16 and UTF-32-BE are sufficient to check both with BOM and # without, and UTF-16 and UTF-32. + import _bootlocale for encoding in ['utf-16', 'utf-32-be']: - old_getpreferredencoding = locale.getpreferredencoding + old_getpreferredencoding = _bootlocale.getpreferredencoding # Indirectly via io.TextIOWrapper, Popen() defaults to # locale.getpreferredencoding(False) and earlier in Python 3.2 to # locale.getpreferredencoding(). @@ -865,7 +896,7 @@ class ProcessTestCase(BaseTestCase): encoding) args = [sys.executable, '-c', code] try: - locale.getpreferredencoding = getpreferredencoding + _bootlocale.getpreferredencoding = getpreferredencoding # We set stdin to be non-None because, as of this writing, # a different code path is used when the number of pipes is # zero or one. @@ -874,7 +905,7 @@ class ProcessTestCase(BaseTestCase): stdout=subprocess.PIPE) stdout, stderr = popen.communicate(input='') finally: - locale.getpreferredencoding = old_getpreferredencoding + _bootlocale.getpreferredencoding = old_getpreferredencoding self.assertEqual(stdout, '1\n2\n3\n4') def test_no_leaking(self): @@ -982,8 +1013,7 @@ class ProcessTestCase(BaseTestCase): # value for that limit, but Windows has 2048, so we loop # 1024 times (each call leaked two fds). for i in range(1024): - # Windows raises IOError. Others raise OSError. - with self.assertRaises(EnvironmentError) as c: + with self.assertRaises(OSError) as c: subprocess.Popen(['nonexisting_i_hope'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -1114,47 +1144,6 @@ class ProcessTestCase(BaseTestCase): fds_after_exception = os.listdir(fd_directory) self.assertEqual(fds_before_popen, fds_after_exception) - -# context manager -class _SuppressCoreFiles(object): - """Try to prevent core files from being created.""" - old_limit = None - - def __enter__(self): - """Try to save previous ulimit, then set it to (0, 0).""" - if resource is not None: - try: - self.old_limit = resource.getrlimit(resource.RLIMIT_CORE) - resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) - except (ValueError, resource.error): - pass - - if sys.platform == 'darwin': - # Check if the 'Crash Reporter' on OSX was configured - # in 'Developer' mode and warn that it will get triggered - # when it is. - # - # This assumes that this context manager is used in tests - # that might trigger the next manager. - value = subprocess.Popen(['/usr/bin/defaults', 'read', - 'com.apple.CrashReporter', 'DialogType'], - stdout=subprocess.PIPE).communicate()[0] - if value.strip() == b'developer': - print("this tests triggers the Crash Reporter, " - "that is intentional", end='') - sys.stdout.flush() - - def __exit__(self, *args): - """Return core file behavior to default.""" - if self.old_limit is None: - return - if resource is not None: - try: - resource.setrlimit(resource.RLIMIT_CORE, self.old_limit) - except (ValueError, resource.error): - pass - - @unittest.skipIf(mswindows, "POSIX specific tests") class POSIXProcessTestCase(BaseTestCase): @@ -1243,7 +1232,7 @@ class POSIXProcessTestCase(BaseTestCase): def test_run_abort(self): # returncode handles signal termination - with _SuppressCoreFiles(): + with support.SuppressCrashReport(): p = subprocess.Popen([sys.executable, "-c", 'import os; os.abort()']) p.wait() @@ -1266,7 +1255,7 @@ class POSIXProcessTestCase(BaseTestCase): try: p = subprocess.Popen([sys.executable, "-c", ""], preexec_fn=raise_it) - except RuntimeError as e: + except subprocess.SubprocessError as e: self.assertTrue( subprocess._posixsubprocess, "Expected a ValueError from the preexec_fn") @@ -1306,9 +1295,10 @@ class POSIXProcessTestCase(BaseTestCase): """Issue16140: Don't double close pipes on preexec error.""" def raise_it(): - raise RuntimeError("force the _execute_child() errpipe_data path.") + raise subprocess.SubprocessError( + "force the _execute_child() errpipe_data path.") - with self.assertRaises(RuntimeError): + with self.assertRaises(subprocess.SubprocessError): self._TestExecuteChildPopen( self, [sys.executable, "-c", "pass"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -1507,16 +1497,28 @@ class POSIXProcessTestCase(BaseTestCase): # Terminating a dead process self._kill_dead_process('terminate') + def _save_fds(self, save_fds): + fds = [] + for fd in save_fds: + inheritable = os.get_inheritable(fd) + saved = os.dup(fd) + fds.append((fd, saved, inheritable)) + return fds + + def _restore_fds(self, fds): + for fd, saved, inheritable in fds: + os.dup2(saved, fd, inheritable=inheritable) + os.close(saved) + def check_close_std_fds(self, fds): # Issue #9905: test that subprocess pipes still work properly with # some standard fds closed stdin = 0 - newfds = [] - for a in fds: - b = os.dup(a) - newfds.append(b) - if a == 0: - stdin = b + saved_fds = self._save_fds(fds) + for fd, saved, inheritable in saved_fds: + if fd == 0: + stdin = saved + break try: for fd in fds: os.close(fd) @@ -1531,10 +1533,7 @@ class POSIXProcessTestCase(BaseTestCase): err = support.strip_python_stderr(err) self.assertEqual((out, err), (b'apple', b'orange')) finally: - for b, a in zip(newfds, fds): - os.dup2(b, a) - for b in newfds: - os.close(b) + self._restore_fds(saved_fds) def test_close_fd_0(self): self.check_close_std_fds([0]) @@ -1574,7 +1573,7 @@ class POSIXProcessTestCase(BaseTestCase): os.lseek(temp_fds[1], 0, 0) # move the standard file descriptors out of the way - saved_fds = [os.dup(fd) for fd in range(3)] + saved_fds = self._save_fds(range(3)) try: # duplicate the file objects over the standard fd's for fd, temp_fd in enumerate(temp_fds): @@ -1590,10 +1589,7 @@ class POSIXProcessTestCase(BaseTestCase): stderr=temp_fds[0]) p.wait() finally: - # restore the original fd's underneath sys.stdin, etc. - for std, saved in enumerate(saved_fds): - os.dup2(saved, std) - os.close(saved) + self._restore_fds(saved_fds) for fd in temp_fds: os.lseek(fd, 0, 0) @@ -1617,7 +1613,7 @@ class POSIXProcessTestCase(BaseTestCase): os.unlink(fname) # save a copy of the standard file descriptors - saved_fds = [os.dup(fd) for fd in range(3)] + saved_fds = self._save_fds(range(3)) try: # duplicate the temp files over the standard fd's 0, 1, 2 for fd, temp_fd in enumerate(temp_fds): @@ -1643,9 +1639,7 @@ class POSIXProcessTestCase(BaseTestCase): out = os.read(stdout_no, 1024) err = support.strip_python_stderr(os.read(stderr_no, 1024)) finally: - for std, saved in enumerate(saved_fds): - os.dup2(saved, std) - os.close(saved) + self._restore_fds(saved_fds) self.assertEqual(out, b"got STDIN") self.assertEqual(err, b"err") @@ -1677,12 +1671,12 @@ class POSIXProcessTestCase(BaseTestCase): # Pure Python implementations keeps the message self.assertIsNone(subprocess._posixsubprocess) self.assertEqual(str(err), "surrogate:\uDCff") - except RuntimeError as err: + except subprocess.SubprocessError as err: # _posixsubprocess uses a default message self.assertIsNotNone(subprocess._posixsubprocess) self.assertEqual(str(err), "Exception occurred in preexec_fn.") else: - self.fail("Expected ValueError or RuntimeError") + self.fail("Expected ValueError or subprocess.SubprocessError") def test_undecodable_env(self): for key, value in (('test', 'abc\uDCFF'), ('test\uDCFF', '42')): @@ -1816,6 +1810,9 @@ class POSIXProcessTestCase(BaseTestCase): self.addCleanup(os.close, fd) open_fds.add(fd) + for fd in open_fds: + os.set_inheritable(fd, True) + p = subprocess.Popen([sys.executable, fd_status], stdout=subprocess.PIPE, close_fds=False) output, ignored = p.communicate() @@ -1860,6 +1857,8 @@ class POSIXProcessTestCase(BaseTestCase): fds = os.pipe() self.addCleanup(os.close, fds[0]) self.addCleanup(os.close, fds[1]) + os.set_inheritable(fds[0], True) + os.set_inheritable(fds[1], True) open_fds.update(fds) for fd in open_fds: @@ -1882,6 +1881,32 @@ class POSIXProcessTestCase(BaseTestCase): close_fds=False, pass_fds=(fd, ))) self.assertIn('overriding close_fds', str(context.warning)) + def test_pass_fds_inheritable(self): + script = support.findfile("fd_status.py", subdir="subprocessdata") + + inheritable, non_inheritable = os.pipe() + self.addCleanup(os.close, inheritable) + self.addCleanup(os.close, non_inheritable) + os.set_inheritable(inheritable, True) + os.set_inheritable(non_inheritable, False) + pass_fds = (inheritable, non_inheritable) + args = [sys.executable, script] + args += list(map(str, pass_fds)) + + p = subprocess.Popen(args, + stdout=subprocess.PIPE, close_fds=True, + pass_fds=pass_fds) + output, ignored = p.communicate() + fds = set(map(int, output.split(b','))) + + # the inheritable file descriptor must be inherited, so its inheritable + # flag must be set in the child process after fork() and before exec() + self.assertEqual(fds, set(pass_fds), "output=%a" % output) + + # inheritable flag must not be changed in the parent process + self.assertEqual(os.get_inheritable(inheritable), True) + self.assertEqual(os.get_inheritable(non_inheritable), False) + def test_stdout_stdin_are_single_inout_fd(self): with io.open(os.devnull, "r+") as inout: p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], @@ -1969,7 +1994,7 @@ class POSIXProcessTestCase(BaseTestCase): # let some time for the process to exit, and create a new Popen: this # should trigger the wait() of p time.sleep(0.2) - with self.assertRaises(EnvironmentError) as c: + with self.assertRaises(OSError) as c: with subprocess.Popen(['nonexisting_i_hope'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc: diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index fcccdf701a..77ec9f0178 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -1,19 +1,26 @@ """Do a minimal test of all the modules that aren't otherwise tested.""" - -from test import support +import importlib import sys +from test import support import unittest class TestUntestedModules(unittest.TestCase): - def test_at_least_import_untested_modules(self): + def test_untested_modules_can_be_imported(self): + untested = ('bdb', 'encodings', 'formatter', 'imghdr', + 'macurl2path', 'nturl2path', 'tabnanny') with support.check_warnings(quiet=True): - import bdb - import cgitb + for name in untested: + try: + support.import_module('test.test_{}'.format(name)) + except unittest.SkipTest: + importlib.import_module(name) + else: + self.fail('{} has tests even though test_sundry claims ' + 'otherwise'.format(name)) import distutils.bcppcompiler import distutils.ccompiler import distutils.cygwinccompiler - import distutils.emxccompiler import distutils.filelist if sys.platform.startswith('win'): import distutils.msvccompiler @@ -39,29 +46,13 @@ class TestUntestedModules(unittest.TestCase): import distutils.command.sdist import distutils.command.upload - import encodings - import formatter - import getpass import html.entities - import imghdr - import keyword - import macurl2path - import mailcap - import nturl2path - import os2emxpath - import pstats - import py_compile - import sndhdr - import tabnanny try: - import tty # not available on Windows + import tty # Not available on Windows except ImportError: if support.verbose: print("skipping tty") -def test_main(): - support.run_unittest(TestUntestedModules) - if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 1e272ee2cf..37fc2d9134 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -44,6 +44,11 @@ class G(A): class TestSuper(unittest.TestCase): + def tearDown(self): + # This fixes the damage that test_various___class___pathologies does. + nonlocal __class__ + __class__ = TestSuper + def test_basics_working(self): self.assertEqual(D().f(), 'ABCD') @@ -81,8 +86,7 @@ class TestSuper(unittest.TestCase): self.assertEqual(E().f(), 'AE') - @unittest.expectedFailure - def test___class___set(self): + def test_various___class___pathologies(self): # See issue #12370 class X(A): def f(self): @@ -91,6 +95,31 @@ class TestSuper(unittest.TestCase): x = X() self.assertEqual(x.f(), 'A') self.assertEqual(x.__class__, 413) + class X: + x = __class__ + def f(): + __class__ + self.assertIs(X.x, type(self)) + with self.assertRaises(NameError) as e: + exec("""class X: + __class__ + def f(): + __class__""", globals(), {}) + self.assertIs(type(e.exception), NameError) # Not UnboundLocalError + class X: + global __class__ + __class__ = 42 + def f(): + __class__ + self.assertEqual(globals()["__class__"], 42) + del globals()["__class__"] + self.assertNotIn("__class__", X.__dict__) + class X: + nonlocal __class__ + __class__ = 42 + def f(): + __class__ + self.assertEqual(__class__, 42) def test___class___instancemethod(self): # See issue #14857 diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 4edb1a8e01..16b660bce1 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -306,6 +306,7 @@ class TestSupport(unittest.TestCase): # args_from_interpreter_flags # can_symlink # skip_unless_symlink + # SuppressCrashReport def test_main(): diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 5926b69c93..a9d3628819 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -33,7 +33,7 @@ SyntaxError: invalid syntax >>> None = 1 Traceback (most recent call last): -SyntaxError: assignment to keyword +SyntaxError: can't assign to keyword It's a syntax error to assign to the empty tuple. Why isn't it an error to assign to the empty list? It will always raise some error at @@ -233,7 +233,7 @@ Traceback (most recent call last): SyntaxError: can't assign to generator expression >>> None += 1 Traceback (most recent call last): -SyntaxError: assignment to keyword +SyntaxError: can't assign to keyword >>> f() += 1 Traceback (most recent call last): SyntaxError: can't assign to function call diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index bc4c0ad46d..8437745cb9 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -6,6 +6,8 @@ import textwrap import warnings import operator import codecs +import gc +import sysconfig # count the number of test runs, used to create unique # strings to intern in test_intern() @@ -248,7 +250,7 @@ class SysModuleTest(unittest.TestCase): sys.setrecursionlimit(%d) f()""") - with test.support.suppress_crash_popup(): + with test.support.SuppressCrashReport(): for i in (50, 1000): sub = subprocess.Popen([sys.executable, '-c', code % i], stderr=subprocess.PIPE) @@ -480,7 +482,7 @@ class SysModuleTest(unittest.TestCase): def test_thread_info(self): info = sys.thread_info self.assertEqual(len(info), 3) - self.assertIn(info.name, ('nt', 'os2', 'pthread', 'solaris', None)) + self.assertIn(info.name, ('nt', 'pthread', 'solaris', None)) self.assertIn(info.lock, ('semaphore', 'mutex+cond', None)) def test_43581(self): @@ -513,7 +515,7 @@ class SysModuleTest(unittest.TestCase): attrs = ("debug", "inspect", "interactive", "optimize", "dont_write_bytecode", "no_user_site", "no_site", "ignore_environment", "verbose", - "bytes_warning", "quiet", "hash_randomization") + "bytes_warning", "quiet", "hash_randomization", "isolated") for attr in attrs: self.assertTrue(hasattr(sys.flags, attr), attr) self.assertEqual(type(getattr(sys.flags, attr)), int, attr) @@ -542,6 +544,42 @@ class SysModuleTest(unittest.TestCase): out = p.communicate()[0].strip() self.assertEqual(out, b'?') + env["PYTHONIOENCODING"] = "ascii" + p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env) + out, err = p.communicate() + self.assertEqual(out, b'') + self.assertIn(b'UnicodeEncodeError:', err) + self.assertIn(rb"'\xa2'", err) + + env["PYTHONIOENCODING"] = "ascii:" + p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env) + out, err = p.communicate() + self.assertEqual(out, b'') + self.assertIn(b'UnicodeEncodeError:', err) + self.assertIn(rb"'\xa2'", err) + + env["PYTHONIOENCODING"] = ":surrogateescape" + p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xdcbd))'], + stdout=subprocess.PIPE, env=env) + out = p.communicate()[0].strip() + self.assertEqual(out, b'\xbd') + + @unittest.skipUnless(test.support.FS_NONASCII, + 'requires OS support of non-ASCII encodings') + def test_ioencoding_nonascii(self): + env = dict(os.environ) + + env["PYTHONIOENCODING"] = "" + p = subprocess.Popen([sys.executable, "-c", + 'print(%a)' % test.support.FS_NONASCII], + stdout=subprocess.PIPE, env=env) + out = p.communicate()[0].strip() + self.assertEqual(out, os.fsencode(test.support.FS_NONASCII)) + @unittest.skipIf(sys.base_prefix != sys.prefix, 'Test is not venv-compatible') def test_executable(self): @@ -609,6 +647,36 @@ class SysModuleTest(unittest.TestCase): ret, out, err = assert_python_ok(*args) self.assertIn(b"free PyDictObjects", err) + @unittest.skipUnless(hasattr(sys, "getallocatedblocks"), + "sys.getallocatedblocks unavailable on this build") + def test_getallocatedblocks(self): + # Some sanity checks + with_pymalloc = sysconfig.get_config_var('WITH_PYMALLOC') + a = sys.getallocatedblocks() + self.assertIs(type(a), int) + if with_pymalloc: + self.assertGreater(a, 0) + else: + # When WITH_PYMALLOC isn't available, we don't know anything + # about the underlying implementation: the function might + # return 0 or something greater. + self.assertGreaterEqual(a, 0) + try: + # While we could imagine a Python session where the number of + # multiple buffer objects would exceed the sharing of references, + # it is unlikely to happen in a normal test run. + self.assertLess(a, sys.gettotalrefcount()) + except AttributeError: + # gettotalrefcount() not available + pass + gc.collect() + b = sys.getallocatedblocks() + self.assertLessEqual(b, a) + gc.collect() + c = sys.getallocatedblocks() + self.assertIn(c, range(b - 50, b + 50)) + + class SizeofTest(unittest.TestCase): def setUp(self): @@ -653,7 +721,7 @@ class SizeofTest(unittest.TestCase): samples = [b'', b'u'*100000] for sample in samples: x = bytearray(sample) - check(x, vsize('inP') + x.__alloc__()) + check(x, vsize('n2Pi') + x.__alloc__()) # bytearray_iterator check(iter(bytearray()), size('nP')) # cell @@ -732,7 +800,7 @@ class SizeofTest(unittest.TestCase): nfrees = len(x.f_code.co_freevars) extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\ ncells + nfrees - 1 - check(x, vsize('12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P')) + check(x, vsize('13P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P')) # function def func(): pass check(func, size('12P')) @@ -778,7 +846,7 @@ class SizeofTest(unittest.TestCase): # memoryview check(memoryview(b''), size('Pnin 2P2n2i5P 3cPn')) # module - check(unittest, size('PnP')) + check(unittest, size('PnPPP')) # None check(None, size('')) # NotImplementedType @@ -832,11 +900,11 @@ class SizeofTest(unittest.TestCase): check((1,2,3), vsize('') + 3*self.P) # type # static type: PyTypeObject - s = vsize('P2n15Pl4Pn9Pn11PI') + s = vsize('P2n15Pl4Pn9Pn11PIP') check(int, s) # (PyTypeObject + PyNumberMethods + PyMappingMethods + # PySequenceMethods + PyBufferProcs + 4P) - s = vsize('P2n15Pl4Pn9Pn11PI') + struct.calcsize('34P 3P 10P 2P 4P') + s = vsize('P2n15Pl4Pn9Pn11PIP') + struct.calcsize('34P 3P 10P 2P 4P') # Separate block for PyDictKeysObject with 4 entries s += struct.calcsize("2nPn") + 4*struct.calcsize("n2P") # class diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 92193602f9..5293649878 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -234,7 +234,7 @@ class TestSysConfig(unittest.TestCase): self.assertTrue(os.path.isfile(config_h), config_h) def test_get_scheme_names(self): - wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user', + wanted = ('nt', 'nt_user', 'osx_framework_user', 'posix_home', 'posix_prefix', 'posix_user') self.assertEqual(get_scheme_names(), wanted) @@ -305,14 +305,13 @@ class TestSysConfig(unittest.TestCase): if 'MACOSX_DEPLOYMENT_TARGET' in env: del env['MACOSX_DEPLOYMENT_TARGET'] - with open('/dev/null', 'w') as devnull_fp: - p = subprocess.Popen([ - sys.executable, '-c', - 'import sysconfig; print(sysconfig.get_platform())', - ], - stdout=subprocess.PIPE, - stderr=devnull_fp, - env=env) + p = subprocess.Popen([ + sys.executable, '-c', + 'import sysconfig; print(sysconfig.get_platform())', + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + env=env) test_platform = p.communicate()[0].strip() test_platform = test_platform.decode('utf-8') status = p.wait() @@ -325,20 +324,19 @@ class TestSysConfig(unittest.TestCase): env = os.environ.copy() env['MACOSX_DEPLOYMENT_TARGET'] = '10.1' - with open('/dev/null') as dev_null: - p = subprocess.Popen([ - sys.executable, '-c', - 'import sysconfig; print(sysconfig.get_platform())', - ], - stdout=subprocess.PIPE, - stderr=dev_null, - env=env) - test_platform = p.communicate()[0].strip() - test_platform = test_platform.decode('utf-8') - status = p.wait() - - self.assertEqual(status, 0) - self.assertEqual(my_platform, test_platform) + p = subprocess.Popen([ + sys.executable, '-c', + 'import sysconfig; print(sysconfig.get_platform())', + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + env=env) + test_platform = p.communicate()[0].strip() + test_platform = test_platform.decode('utf-8') + status = p.wait() + + self.assertEqual(status, 0) + self.assertEqual(my_platform, test_platform) def test_srcdir(self): # See Issues #15322, #15364. diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 238175ff3c..2f2bf6b8be 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1732,20 +1732,20 @@ class ContextManagerTest(unittest.TestCase): self.assertTrue(tar.closed, "context manager failed") def test_closed(self): - # The __enter__() method is supposed to raise IOError + # The __enter__() method is supposed to raise OSError # if the TarFile object is already closed. tar = tarfile.open(tarname) tar.close() - with self.assertRaises(IOError): + with self.assertRaises(OSError): with tar: pass def test_exception(self): - # Test if the IOError exception is passed through properly. + # Test if the OSError exception is passed through properly. with self.assertRaises(Exception) as exc: with tarfile.open(tarname) as tar: - raise IOError - self.assertIsInstance(exc.exception, IOError, + raise OSError + self.assertIsInstance(exc.exception, OSError, "wrong exception raised in context manager") self.assertTrue(tar.closed, "context manager failed") diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 2cdac2bd97..cf717d8ce5 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -254,37 +254,6 @@ class TclTest(unittest.TestCase): for arg, res in testcases: self.assertEqual(split(arg), res, msg=arg) - def test_merge(self): - with support.check_warnings(('merge is deprecated', - DeprecationWarning)): - merge = self.interp.tk.merge - call = self.interp.tk.call - testcases = [ - ((), ''), - (('a',), 'a'), - ((2,), '2'), - (('',), '{}'), - ('{', '\\{'), - (('a', 'b', 'c'), 'a b c'), - ((' ', '\t', '\r', '\n'), '{ } {\t} {\r} {\n}'), - (('a', ' ', 'c'), 'a { } c'), - (('a', '€'), 'a €'), - (('a', '\U000104a2'), 'a \U000104a2'), - (('a', b'\xe2\x82\xac'), 'a €'), - (('a', ('b', 'c')), 'a {b c}'), - (('a', 2), 'a 2'), - (('a', 3.4), 'a 3.4'), - (('a', (2, 3.4)), 'a {2 3.4}'), - ((), ''), - ((call('list', 1, '2', (3.4,)),), '{1 2 3.4}'), - ((call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)),), - '{12 € € 3.4}'), - ] - for args, res in testcases: - self.assertEqual(merge(*args), res, msg=args) - self.assertRaises(UnicodeDecodeError, merge, b'\x80') - self.assertRaises(UnicodeEncodeError, merge, '\udc80') - class BigmemTclTest(unittest.TestCase): diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 4ff56b8bf6..1ee36e223b 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -36,7 +36,7 @@ else: # Common functionality. class BaseTestCase(unittest.TestCase): - str_check = re.compile(r"[a-zA-Z0-9_-]{6}$") + str_check = re.compile(r"^[a-z0-9_-]{8}$") def setUp(self): self._warnings_manager = support.check_warnings() @@ -63,7 +63,7 @@ class BaseTestCase(unittest.TestCase): nbase = nbase[len(pre):len(nbase)-len(suf)] self.assertTrue(self.str_check.match(nbase), - "random string '%s' does not match /^[a-zA-Z0-9_-]{6}$/" + "random string '%s' does not match ^[a-z0-9_-]{8}$" % nbase) @@ -152,7 +152,7 @@ class TestRandomNameSequence(BaseTestCase): # via any bugs above try: os.kill(pid, signal.SIGKILL) - except EnvironmentError: + except OSError: pass os.close(read_fd) os.close(write_fd) @@ -191,7 +191,7 @@ class TestCandidateTempdirList(BaseTestCase): try: dirname = os.getcwd() - except (AttributeError, os.error): + except (AttributeError, OSError): dirname = os.curdir self.assertIn(dirname, cand) @@ -332,7 +332,7 @@ class TestMkstempInner(BaseTestCase): file = self.do_create() mode = stat.S_IMODE(os.stat(file.name).st_mode) expected = 0o600 - if sys.platform in ('win32', 'os2emx'): + if sys.platform == 'win32': # There's no distinction among 'user', 'group' and 'world'; # replicate the 'user' bits. user = expected >> 6 @@ -350,6 +350,7 @@ class TestMkstempInner(BaseTestCase): v="q" file = self.do_create() + self.assertEqual(os.get_inheritable(file.fd), False) fd = "%d" % file.fd try: @@ -366,7 +367,7 @@ class TestMkstempInner(BaseTestCase): # On Windows a spawn* /path/ with embedded spaces shouldn't be quoted, # but an arg with embedded spaces should be decorated with double # quotes on each end - if sys.platform in ('win32',): + if sys.platform == 'win32': decorated = '"%s"' % sys.executable tester = '"%s"' % tester else: @@ -566,7 +567,7 @@ class TestMkdtemp(BaseTestCase): mode = stat.S_IMODE(os.stat(dir).st_mode) mode &= 0o777 # Mask off sticky bits inherited from /tmp expected = 0o700 - if sys.platform in ('win32', 'os2emx'): + if sys.platform == 'win32': # There's no distinction among 'user', 'group' and 'world'; # replicate the 'user' bits. user = expected >> 6 diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py index c86f5cfae8..36c15cc03c 100644 --- a/Lib/test/test_textwrap.py +++ b/Lib/test/test_textwrap.py @@ -9,9 +9,8 @@ # import unittest -from test import support -from textwrap import TextWrapper, wrap, fill, dedent, indent +from textwrap import TextWrapper, wrap, fill, dedent, indent, shorten class BaseTestCase(unittest.TestCase): @@ -43,6 +42,10 @@ class BaseTestCase(unittest.TestCase): "\nexpected %r\n" "but got %r" % (expect, result)) + def check_shorten(self, text, width, expect, **kwargs): + result = shorten(text, width, **kwargs) + self.check(result, expect) + class WrapTestCase(BaseTestCase): @@ -777,12 +780,59 @@ class IndentTestCase(unittest.TestCase): self.assertEqual(indent(text, prefix, predicate), expect) -def test_main(): - support.run_unittest(WrapTestCase, - LongWordTestCase, - IndentTestCases, - DedentTestCase, - IndentTestCase) +class ShortenTestCase(BaseTestCase): + + def test_simple(self): + # Simple case: just words, spaces, and a bit of punctuation + text = "Hello there, how are you this fine day? I'm glad to hear it!" + + self.check_shorten(text, 18, "Hello there, [...]") + self.check_shorten(text, len(text), text) + self.check_shorten(text, len(text) - 1, + "Hello there, how are you this fine day? " + "I'm glad to [...]") + + def test_placeholder(self): + text = "Hello there, how are you this fine day? I'm glad to hear it!" + + self.check_shorten(text, 17, "Hello there,$$", placeholder='$$') + self.check_shorten(text, 18, "Hello there, how$$", placeholder='$$') + self.check_shorten(text, 18, "Hello there, $$", placeholder=' $$') + self.check_shorten(text, len(text), text, placeholder='$$') + self.check_shorten(text, len(text) - 1, + "Hello there, how are you this fine day? " + "I'm glad to hear$$", placeholder='$$') + + def test_empty_string(self): + self.check_shorten("", 6, "") + + def test_whitespace(self): + # Whitespace collapsing + text = """ + This is a paragraph that already has + line breaks and \t tabs too.""" + self.check_shorten(text, 62, + "This is a paragraph that already has line " + "breaks and tabs too.") + self.check_shorten(text, 61, + "This is a paragraph that already has line " + "breaks and [...]") + + self.check_shorten("hello world! ", 12, "hello world!") + self.check_shorten("hello world! ", 11, "hello [...]") + # The leading space is trimmed from the placeholder + # (it would be ugly otherwise). + self.check_shorten("hello world! ", 10, "[...]") + + def test_width_too_small_for_placeholder(self): + wrapper = TextWrapper(width=8) + wrapper.shorten("x" * 20, placeholder="(......)") + with self.assertRaises(ValueError): + wrapper.shorten("x" * 20, placeholder="(.......)") + + def test_first_word_too_long_but_placeholder_fits(self): + self.check_shorten("Helloo", 5, "[...]") + if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index a191e157bc..f9a721b03b 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -68,7 +68,7 @@ class ThreadRunningTests(BasicThreadTest): thread.stack_size(0) self.assertEqual(thread.stack_size(), 0, "stack_size not reset to default") - if os.name not in ("nt", "os2", "posix"): + if os.name not in ("nt", "posix"): return tss_supported = True diff --git a/Lib/test/test_threaded_import.py b/Lib/test/test_threaded_import.py index 6c2965ba6e..3d961b5ee2 100644 --- a/Lib/test/test_threaded_import.py +++ b/Lib/test/test_threaded_import.py @@ -5,8 +5,8 @@ # complains several times about module random having no attribute # randrange, and then Python hangs. +import _imp as imp import os -import imp import importlib import sys import time diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 0ebeb39cbd..7bd3082842 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -11,6 +11,7 @@ import re import sys _thread = import_module('_thread') threading = import_module('threading') +import _testcapi import time import unittest import weakref @@ -20,6 +21,15 @@ import subprocess from test import lock_tests + +# Between fork() and exec(), only async-safe functions are allowed (issues +# #12316 and #11870), and fork() from a worker thread is known to trigger +# problems with some operating systems (issue #3863): skip problematic tests +# on platforms known to behave badly. +platforms_to_skip = ('freebsd4', 'freebsd5', 'freebsd6', 'netbsd5', + 'hp-ux11') + + # A trivial mutable counter. class Counter(object): def __init__(self): @@ -99,7 +109,7 @@ class ThreadTests(BaseTestCase): if verbose: print('waiting for all tasks to complete') for t in threads: - t.join(NUMTASKS) + t.join() self.assertTrue(not t.is_alive()) self.assertNotEqual(t.ident, 0) self.assertFalse(t.ident is None) @@ -467,6 +477,127 @@ class ThreadTests(BaseTestCase): pid, status = os.waitpid(pid, 0) self.assertEqual(0, status) + def test_main_thread(self): + main = threading.main_thread() + self.assertEqual(main.name, 'MainThread') + self.assertEqual(main.ident, threading.current_thread().ident) + self.assertEqual(main.ident, threading.get_ident()) + + def f(): + self.assertNotEqual(threading.main_thread().ident, + threading.current_thread().ident) + th = threading.Thread(target=f) + th.start() + th.join() + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork(self): + code = """if 1: + import os, threading + + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + else: + os.waitpid(pid, 0) + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err, b"") + self.assertEqual(data, "MainThread\nTrue\nTrue\n") + + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork_from_nonmain_thread(self): + code = """if 1: + import os, threading, sys + + def f(): + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + # stdout is fully buffered because not a tty, + # we have to flush before exit. + sys.stdout.flush() + else: + os.waitpid(pid, 0) + + th = threading.Thread(target=f) + th.start() + th.join() + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err, b"") + self.assertEqual(data, "Thread-1\nTrue\nTrue\n") + + def test_tstate_lock(self): + # Test an implementation detail of Thread objects. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + time.sleep(0.01) + # The tstate lock is None until the thread is started + t = threading.Thread(target=f) + self.assertIs(t._tstate_lock, None) + t.start() + started.acquire() + self.assertTrue(t.is_alive()) + # The tstate lock can't be acquired when the thread is running + # (or suspended). + tstate_lock = t._tstate_lock + self.assertFalse(tstate_lock.acquire(timeout=0), False) + finish.release() + # When the thread ends, the state_lock can be successfully + # acquired. + self.assertTrue(tstate_lock.acquire(timeout=5), False) + # But is_alive() is still True: we hold _tstate_lock now, which + # prevents is_alive() from knowing the thread's end-of-life C code + # is done. + self.assertTrue(t.is_alive()) + # Let is_alive() find out the C code is done. + tstate_lock.release() + self.assertFalse(t.is_alive()) + # And verify the thread disposed of _tstate_lock. + self.assertTrue(t._tstate_lock is None) + + def test_repr_stopped(self): + # Verify that "stopped" shows up in repr(Thread) appropriately. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + t = threading.Thread(target=f) + t.start() + started.acquire() + self.assertIn("started", repr(t)) + finish.release() + # "stopped" should appear in the repr in a reasonable amount of time. + # Implementation detail: as of this writing, that's trivially true + # if .join() is called, and almost trivially true if .is_alive() is + # called. The detail we're testing here is that "stopped" shows up + # "all on its own". + LOOKING_FOR = "stopped" + for i in range(500): + if LOOKING_FOR in repr(t): + break + time.sleep(0.01) + self.assertIn(LOOKING_FOR, repr(t)) # we waited at least 5 seconds def test_BoundedSemaphore_limit(self): # BoundedSemaphore should raise ValueError if released too often. @@ -488,13 +619,6 @@ class ThreadTests(BaseTestCase): class ThreadJoinOnShutdown(BaseTestCase): - # Between fork() and exec(), only async-safe functions are allowed (issues - # #12316 and #11870), and fork() from a worker thread is known to trigger - # problems with some operating systems (issue #3863): skip problematic tests - # on platforms known to behave badly. - platforms_to_skip = ('freebsd4', 'freebsd5', 'freebsd6', 'netbsd5', - 'os2emx', 'hp-ux11') - def _run_and_join(self, script): script = """if 1: import sys, os, time, threading @@ -566,144 +690,8 @@ class ThreadJoinOnShutdown(BaseTestCase): """ self._run_and_join(script) - def assertScriptHasOutput(self, script, expected_output): - rc, out, err = assert_python_ok("-c", script) - data = out.decode().replace('\r', '') - self.assertEqual(data, expected_output) - - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_4_joining_across_fork_in_worker_thread(self): - # There used to be a possible deadlock when forking from a child - # thread. See http://bugs.python.org/issue6643. - - # The script takes the following steps: - # - The main thread in the parent process starts a new thread and then - # tries to join it. - # - The join operation acquires the Lock inside the thread's _block - # Condition. (See threading.py:Thread.join().) - # - We stub out the acquire method on the condition to force it to wait - # until the child thread forks. (See LOCK ACQUIRED HERE) - # - The child thread forks. (See LOCK HELD and WORKER THREAD FORKS - # HERE) - # - The main thread of the parent process enters Condition.wait(), - # which releases the lock on the child thread. - # - The child process returns. Without the necessary fix, when the - # main thread of the child process (which used to be the child thread - # in the parent process) attempts to exit, it will try to acquire the - # lock in the Thread._block Condition object and hang, because the - # lock was held across the fork. - - script = """if 1: - import os, time, threading - - finish_join = False - start_fork = False - - def worker(): - # Wait until this thread's lock is acquired before forking to - # create the deadlock. - global finish_join - while not start_fork: - time.sleep(0.01) - # LOCK HELD: Main thread holds lock across this call. - childpid = os.fork() - finish_join = True - if childpid != 0: - # Parent process just waits for child. - os.waitpid(childpid, 0) - # Child process should just return. - - w = threading.Thread(target=worker) - - # Stub out the private condition variable's lock acquire method. - # This acquires the lock and then waits until the child has forked - # before returning, which will release the lock soon after. If - # someone else tries to fix this test case by acquiring this lock - # before forking instead of resetting it, the test case will - # deadlock when it shouldn't. - condition = w._block - orig_acquire = condition.acquire - call_count_lock = threading.Lock() - call_count = 0 - def my_acquire(): - global call_count - global start_fork - orig_acquire() # LOCK ACQUIRED HERE - start_fork = True - if call_count == 0: - while not finish_join: - time.sleep(0.01) # WORKER THREAD FORKS HERE - with call_count_lock: - call_count += 1 - condition.acquire = my_acquire - - w.start() - w.join() - print('end of main') - """ - self.assertScriptHasOutput(script, "end of main\n") - - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_5_clear_waiter_locks_to_avoid_crash(self): - # Check that a spawned thread that forks doesn't segfault on certain - # platforms, namely OS X. This used to happen if there was a waiter - # lock in the thread's condition variable's waiters list. Even though - # we know the lock will be held across the fork, it is not safe to - # release locks held across forks on all platforms, so releasing the - # waiter lock caused a segfault on OS X. Furthermore, since locks on - # OS X are (as of this writing) implemented with a mutex + condition - # variable instead of a semaphore, while we know that the Python-level - # lock will be acquired, we can't know if the internal mutex will be - # acquired at the time of the fork. - - script = """if True: - import os, time, threading - - start_fork = False - - def worker(): - # Wait until the main thread has attempted to join this thread - # before continuing. - while not start_fork: - time.sleep(0.01) - childpid = os.fork() - if childpid != 0: - # Parent process just waits for child. - (cpid, rc) = os.waitpid(childpid, 0) - assert cpid == childpid - assert rc == 0 - print('end of worker thread') - else: - # Child process should just return. - pass - - w = threading.Thread(target=worker) - - # Stub out the private condition variable's _release_save method. - # This releases the condition's lock and flips the global that - # causes the worker to fork. At this point, the problematic waiter - # lock has been acquired once by the waiter and has been put onto - # the waiters list. - condition = w._block - orig_release_save = condition._release_save - def my_release_save(): - global start_fork - orig_release_save() - # Waiter lock held here, condition lock released. - start_fork = True - condition._release_save = my_release_save - - w.start() - w.join() - print('end of main thread') - """ - output = "end of worker thread\nend of main thread\n" - self.assertScriptHasOutput(script, output) - - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_6_daemon_threads(self): + def test_4_daemon_threads(self): # Check that a daemon thread cannot crash the interpreter on shutdown # by manipulating internal structures that are being disposed of in # the main thread. @@ -769,6 +757,111 @@ class ThreadJoinOnShutdown(BaseTestCase): for t in threads: t.join() + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_clear_threads_states_after_fork(self): + # Issue #17094: check that threads states are cleared after fork() + + # start a bunch of threads + threads = [] + for i in range(16): + t = threading.Thread(target=lambda : time.sleep(0.3)) + threads.append(t) + t.start() + + pid = os.fork() + if pid == 0: + # check that threads states have been cleared + if len(sys._current_frames()) == 1: + os._exit(0) + else: + os._exit(1) + else: + _, status = os.waitpid(pid, 0) + self.assertEqual(0, status) + + for t in threads: + t.join() + + +class SubinterpThreadingTests(BaseTestCase): + + def test_threads_join(self): + # Non-daemon threads should be joined at subinterpreter shutdown + # (issue #18808) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = _testcapi.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + + def test_threads_join_2(self): + # Same as above, but a delay gets introduced after the thread's + # Python code returned but before the thread state is deleted. + # To achieve this, we register a thread-local object which sleeps + # a bit when deallocated. + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + class Sleeper: + def __del__(self): + time.sleep(0.05) + + tls = threading.local() + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + tls.x = Sleeper() + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = _testcapi.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + + def test_daemon_threads_fatal_error(self): + subinterp_code = r"""if 1: + import os + import threading + import time + + def f(): + # Make sure the daemon thread is still running when + # Py_EndInterpreter is called. + time.sleep(10) + threading.Thread(target=f, daemon=True).start() + """ + script = r"""if 1: + import _testcapi + + _testcapi.run_in_subinterp(%r) + """ % (subinterp_code,) + with test.support.SuppressCrashReport(): + rc, out, err = assert_python_failure("-c", script) + self.assertIn("Fatal Python error: Py_EndInterpreter: " + "not the last thread", err.decode()) + class ThreadingExceptionTests(BaseTestCase): # A RuntimeError should be raised if Thread.start() is called diff --git a/Lib/test/test_threadsignals.py b/Lib/test/test_threadsignals.py index f975a75e85..b1004e6689 100644 --- a/Lib/test/test_threadsignals.py +++ b/Lib/test/test_threadsignals.py @@ -8,7 +8,7 @@ from test.support import run_unittest, import_module thread = import_module('_thread') import time -if sys.platform[:3] in ('win', 'os2') or sys.platform=='riscos': +if (sys.platform[:3] == 'win'): raise unittest.SkipTest("Can't test signal on %s" % sys.platform) process_pid = os.getpid() diff --git a/Lib/test/test_timeout.py b/Lib/test/test_timeout.py index dcf201b507..703c43ab2b 100644 --- a/Lib/test/test_timeout.py +++ b/Lib/test/test_timeout.py @@ -207,7 +207,7 @@ class TCPTimeoutTestCase(TimeoutTestCase): sock.connect((whitehole)) except socket.timeout: pass - except IOError as err: + except OSError as err: if err.errno == errno.ECONNREFUSED: skip = False finally: diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 5bce2af68a..96ef951139 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -150,30 +150,74 @@ class SyntaxTracebackCases(unittest.TestCase): class TracebackFormatTests(unittest.TestCase): - def test_traceback_format(self): + def some_exception(self): + raise KeyError('blah') + + def check_traceback_format(self, cleanup_func=None): try: - raise KeyError('blah') + self.some_exception() except KeyError: type_, value, tb = sys.exc_info() + if cleanup_func is not None: + # Clear the inner frames, not this one + cleanup_func(tb.tb_next) traceback_fmt = 'Traceback (most recent call last):\n' + \ ''.join(traceback.format_tb(tb)) file_ = StringIO() traceback_print(tb, file_) python_fmt = file_.getvalue() + # Call all _tb and _exc functions + with captured_output("stderr") as tbstderr: + traceback.print_tb(tb) + tbfile = StringIO() + traceback.print_tb(tb, file=tbfile) + with captured_output("stderr") as excstderr: + traceback.print_exc() + excfmt = traceback.format_exc() + excfile = StringIO() + traceback.print_exc(file=excfile) else: raise Error("unable to create test traceback string") # Make sure that Python and the traceback module format the same thing self.assertEqual(traceback_fmt, python_fmt) + # Now verify the _tb func output + self.assertEqual(tbstderr.getvalue(), tbfile.getvalue()) + # Now verify the _exc func output + self.assertEqual(excstderr.getvalue(), excfile.getvalue()) + self.assertEqual(excfmt, excfile.getvalue()) # Make sure that the traceback is properly indented. tb_lines = python_fmt.splitlines() - self.assertEqual(len(tb_lines), 3) - banner, location, source_line = tb_lines + self.assertEqual(len(tb_lines), 5) + banner = tb_lines[0] + location, source_line = tb_lines[-2:] self.assertTrue(banner.startswith('Traceback')) self.assertTrue(location.startswith(' File')) self.assertTrue(source_line.startswith(' raise')) + def test_traceback_format(self): + self.check_traceback_format() + + def test_traceback_format_with_cleared_frames(self): + # Check that traceback formatting also works with a clear()ed frame + def cleanup_tb(tb): + tb.tb_frame.clear() + self.check_traceback_format(cleanup_tb) + + def test_stack_format(self): + # Verify _stack functions. Note we have to use _getframe(1) to + # compare them without this frame appearing in the output + with captured_output("stderr") as ststderr: + traceback.print_stack(sys._getframe(1)) + stfile = StringIO() + traceback.print_stack(sys._getframe(1), file=stfile) + self.assertEqual(ststderr.getvalue(), stfile.getvalue()) + + stfmt = traceback.format_stack(sys._getframe(1)) + + self.assertEqual(ststderr.getvalue(), "".join(stfmt)) + cause_message = ( "\nThe above exception was the direct cause " @@ -344,6 +388,36 @@ class CExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): return s.getvalue() +class MiscTracebackCases(unittest.TestCase): + # + # Check non-printing functions in traceback module + # + + def test_clear(self): + def outer(): + middle() + def middle(): + inner() + def inner(): + i = 1 + 1/0 + + try: + outer() + except: + type_, value, tb = sys.exc_info() + + # Initial assertion: there's one local in the inner frame. + inner_frame = tb.tb_next.tb_next.tb_next.tb_frame + self.assertEqual(len(inner_frame.f_locals), 1) + + # Clear traceback frames + traceback.clear_frames(tb) + + # Local variable dict should now be empty. + self.assertEqual(len(inner_frame.f_locals), 0) + + def test_main(): run_unittest(__name__) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 3ee4c6bc5f..ec10752e6a 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -2,6 +2,7 @@ from test.support import run_unittest, run_with_locale import collections +import pickle import locale import sys import types @@ -1077,9 +1078,19 @@ class SimpleNamespaceTests(unittest.TestCase): ns2 = types.SimpleNamespace() ns2.x = "spam" ns2._y = 5 + name = "namespace" - self.assertEqual(repr(ns1), "namespace(w=3, x=1, y=2)") - self.assertEqual(repr(ns2), "namespace(_y=5, x='spam')") + self.assertEqual(repr(ns1), "{name}(w=3, x=1, y=2)".format(name=name)) + self.assertEqual(repr(ns2), "{name}(_y=5, x='spam')".format(name=name)) + + def test_equal(self): + ns1 = types.SimpleNamespace(x=1) + ns2 = types.SimpleNamespace() + ns2.x = 1 + + self.assertEqual(types.SimpleNamespace(), types.SimpleNamespace()) + self.assertEqual(ns1, ns2) + self.assertNotEqual(ns2, types.SimpleNamespace()) def test_nested(self): ns1 = types.SimpleNamespace(a=1, b=2) @@ -1117,11 +1128,12 @@ class SimpleNamespaceTests(unittest.TestCase): ns1.spam = ns1 ns2.spam = ns3 ns3.spam = ns2 + name = "namespace" + repr1 = "{name}(c='cookie', spam={name}(...))".format(name=name) + repr2 = "{name}(spam={name}(spam={name}(...), x=1))".format(name=name) - self.assertEqual(repr(ns1), - "namespace(c='cookie', spam=namespace(...))") - self.assertEqual(repr(ns2), - "namespace(spam=namespace(spam=namespace(...), x=1))") + self.assertEqual(repr(ns1), repr1) + self.assertEqual(repr(ns2), repr2) def test_as_dict(self): ns = types.SimpleNamespace(spam='spamspamspam') @@ -1144,6 +1156,19 @@ class SimpleNamespaceTests(unittest.TestCase): self.assertIs(type(spam), Spam) self.assertEqual(vars(spam), {'ham': 8, 'eggs': 9}) + def test_pickle(self): + ns = types.SimpleNamespace(breakfast="spam", lunch="spam") + + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + pname = "protocol {}".format(protocol) + try: + ns_pickled = pickle.dumps(ns, protocol) + except TypeError as e: + raise TypeError(pname) from e + ns_roundtrip = pickle.loads(ns_pickled) + + self.assertEqual(ns, ns_roundtrip, pname) + def test_main(): run_unittest(TypesTests, MappingProxyTests, ClassCreationTests, diff --git a/Lib/test/test_ucn.py b/Lib/test/test_ucn.py index 2e6374561f..59bde7475a 100644 --- a/Lib/test/test_ucn.py +++ b/Lib/test/test_ucn.py @@ -173,7 +173,7 @@ class UnicodeNamesTest(unittest.TestCase): try: testdata = support.open_urlresource(url, encoding="utf-8", check=check_version) - except (IOError, HTTPException): + except (OSError, HTTPException): self.skipTest("Could not retrieve " + url) self.addCleanup(testdata.close) for line in testdata: diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 9dc3438bea..c8fa7f3c94 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -7,6 +7,7 @@ Written by Marc-Andre Lemburg (mal@lemburg.com). """#" import _string import codecs +import itertools import struct import sys import unittest @@ -31,6 +32,16 @@ def search_function(encoding): return None codecs.register(search_function) +def duplicate_string(text): + """ + Try to get a fresh clone of the specified text: + new object with a reference count of 1. + + This is a best-effort: latin1 single letters and the empty + string ('') are singletons and cannot be cloned. + """ + return text.encode().decode() + class UnicodeTest(string_tests.CommonTest, string_tests.MixinStrUnicodeUserStringTest, string_tests.MixinStrUnicodeTest, @@ -863,11 +874,9 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual('{0:d}'.format(G('data')), 'G(data)') self.assertEqual('{0!s}'.format(G('data')), 'string is data') - msg = 'object.__format__ with a non-empty format string is deprecated' - with support.check_warnings((msg, DeprecationWarning)): - self.assertEqual('{0:^10}'.format(E('data')), ' E(data) ') - self.assertEqual('{0:^10s}'.format(E('data')), ' E(data) ') - self.assertEqual('{0:>15s}'.format(G('data')), ' string is data') + self.assertRaises(TypeError, '{0:^10}'.format, E('data')) + self.assertRaises(TypeError, '{0:^10s}'.format, E('data')) + self.assertRaises(TypeError, '{0:>15s}'.format, G('data')) self.assertEqual("{0:date: %Y-%m-%d}".format(I(year=2007, month=8, @@ -903,7 +912,7 @@ class UnicodeTest(string_tests.CommonTest, self.assertRaises(ValueError, "{0".format) self.assertRaises(IndexError, "{0.}".format) self.assertRaises(ValueError, "{0.}".format, 0) - self.assertRaises(IndexError, "{0[}".format) + self.assertRaises(ValueError, "{0[}".format) self.assertRaises(ValueError, "{0[}".format, []) self.assertRaises(KeyError, "{0]}".format) self.assertRaises(ValueError, "{0.[]}".format, 0) @@ -955,6 +964,14 @@ class UnicodeTest(string_tests.CommonTest, '') self.assertEqual("{[{}]}".format({"{}": 5}), "5") + self.assertEqual("{[{}]}".format({"{}" : "a"}), "a") + self.assertEqual("{[{]}".format({"{" : "a"}), "a") + self.assertEqual("{[}]}".format({"}" : "a"}), "a") + self.assertEqual("{[[]}".format({"[" : "a"}), "a") + self.assertEqual("{[!]}".format({"!" : "a"}), "a") + self.assertRaises(ValueError, "{a{}b}".format, 42) + self.assertRaises(ValueError, "{a{b}".format, 42) + self.assertRaises(ValueError, "{[}".format, 42) def test_format_map(self): self.assertEqual(''.format_map({}), '') @@ -1107,6 +1124,38 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual('%.1s' % "a\xe9\u20ac", 'a') self.assertEqual('%.2s' % "a\xe9\u20ac", 'a\xe9') + def test_formatting_with_enum(self): + # issue18780 + import enum + class Float(float, enum.Enum): + PI = 3.1415926 + class Int(enum.IntEnum): + IDES = 15 + class Str(str, enum.Enum): + ABC = 'abc' + # Testing Unicode formatting strings... + self.assertEqual("%s, %s" % (Str.ABC, Str.ABC), + 'Str.ABC, Str.ABC') + self.assertEqual("%s, %s, %d, %i, %u, %f, %5.2f" % + (Str.ABC, Str.ABC, + Int.IDES, Int.IDES, Int.IDES, + Float.PI, Float.PI), + 'Str.ABC, Str.ABC, 15, 15, 15, 3.141593, 3.14') + + # formatting jobs delegated from the string implementation: + self.assertEqual('...%(foo)s...' % {'foo':Str.ABC}, + '...Str.ABC...') + self.assertEqual('...%(foo)s...' % {'foo':Int.IDES}, + '...Int.IDES...') + self.assertEqual('...%(foo)i...' % {'foo':Int.IDES}, + '...15...') + self.assertEqual('...%(foo)d...' % {'foo':Int.IDES}, + '...15...') + self.assertEqual('...%(foo)u...' % {'foo':Int.IDES, 'def':Float.PI}, + '...15...') + self.assertEqual('...%(foo)f...' % {'foo':Float.PI,'def':123}, + '...3.141593...') + @support.cpython_only def test_formatting_huge_precision(self): from _testcapi import INT_MAX @@ -2004,9 +2053,10 @@ class UnicodeTest(string_tests.CommonTest, # Test PyUnicode_FromFormat() def test_from_format(self): support.import_module('ctypes') - from ctypes import (pythonapi, py_object, + from ctypes import ( + pythonapi, py_object, sizeof, c_int, c_long, c_longlong, c_ssize_t, - c_uint, c_ulong, c_ulonglong, c_size_t) + c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) name = "PyUnicode_FromFormat" _PyUnicode_FromFormat = getattr(pythonapi, name) _PyUnicode_FromFormat.restype = py_object @@ -2017,9 +2067,13 @@ class UnicodeTest(string_tests.CommonTest, for arg in args) return _PyUnicode_FromFormat(format, *cargs) + def check_format(expected, format, *args): + text = PyUnicode_FromFormat(format, *args) + self.assertEqual(expected, text) + # ascii format, non-ascii argument - text = PyUnicode_FromFormat(b'ascii\x7f=%U', 'unicode\xe9') - self.assertEqual(text, 'ascii\x7f=unicode\xe9') + check_format('ascii\x7f=unicode\xe9', + b'ascii\x7f=%U', 'unicode\xe9') # non-ascii format, ascii argument: ensure that PyUnicode_FromFormatV() # raises an error @@ -2029,64 +2083,205 @@ class UnicodeTest(string_tests.CommonTest, PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii') # test "%c" - self.assertEqual(PyUnicode_FromFormat(b'%c', c_int(0xabcd)), '\uabcd') - self.assertEqual(PyUnicode_FromFormat(b'%c', c_int(0x10ffff)), '\U0010ffff') + check_format('\uabcd', + b'%c', c_int(0xabcd)) + check_format('\U0010ffff', + b'%c', c_int(0x10ffff)) with self.assertRaises(OverflowError): PyUnicode_FromFormat(b'%c', c_int(0x110000)) # Issue #18183 - self.assertEqual( - PyUnicode_FromFormat(b'%c%c', c_int(0x10000), c_int(0x100000)), - '\U00010000\U00100000') + check_format('\U00010000\U00100000', + b'%c%c', c_int(0x10000), c_int(0x100000)) # test "%" - self.assertEqual(PyUnicode_FromFormat(b'%'), '%') - self.assertEqual(PyUnicode_FromFormat(b'%%'), '%') - self.assertEqual(PyUnicode_FromFormat(b'%%s'), '%s') - self.assertEqual(PyUnicode_FromFormat(b'[%%]'), '[%]') - self.assertEqual(PyUnicode_FromFormat(b'%%%s', b'abc'), '%abc') + check_format('%', + b'%') + check_format('%', + b'%%') + check_format('%s', + b'%%s') + check_format('[%]', + b'[%%]') + check_format('%abc', + b'%%%s', b'abc') + + # truncated string + check_format('abc', + b'%.3s', b'abcdef') + check_format('abc[\ufffd', + b'%.5s', 'abc[\u20ac]'.encode('utf8')) + check_format("'\\u20acABC'", + b'%A', '\u20acABC') + check_format("'\\u20", + b'%.5A', '\u20acABCDEF') + check_format("'\u20acABC'", + b'%R', '\u20acABC') + check_format("'\u20acA", + b'%.3R', '\u20acABCDEF') + check_format('\u20acAB', + b'%.3S', '\u20acABCDEF') + check_format('\u20acAB', + b'%.3U', '\u20acABCDEF') + check_format('\u20acAB', + b'%.3V', '\u20acABCDEF', None) + check_format('abc[\ufffd', + b'%.5V', None, 'abc[\u20ac]'.encode('utf8')) + + # following tests comes from #7330 + # test width modifier and precision modifier with %S + check_format("repr= abc", + b'repr=%5S', 'abc') + check_format("repr=ab", + b'repr=%.2S', 'abc') + check_format("repr= ab", + b'repr=%5.2S', 'abc') + + # test width modifier and precision modifier with %R + check_format("repr= 'abc'", + b'repr=%8R', 'abc') + check_format("repr='ab", + b'repr=%.3R', 'abc') + check_format("repr= 'ab", + b'repr=%5.3R', 'abc') + + # test width modifier and precision modifier with %A + check_format("repr= 'abc'", + b'repr=%8A', 'abc') + check_format("repr='ab", + b'repr=%.3A', 'abc') + check_format("repr= 'ab", + b'repr=%5.3A', 'abc') + + # test width modifier and precision modifier with %s + check_format("repr= abc", + b'repr=%5s', b'abc') + check_format("repr=ab", + b'repr=%.2s', b'abc') + check_format("repr= ab", + b'repr=%5.2s', b'abc') + + # test width modifier and precision modifier with %U + check_format("repr= abc", + b'repr=%5U', 'abc') + check_format("repr=ab", + b'repr=%.2U', 'abc') + check_format("repr= ab", + b'repr=%5.2U', 'abc') + + # test width modifier and precision modifier with %V + check_format("repr= abc", + b'repr=%5V', 'abc', b'123') + check_format("repr=ab", + b'repr=%.2V', 'abc', b'123') + check_format("repr= ab", + b'repr=%5.2V', 'abc', b'123') + check_format("repr= 123", + b'repr=%5V', None, b'123') + check_format("repr=12", + b'repr=%.2V', None, b'123') + check_format("repr= 12", + b'repr=%5.2V', None, b'123') # test integer formats (%i, %d, %u) - self.assertEqual(PyUnicode_FromFormat(b'%03i', c_int(10)), '010') - self.assertEqual(PyUnicode_FromFormat(b'%0.4i', c_int(10)), '0010') - self.assertEqual(PyUnicode_FromFormat(b'%i', c_int(-123)), '-123') - self.assertEqual(PyUnicode_FromFormat(b'%li', c_long(-123)), '-123') - self.assertEqual(PyUnicode_FromFormat(b'%lli', c_longlong(-123)), '-123') - self.assertEqual(PyUnicode_FromFormat(b'%zi', c_ssize_t(-123)), '-123') - - self.assertEqual(PyUnicode_FromFormat(b'%d', c_int(-123)), '-123') - self.assertEqual(PyUnicode_FromFormat(b'%ld', c_long(-123)), '-123') - self.assertEqual(PyUnicode_FromFormat(b'%lld', c_longlong(-123)), '-123') - self.assertEqual(PyUnicode_FromFormat(b'%zd', c_ssize_t(-123)), '-123') - - self.assertEqual(PyUnicode_FromFormat(b'%u', c_uint(123)), '123') - self.assertEqual(PyUnicode_FromFormat(b'%lu', c_ulong(123)), '123') - self.assertEqual(PyUnicode_FromFormat(b'%llu', c_ulonglong(123)), '123') - self.assertEqual(PyUnicode_FromFormat(b'%zu', c_size_t(123)), '123') + check_format('010', + b'%03i', c_int(10)) + check_format('0010', + b'%0.4i', c_int(10)) + check_format('-123', + b'%i', c_int(-123)) + check_format('-123', + b'%li', c_long(-123)) + check_format('-123', + b'%lli', c_longlong(-123)) + check_format('-123', + b'%zi', c_ssize_t(-123)) + + check_format('-123', + b'%d', c_int(-123)) + check_format('-123', + b'%ld', c_long(-123)) + check_format('-123', + b'%lld', c_longlong(-123)) + check_format('-123', + b'%zd', c_ssize_t(-123)) + + check_format('123', + b'%u', c_uint(123)) + check_format('123', + b'%lu', c_ulong(123)) + check_format('123', + b'%llu', c_ulonglong(123)) + check_format('123', + b'%zu', c_size_t(123)) + + # test long output + min_longlong = -(2 ** (8 * sizeof(c_longlong) - 1)) + max_longlong = -min_longlong - 1 + check_format(str(min_longlong), + b'%lld', c_longlong(min_longlong)) + check_format(str(max_longlong), + b'%lld', c_longlong(max_longlong)) + max_ulonglong = 2 ** (8 * sizeof(c_ulonglong)) - 1 + check_format(str(max_ulonglong), + b'%llu', c_ulonglong(max_ulonglong)) + PyUnicode_FromFormat(b'%p', c_void_p(-1)) + + # test padding (width and/or precision) + check_format('123'.rjust(10, '0'), + b'%010i', c_int(123)) + check_format('123'.rjust(100), + b'%100i', c_int(123)) + check_format('123'.rjust(100, '0'), + b'%.100i', c_int(123)) + check_format('123'.rjust(80, '0').rjust(100), + b'%100.80i', c_int(123)) + + check_format('123'.rjust(10, '0'), + b'%010u', c_uint(123)) + check_format('123'.rjust(100), + b'%100u', c_uint(123)) + check_format('123'.rjust(100, '0'), + b'%.100u', c_uint(123)) + check_format('123'.rjust(80, '0').rjust(100), + b'%100.80u', c_uint(123)) + + check_format('123'.rjust(10, '0'), + b'%010x', c_int(0x123)) + check_format('123'.rjust(100), + b'%100x', c_int(0x123)) + check_format('123'.rjust(100, '0'), + b'%.100x', c_int(0x123)) + check_format('123'.rjust(80, '0').rjust(100), + b'%100.80x', c_int(0x123)) # test %A - text = PyUnicode_FromFormat(b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') - self.assertEqual(text, r"%A:'abc\xe9\uabcd\U0010ffff'") + check_format(r"%A:'abc\xe9\uabcd\U0010ffff'", + b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') # test %V - text = PyUnicode_FromFormat(b'repr=%V', 'abc', b'xyz') - self.assertEqual(text, 'repr=abc') + check_format('repr=abc', + b'repr=%V', 'abc', b'xyz') # Test string decode from parameter of %s using utf-8. # b'\xe4\xba\xba\xe6\xb0\x91' is utf-8 encoded byte sequence of # '\u4eba\u6c11' - text = PyUnicode_FromFormat(b'repr=%V', None, b'\xe4\xba\xba\xe6\xb0\x91') - self.assertEqual(text, 'repr=\u4eba\u6c11') + check_format('repr=\u4eba\u6c11', + b'repr=%V', None, b'\xe4\xba\xba\xe6\xb0\x91') #Test replace error handler. - text = PyUnicode_FromFormat(b'repr=%V', None, b'abc\xff') - self.assertEqual(text, 'repr=abc\ufffd') + check_format('repr=abc\ufffd', + b'repr=%V', None, b'abc\xff') # not supported: copy the raw format string. these tests are just here # to check for crashs and should not be considered as specifications - self.assertEqual(PyUnicode_FromFormat(b'%1%s', b'abc'), '%s') - self.assertEqual(PyUnicode_FromFormat(b'%1abc'), '%1abc') - self.assertEqual(PyUnicode_FromFormat(b'%+i', c_int(10)), '%+i') - self.assertEqual(PyUnicode_FromFormat(b'%.%s', b'abc'), '%.%s') + check_format('%s', + b'%1%s', b'abc') + check_format('%1abc', + b'%1abc') + check_format('%+i', + b'%+i', c_int(10)) + check_format('%.%s', + b'%.%s', b'abc') # Test PyUnicode_AsWideChar() def test_aswidechar(self): @@ -2210,6 +2405,80 @@ class UnicodeTest(string_tests.CommonTest, self.assertNotEqual(abc, abcdef) self.assertEqual(abcdef.decode('unicode_internal'), text) + def test_compare(self): + # Issue #17615 + N = 10 + ascii = 'a' * N + ascii2 = 'z' * N + latin = '\x80' * N + latin2 = '\xff' * N + bmp = '\u0100' * N + bmp2 = '\uffff' * N + astral = '\U00100000' * N + astral2 = '\U0010ffff' * N + strings = ( + ascii, ascii2, + latin, latin2, + bmp, bmp2, + astral, astral2) + for text1, text2 in itertools.combinations(strings, 2): + equal = (text1 is text2) + self.assertEqual(text1 == text2, equal) + self.assertEqual(text1 != text2, not equal) + + if equal: + self.assertTrue(text1 <= text2) + self.assertTrue(text1 >= text2) + + # text1 is text2: duplicate strings to skip the "str1 == str2" + # optimization in unicode_compare_eq() and really compare + # character per character + copy1 = duplicate_string(text1) + copy2 = duplicate_string(text2) + self.assertIsNot(copy1, copy2) + + self.assertTrue(copy1 == copy2) + self.assertFalse(copy1 != copy2) + + self.assertTrue(copy1 <= copy2) + self.assertTrue(copy2 >= copy2) + + self.assertTrue(ascii < ascii2) + self.assertTrue(ascii < latin) + self.assertTrue(ascii < bmp) + self.assertTrue(ascii < astral) + self.assertFalse(ascii >= ascii2) + self.assertFalse(ascii >= latin) + self.assertFalse(ascii >= bmp) + self.assertFalse(ascii >= astral) + + self.assertFalse(latin < ascii) + self.assertTrue(latin < latin2) + self.assertTrue(latin < bmp) + self.assertTrue(latin < astral) + self.assertTrue(latin >= ascii) + self.assertFalse(latin >= latin2) + self.assertFalse(latin >= bmp) + self.assertFalse(latin >= astral) + + self.assertFalse(bmp < ascii) + self.assertFalse(bmp < latin) + self.assertTrue(bmp < bmp2) + self.assertTrue(bmp < astral) + self.assertTrue(bmp >= ascii) + self.assertTrue(bmp >= latin) + self.assertFalse(bmp >= bmp2) + self.assertFalse(bmp >= astral) + + self.assertFalse(astral < ascii) + self.assertFalse(astral < latin) + self.assertFalse(astral < bmp2) + self.assertTrue(astral < astral2) + self.assertTrue(astral >= ascii) + self.assertTrue(astral >= latin) + self.assertTrue(astral >= bmp2) + self.assertFalse(astral >= astral2) + class StringModuleTest(unittest.TestCase): def test_formatter_parser(self): diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 99aa0033cd..707b30e50c 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -21,7 +21,7 @@ errors = 'surrogatepass' class UnicodeMethodsTest(unittest.TestCase): # update this, if the database changes - expectedchecksum = 'bf7a78f1a532421b5033600102e23a92044dbba9' + expectedchecksum = 'e74e878de71b6e780ffac271785c3cb58f6251f3' def test_method_checksum(self): h = hashlib.sha1() @@ -80,7 +80,7 @@ class UnicodeDatabaseTest(unittest.TestCase): class UnicodeFunctionsTest(UnicodeDatabaseTest): # update this, if the database changes - expectedchecksum = '17fe2f12b788e4fff5479b469c4404bb6ecf841f' + expectedchecksum = 'f0b74d26776331cc7bdc3a4698f037d73f2cee2b' def test_function_checksum(self): data = [] h = hashlib.sha1() diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 7a34a05d05..94f640b923 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -16,6 +16,7 @@ from nturl2path import url2pathname, pathname2url from base64 import b64encode import collections + def hexescape(char): """Escape char as RFC 2396 specifies""" hex_repr = hex(ord(char))[2:].upper() @@ -238,7 +239,7 @@ class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin): self.check_read(b"1.1") def test_read_bogus(self): - # urlopen() should raise IOError for many error codes. + # urlopen() should raise OSError for many error codes. self.fakehttp(b'''HTTP/1.1 401 Authentication Required Date: Wed, 02 Jan 2008 03:03:54 GMT Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e @@ -246,12 +247,12 @@ Connection: close Content-Type: text/html; charset=iso-8859-1 ''') try: - self.assertRaises(IOError, urlopen, "http://python.org/") + self.assertRaises(OSError, urlopen, "http://python.org/") finally: self.unfakehttp() def test_invalid_redirect(self): - # urlopen() should raise IOError for many error codes. + # urlopen() should raise OSError for many error codes. self.fakehttp(b'''HTTP/1.1 302 Found Date: Wed, 02 Jan 2008 03:03:54 GMT Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e @@ -266,19 +267,20 @@ Content-Type: text/html; charset=iso-8859-1 self.unfakehttp() def test_empty_socket(self): - # urlopen() raises IOError if the underlying socket does not send any + # urlopen() raises OSError if the underlying socket does not send any # data. (#1680230) self.fakehttp(b'') try: - self.assertRaises(IOError, urlopen, "http://something") + self.assertRaises(OSError, urlopen, "http://something") finally: self.unfakehttp() def test_missing_localfile(self): # Test for #10836 - # 3.3 - URLError is not captured, explicit IOError is raised. - with self.assertRaises(IOError): + with self.assertRaises(urllib.error.URLError) as e: urlopen('file://localhost/a/file/which/doesnot/exists.py') + self.assertTrue(e.exception.filename) + self.assertTrue(e.exception.reason) def test_file_notexists(self): fd, tmp_file = tempfile.mkstemp() @@ -291,20 +293,21 @@ Content-Type: text/html; charset=iso-8859-1 os.close(fd) os.unlink(tmp_file) self.assertFalse(os.path.exists(tmp_file)) - # 3.3 - IOError instead of URLError - with self.assertRaises(IOError): + with self.assertRaises(urllib.error.URLError): urlopen(tmp_fileurl) def test_ftp_nohost(self): test_ftp_url = 'ftp:///path' - # 3.3 - IOError instead of URLError - with self.assertRaises(IOError): + with self.assertRaises(urllib.error.URLError) as e: urlopen(test_ftp_url) + self.assertFalse(e.exception.filename) + self.assertTrue(e.exception.reason) def test_ftp_nonexisting(self): - # 3.3 - IOError instead of URLError - with self.assertRaises(IOError): + with self.assertRaises(urllib.error.URLError) as e: urlopen('ftp://localhost/a/file/which/doesnot/exists.py') + self.assertFalse(e.exception.filename) + self.assertTrue(e.exception.reason) def test_userpass_inurl(self): @@ -341,6 +344,79 @@ Content-Type: text/html; charset=iso-8859-1 with support.check_warnings(('',DeprecationWarning)): urllib.request.URLopener() +class urlopen_DataTests(unittest.TestCase): + """Test urlopen() opening a data URL.""" + + def setUp(self): + # text containing URL special- and unicode-characters + self.text = "test data URLs :;,%=& \u00f6 \u00c4 " + # 2x1 pixel RGB PNG image with one black and one white pixel + self.image = ( + b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x02\x00\x00\x00' + b'\x01\x08\x02\x00\x00\x00{@\xe8\xdd\x00\x00\x00\x01sRGB\x00\xae' + b'\xce\x1c\xe9\x00\x00\x00\x0fIDAT\x08\xd7c```\xf8\xff\xff?\x00' + b'\x06\x01\x02\xfe\no/\x1e\x00\x00\x00\x00IEND\xaeB`\x82') + + self.text_url = ( + "data:text/plain;charset=UTF-8,test%20data%20URLs%20%3A%3B%2C%25%3" + "D%26%20%C3%B6%20%C3%84%20") + self.text_url_base64 = ( + "data:text/plain;charset=ISO-8859-1;base64,dGVzdCBkYXRhIFVSTHMgOjs" + "sJT0mIPYgxCA%3D") + # base64 encoded data URL that contains ignorable spaces, + # such as "\n", " ", "%0A", and "%20". + self.image_url = ( + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAIAAAB7\n" + "QOjdAAAAAXNSR0IArs4c6QAAAA9JREFUCNdj%0AYGBg%2BP//PwAGAQL%2BCm8 " + "vHgAAAABJRU5ErkJggg%3D%3D%0A%20") + + self.text_url_resp = urllib.request.urlopen(self.text_url) + self.text_url_base64_resp = urllib.request.urlopen( + self.text_url_base64) + self.image_url_resp = urllib.request.urlopen(self.image_url) + + def test_interface(self): + # Make sure object returned by urlopen() has the specified methods + for attr in ("read", "readline", "readlines", + "close", "info", "geturl", "getcode", "__iter__"): + self.assertTrue(hasattr(self.text_url_resp, attr), + "object returned by urlopen() lacks %s attribute" % + attr) + + def test_info(self): + self.assertIsInstance(self.text_url_resp.info(), email.message.Message) + self.assertEqual(self.text_url_base64_resp.info().get_params(), + [('text/plain', ''), ('charset', 'ISO-8859-1')]) + self.assertEqual(self.image_url_resp.info()['content-length'], + str(len(self.image))) + self.assertEqual(urllib.request.urlopen("data:,").info().get_params(), + [('text/plain', ''), ('charset', 'US-ASCII')]) + + def test_geturl(self): + self.assertEqual(self.text_url_resp.geturl(), self.text_url) + self.assertEqual(self.text_url_base64_resp.geturl(), + self.text_url_base64) + self.assertEqual(self.image_url_resp.geturl(), self.image_url) + + def test_read_text(self): + self.assertEqual(self.text_url_resp.read().decode( + dict(self.text_url_resp.info().get_params())['charset']), self.text) + + def test_read_text_base64(self): + self.assertEqual(self.text_url_base64_resp.read().decode( + dict(self.text_url_base64_resp.info().get_params())['charset']), + self.text) + + def test_read_image(self): + self.assertEqual(self.image_url_resp.read(), self.image) + + def test_missing_comma(self): + self.assertRaises(ValueError,urllib.request.urlopen,'data:text/plain') + + def test_invalid_base64_data(self): + # missing padding character + self.assertRaises(ValueError,urllib.request.urlopen,'data:;base64,Cg=') + class urlretrieve_FileTests(unittest.TestCase): """Test urllib.urlretrieve() on local files""" @@ -1291,6 +1367,7 @@ class URLopener_Tests(unittest.TestCase): # self.assertEqual(ftp.ftp.sock.gettimeout(), 30) # ftp.close() + class RequestTests(unittest.TestCase): """Unit tests for urllib.request.Request.""" diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 33f90f48ca..dbd1c60ec2 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -11,6 +11,7 @@ import urllib.request # The proxy bypass method imported below has logic specific to the OSX # proxy config data structure but is testable on all platforms. from urllib.request import Request, OpenerDirector, _proxy_bypass_macosx_sysconf +from urllib.parse import urlparse import urllib.error # XXX @@ -118,6 +119,15 @@ class RequestHdrsTests(unittest.TestCase): self.assertIsNone(req.get_header("Not-there")) self.assertEqual(req.get_header("Not-there", "default"), "default") + req.remove_header("Spam-eggs") + self.assertFalse(req.has_header("Spam-eggs")) + + req.add_unredirected_header("Unredirected-spam", "Eggs") + self.assertTrue(req.has_header("Unredirected-spam")) + + req.remove_header("Unredirected-spam") + self.assertFalse(req.has_header("Unredirected-spam")) + def test_password_manager(self): mgr = urllib.request.HTTPPasswordMgr() @@ -283,6 +293,7 @@ class MockHTTPClass: self.req_headers = [] self.data = None self.raise_on_endheaders = False + self.sock = None self._tunnel_headers = {} def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): @@ -310,8 +321,7 @@ class MockHTTPClass: if body: self.data = body if self.raise_on_endheaders: - import socket - raise socket.error() + raise OSError() def getresponse(self): return MockHTTPResponse(MockFile(), {}, 200, "OK") @@ -596,27 +606,6 @@ class OpenerDirectorTests(unittest.TestCase): self.assertTrue(args[1] is None or isinstance(args[1], MockResponse)) - def test_method_deprecations(self): - req = Request("http://www.example.com") - - with self.assertWarns(DeprecationWarning): - req.add_data("data") - with self.assertWarns(DeprecationWarning): - req.get_data() - with self.assertWarns(DeprecationWarning): - req.has_data() - with self.assertWarns(DeprecationWarning): - req.get_host() - with self.assertWarns(DeprecationWarning): - req.get_selector() - with self.assertWarns(DeprecationWarning): - req.is_unverifiable() - with self.assertWarns(DeprecationWarning): - req.get_origin_req_host() - with self.assertWarns(DeprecationWarning): - req.get_type() - - def sanepathname2url(path): try: path.encode("utf-8") @@ -811,7 +800,7 @@ class HandlerTests(unittest.TestCase): ("Foo", "bar"), ("Spam", "eggs")]) self.assertEqual(http.data, data) - # check socket.error converted to URLError + # check OSError converted to URLError http.raise_on_endheaders = True self.assertRaises(urllib.error.URLError, h.do_open, http, req) @@ -916,6 +905,36 @@ class HandlerTests(unittest.TestCase): p_ds_req = h.do_request_(ds_req) self.assertEqual(p_ds_req.unredirected_hdrs["Host"],"example.com") + def test_full_url_setter(self): + # Checks to ensure that components are set correctly after setting the + # full_url of a Request object + + urls = [ + 'http://example.com?foo=bar#baz', + 'http://example.com?foo=bar&spam=eggs#bash', + 'http://example.com', + ] + + # testing a reusable request instance, but the url parameter is + # required, so just use a dummy one to instantiate + r = Request('http://example.com') + for url in urls: + r.full_url = url + parsed = urlparse(url) + + self.assertEqual(r.get_full_url(), url) + # full_url setter uses splittag to split into components. + # splittag sets the fragment as None while urlparse sets it to '' + self.assertEqual(r.fragment or '', parsed.fragment) + self.assertEqual(urlparse(r.get_full_url()).query, parsed.query) + + def test_full_url_deleter(self): + r = Request('http://www.example.com') + del r.full_url + self.assertIsNone(r.full_url) + self.assertIsNone(r.fragment) + self.assertEqual(r.selector, '') + def test_fixpath_in_weirdurls(self): # Issue4493: urllib2 to supply '/' when to urls where path does not # start with'/' @@ -1415,6 +1434,21 @@ class MiscTests(unittest.TestCase): self.opener_has_handler(o, MyHTTPHandler) self.opener_has_handler(o, MyOtherHTTPHandler) + @unittest.skipUnless(support.is_resource_enabled('network'), + 'test requires network access') + def test_issue16464(self): + opener = urllib.request.build_opener() + request = urllib.request.Request("http://www.python.org/~jeremy/") + self.assertEqual(None, request.data) + + opener.open(request, "1".encode("us-ascii")) + self.assertEqual(b"1", request.data) + self.assertEqual("1", request.get_header("Content-length")) + + opener.open(request, "1234567890".encode("us-ascii")) + self.assertEqual(b"1234567890", request.data) + self.assertEqual("10", request.get_header("Content-length")) + def test_HTTPError_interface(self): """ Issue 13211 reveals that HTTPError didn't implement the URLError @@ -1426,23 +1460,31 @@ class MiscTests(unittest.TestCase): err = urllib.error.HTTPError(url, code, msg, hdrs, fp) self.assertTrue(hasattr(err, 'reason')) self.assertEqual(err.reason, 'something bad happened') - self.assertTrue(hasattr(err, 'hdrs')) - self.assertEqual(err.hdrs, 'Content-Length: 42') + self.assertTrue(hasattr(err, 'headers')) + self.assertEqual(err.headers, 'Content-Length: 42') expected_errmsg = 'HTTP Error %s: %s' % (err.code, err.msg) self.assertEqual(str(err), expected_errmsg) - class RequestTests(unittest.TestCase): + class PutRequest(Request): + method='PUT' def setUp(self): self.get = Request("http://www.python.org/~jeremy/") self.post = Request("http://www.python.org/~jeremy/", "data", headers={"X-Test": "test"}) + self.head = Request("http://www.python.org/~jeremy/", method='HEAD') + self.put = self.PutRequest("http://www.python.org/~jeremy/") + self.force_post = self.PutRequest("http://www.python.org/~jeremy/", + method="POST") def test_method(self): self.assertEqual("POST", self.post.get_method()) self.assertEqual("GET", self.get.get_method()) + self.assertEqual("HEAD", self.head.get_method()) + self.assertEqual("PUT", self.put.get_method()) + self.assertEqual("POST", self.force_post.get_method()) def test_data(self): self.assertFalse(self.get.data) @@ -1451,6 +1493,25 @@ class RequestTests(unittest.TestCase): self.assertTrue(self.get.data) self.assertEqual("POST", self.get.get_method()) + # issue 16464 + # if we change data we need to remove content-length header + # (cause it's most probably calculated for previous value) + def test_setting_data_should_remove_content_length(self): + self.assertNotIn("Content-length", self.get.unredirected_hdrs) + self.get.add_unredirected_header("Content-length", 42) + self.assertEqual(42, self.get.unredirected_hdrs["Content-length"]) + self.get.data = "spam" + self.assertNotIn("Content-length", self.get.unredirected_hdrs) + + # issue 17485 same for deleting data. + def test_deleting_data_should_remove_content_length(self): + self.assertNotIn("Content-length", self.get.unredirected_hdrs) + self.get.data = 'foo' + self.get.add_unredirected_header("Content-length", 3) + self.assertEqual(3, self.get.unredirected_hdrs["Content-length"]) + del self.get.data + self.assertNotIn("Content-length", self.get.unredirected_hdrs) + def test_get_full_url(self): self.assertEqual("http://www.python.org/~jeremy/", self.get.get_full_url()) @@ -1492,21 +1553,13 @@ class RequestTests(unittest.TestCase): req = Request(url) self.assertEqual(req.get_full_url(), url) - def test_HTTPError_interface_call(self): - """ - Issue 15701 - HTTPError interface has info method available from URLError - """ - err = urllib.request.HTTPError(msg="something bad happened", url=None, - code=None, hdrs='Content-Length:42', fp=None) - self.assertTrue(hasattr(err, 'reason')) - assert hasattr(err, 'reason') - assert hasattr(err, 'info') - assert callable(err.info) - try: - err.info() - except AttributeError: - self.fail('err.info call failed.') - self.assertEqual(err.info(), "Content-Length:42") + def test_url_fullurl_get_full_url(self): + urls = ['http://docs.python.org', + 'http://docs.python.org/library/urllib2.html#OK', + 'http://www.python.org/?qs=query#fragment=true' ] + for url in urls: + req = Request(url) + self.assertEqual(req.get_full_url(), req.full_url) def test_main(verbose=None): from test import test_urllib2 diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index b1aa158455..08250c36e5 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -9,7 +9,10 @@ import unittest import hashlib from test import support threading = support.import_module('threading') - +try: + import ssl +except ImportError: + ssl = None here = os.path.dirname(__file__) # Self-signed cert file for 'localhost' @@ -17,6 +20,7 @@ CERT_localhost = os.path.join(here, 'keycert.pem') # Self-signed cert file for 'fakehostname' CERT_fakehostname = os.path.join(here, 'keycert2.pem') + # Loopback http server infrastructure class LoopbackHttpServer(http.server.HTTPServer): @@ -353,12 +357,15 @@ class TestUrlopen(unittest.TestCase): def setUp(self): super(TestUrlopen, self).setUp() # Ignore proxies for localhost tests. + self.old_environ = os.environ.copy() os.environ['NO_PROXY'] = '*' self.server = None def tearDown(self): if self.server is not None: self.server.stop() + os.environ.clear() + os.environ.update(self.old_environ) super(TestUrlopen, self).tearDown() def urlopen(self, url, data=None, **kwargs): @@ -386,14 +393,14 @@ class TestUrlopen(unittest.TestCase): handler.port = port return handler - def start_https_server(self, responses=None, certfile=CERT_localhost): + def start_https_server(self, responses=None, **kwargs): if not hasattr(urllib.request, 'HTTPSHandler'): self.skipTest('ssl support required') from test.ssl_servers import make_https_server if responses is None: responses = [(200, [], b"we care a bit")] handler = GetRequestHandler(responses) - server = make_https_server(self, certfile=certfile, handler_class=handler) + server = make_https_server(self, handler_class=handler, **kwargs) handler.port = server.port return handler @@ -483,6 +490,21 @@ class TestUrlopen(unittest.TestCase): self.urlopen("https://localhost:%s/bizarre" % handler.port, cadefault=True) + def test_https_sni(self): + if ssl is None: + self.skipTest("ssl module required") + if not ssl.HAS_SNI: + self.skipTest("SNI support required in OpenSSL") + sni_name = None + def cb_sni(ssl_sock, server_name, initial_context): + nonlocal sni_name + sni_name = server_name + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.set_servername_callback(cb_sni) + handler = self.start_https_server(context=context, certfile=CERT_localhost) + self.urlopen("https://localhost:%s" % handler.port) + self.assertEqual(sni_name, "localhost") + def test_sending_headers(self): handler = self.start_server() req = urllib.request.Request("http://localhost:%s/" % handler.port, @@ -529,7 +551,7 @@ class TestUrlopen(unittest.TestCase): # so we run the test only when -unetwork/-uall is specified to # mitigate the problem a bit (see #17564) support.requires('network') - self.assertRaises(IOError, + self.assertRaises(OSError, # Given that both VeriSign and various ISPs have in # the past or are presently hijacking various invalid # domain name requests in an attempt to boost traffic diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index 7f3c93adaa..fba3ceac83 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -164,6 +164,14 @@ class OtherNetworkTests(unittest.TestCase): self.assertEqual(res.geturl(), "http://docs.python.org/2/glossary.html#glossary") + def test_redirect_url_withfrag(self): + redirect_url_with_frag = "http://bitly.com/urllibredirecttest" + with support.transient_internet(redirect_url_with_frag): + req = urllib.request.Request(redirect_url_with_frag) + res = urllib.request.urlopen(req) + self.assertEqual(res.geturl(), + "http://docs.python.org/3.4/glossary.html#term-global-interpreter-lock") + def test_custom_headers(self): url = "http://www.example.com" with support.transient_internet(url): @@ -216,7 +224,7 @@ class OtherNetworkTests(unittest.TestCase): debug(url) try: f = urlopen(url, req, TIMEOUT) - except EnvironmentError as err: + except OSError as err: debug(err) if expected_err: msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" % @@ -330,35 +338,6 @@ class TimeoutTest(unittest.TestCase): self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60) -@unittest.skipUnless(ssl, "requires SSL support") -class HTTPSTests(unittest.TestCase): - - def test_sni(self): - self.skipTest("test disabled - test server needed") - # Checks that Server Name Indication works, if supported by the - # OpenSSL linked to. - # The ssl module itself doesn't have server-side support for SNI, - # so we rely on a third-party test site. - expect_sni = ssl.HAS_SNI - with support.transient_internet("XXX"): - u = urllib.request.urlopen("XXX") - contents = u.readall() - if expect_sni: - self.assertIn(b"Great", contents) - self.assertNotIn(b"Unfortunately", contents) - else: - self.assertNotIn(b"Great", contents) - self.assertIn(b"Unfortunately", contents) - - -def test_main(): - support.requires("network") - support.run_unittest(AuthTests, - HTTPSTests, - OtherNetworkTests, - CloseSocketTest, - TimeoutTest, - ) - if __name__ == "__main__": - test_main() + support.requires("network") + unittest.main() diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index 20efca6adb..b6888f3d7b 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -124,16 +124,15 @@ class urlopenNetworkTests(unittest.TestCase): else: # This happens with some overzealous DNS providers such as OpenDNS self.skipTest("%r should not resolve for test to work" % bogus_domain) - self.assertRaises(IOError, - # SF patch 809915: In Sep 2003, VeriSign started - # highjacking invalid .com and .net addresses to - # boost traffic to their own site. This test - # started failing then. One hopes the .invalid - # domain will be spared to serve its defined - # purpose. - # urllib.urlopen, "http://www.sadflkjsasadf.com/") - urllib.request.urlopen, - "http://sadflkjsasf.i.nvali.d/") + failure_explanation = ('opening an invalid URL did not raise OSError; ' + 'can be caused by a broken DNS server ' + '(e.g. returns 404 or hijacks page)') + with self.assertRaises(OSError, msg=failure_explanation): + # SF patch 809915: In Sep 2003, VeriSign started highjacking + # invalid .com and .net addresses to boost traffic to their own + # site. This test started failing then. One hopes the .invalid + # domain will be spared to serve its defined purpose. + urllib.request.urlopen("http://sadflkjsasf.i.nvali.d/") class urlretrieveNetworkTests(unittest.TestCase): diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 378a427bc5..c938f09951 100755 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -847,6 +847,14 @@ class UrlParseTestCase(unittest.TestCase): self.assertEqual(p1.path, '863-1234') self.assertEqual(p1.params, 'phone-context=+1-914-555') + def test_unwrap(self): + url = urllib.parse.unwrap('<URL:type://host/path>') + self.assertEqual(url, 'type://host/path') + + def test_Quoter_repr(self): + quoter = urllib.parse.Quoter(urllib.parse._ALWAYS_SAFE) + self.assertIn('Quoter', repr(quoter)) + def test_main(): support.run_unittest(UrlParseTestCase) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 0fae88bcf9..dbbe1570c9 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -109,13 +109,68 @@ class BasicTest(BaseTest): out, err = p.communicate() self.assertEqual(out.strip(), expected.encode()) + if sys.platform == 'win32': + ENV_SUBDIRS = ( + ('Scripts',), + ('Include',), + ('Lib',), + ('Lib', 'site-packages'), + ) + else: + ENV_SUBDIRS = ( + ('bin',), + ('include',), + ('lib',), + ('lib', 'python%d.%d' % sys.version_info[:2]), + ('lib', 'python%d.%d' % sys.version_info[:2], 'site-packages'), + ) + + def create_contents(self, paths, filename): + """ + Create some files in the environment which are unrelated + to the virtual environment. + """ + for subdirs in paths: + d = os.path.join(self.env_dir, *subdirs) + os.mkdir(d) + fn = os.path.join(d, filename) + with open(fn, 'wb') as f: + f.write(b'Still here?') + def test_overwrite_existing(self): """ - Test control of overwriting an existing environment directory. + Test creating environment in an existing directory. """ - self.assertRaises(ValueError, venv.create, self.env_dir) + self.create_contents(self.ENV_SUBDIRS, 'foo') + venv.create(self.env_dir) + for subdirs in self.ENV_SUBDIRS: + fn = os.path.join(self.env_dir, *(subdirs + ('foo',))) + self.assertTrue(os.path.exists(fn)) + with open(fn, 'rb') as f: + self.assertEqual(f.read(), b'Still here?') + builder = venv.EnvBuilder(clear=True) builder.create(self.env_dir) + for subdirs in self.ENV_SUBDIRS: + fn = os.path.join(self.env_dir, *(subdirs + ('foo',))) + self.assertFalse(os.path.exists(fn)) + + def clear_directory(self, path): + for fn in os.listdir(path): + fn = os.path.join(path, fn) + if os.path.islink(fn) or os.path.isfile(fn): + os.remove(fn) + elif os.path.isdir(fn): + shutil.rmtree(fn) + + def test_unoverwritable_fails(self): + #create a file clashing with directories in the env dir + for paths in self.ENV_SUBDIRS[:3]: + fn = os.path.join(self.env_dir, *paths) + with open(fn, 'wb') as f: + f.write(b'') + self.assertRaises((ValueError, OSError), venv.create, self.env_dir) + self.clear_directory(self.env_dir) def test_upgrade(self): """ diff --git a/Lib/test/test_wait3.py b/Lib/test/test_wait3.py index bd06c8d8bd..f6a065d850 100644 --- a/Lib/test/test_wait3.py +++ b/Lib/test/test_wait3.py @@ -7,15 +7,11 @@ import unittest from test.fork_wait import ForkWait from test.support import run_unittest, reap_children -try: - os.fork -except AttributeError: - raise unittest.SkipTest("os.fork not defined -- skipping test_wait3") +if not hasattr(os, 'fork'): + raise unittest.SkipTest("os.fork not defined") -try: - os.wait3 -except AttributeError: - raise unittest.SkipTest("os.wait3 not defined -- skipping test_wait3") +if not hasattr(os, 'wait3'): + raise unittest.SkipTest("os.wait3 not defined") class Wait3Test(ForkWait): def wait_impl(self, cpid): diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 571e33f492..551d95cb91 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -7,11 +7,15 @@ import operator import contextlib import copy -from test import support +from test import support, script_helper # Used in ReferencesTestCase.test_ref_created_during_del() . ref_from_del = None +# Used by FinalizeTestCase as a global that may be replaced by None +# when the interpreter shuts down. +_global_var = 'foobar' + class C: def method(self): pass @@ -47,6 +51,11 @@ class Object: return NotImplemented def __hash__(self): return hash(self.arg) + def some_method(self): + return 4 + def other_method(self): + return 5 + class RefCycle: def __init__(self): @@ -797,6 +806,30 @@ class ReferencesTestCase(TestBase): del root gc.collect() + def test_callback_attribute(self): + x = Object(1) + callback = lambda ref: None + ref1 = weakref.ref(x, callback) + self.assertIs(ref1.__callback__, callback) + + ref2 = weakref.ref(x) + self.assertIsNone(ref2.__callback__) + + def test_callback_attribute_after_deletion(self): + x = Object(1) + ref = weakref.ref(x, self.callback) + self.assertIsNotNone(ref.__callback__) + del x + support.gc_collect() + self.assertIsNone(ref.__callback__) + + def test_set_callback_attribute(self): + x = Object(1) + callback = lambda ref: None + ref1 = weakref.ref(x, callback) + with self.assertRaises(AttributeError): + ref1.__callback__ = lambda ref: None + class SubclassableWeakrefTestCase(TestBase): @@ -901,6 +934,140 @@ class SubclassableWeakrefTestCase(TestBase): self.assertEqual(self.cbcalled, 0) +class WeakMethodTestCase(unittest.TestCase): + + def _subclass(self): + """Return a Object subclass overriding `some_method`.""" + class C(Object): + def some_method(self): + return 6 + return C + + def test_alive(self): + o = Object(1) + r = weakref.WeakMethod(o.some_method) + self.assertIsInstance(r, weakref.ReferenceType) + self.assertIsInstance(r(), type(o.some_method)) + self.assertIs(r().__self__, o) + self.assertIs(r().__func__, o.some_method.__func__) + self.assertEqual(r()(), 4) + + def test_object_dead(self): + o = Object(1) + r = weakref.WeakMethod(o.some_method) + del o + gc.collect() + self.assertIs(r(), None) + + def test_method_dead(self): + C = self._subclass() + o = C(1) + r = weakref.WeakMethod(o.some_method) + del C.some_method + gc.collect() + self.assertIs(r(), None) + + def test_callback_when_object_dead(self): + # Test callback behaviour when object dies first. + C = self._subclass() + calls = [] + def cb(arg): + calls.append(arg) + o = C(1) + r = weakref.WeakMethod(o.some_method, cb) + del o + gc.collect() + self.assertEqual(calls, [r]) + # Callback is only called once. + C.some_method = Object.some_method + gc.collect() + self.assertEqual(calls, [r]) + + def test_callback_when_method_dead(self): + # Test callback behaviour when method dies first. + C = self._subclass() + calls = [] + def cb(arg): + calls.append(arg) + o = C(1) + r = weakref.WeakMethod(o.some_method, cb) + del C.some_method + gc.collect() + self.assertEqual(calls, [r]) + # Callback is only called once. + del o + gc.collect() + self.assertEqual(calls, [r]) + + @support.cpython_only + def test_no_cycles(self): + # A WeakMethod doesn't create any reference cycle to itself. + o = Object(1) + def cb(_): + pass + r = weakref.WeakMethod(o.some_method, cb) + wr = weakref.ref(r) + del r + self.assertIs(wr(), None) + + def test_equality(self): + def _eq(a, b): + self.assertTrue(a == b) + self.assertFalse(a != b) + def _ne(a, b): + self.assertTrue(a != b) + self.assertFalse(a == b) + x = Object(1) + y = Object(1) + a = weakref.WeakMethod(x.some_method) + b = weakref.WeakMethod(y.some_method) + c = weakref.WeakMethod(x.other_method) + d = weakref.WeakMethod(y.other_method) + # Objects equal, same method + _eq(a, b) + _eq(c, d) + # Objects equal, different method + _ne(a, c) + _ne(a, d) + _ne(b, c) + _ne(b, d) + # Objects unequal, same or different method + z = Object(2) + e = weakref.WeakMethod(z.some_method) + f = weakref.WeakMethod(z.other_method) + _ne(a, e) + _ne(a, f) + _ne(b, e) + _ne(b, f) + del x, y, z + gc.collect() + # Dead WeakMethods compare by identity + refs = a, b, c, d, e, f + for q in refs: + for r in refs: + self.assertEqual(q == r, q is r) + self.assertEqual(q != r, q is not r) + + def test_hashing(self): + # Alive WeakMethods are hashable if the underlying object is + # hashable. + x = Object(1) + y = Object(1) + a = weakref.WeakMethod(x.some_method) + b = weakref.WeakMethod(y.some_method) + c = weakref.WeakMethod(y.other_method) + # Since WeakMethod objects are equal, the hashes should be equal. + self.assertEqual(hash(a), hash(b)) + ha = hash(a) + # Dead WeakMethods retain their old hash value + del x, y + gc.collect() + self.assertEqual(hash(a), ha) + self.assertEqual(hash(b), ha) + # If it wasn't hashed when alive, a dead WeakMethod cannot be hashed. + self.assertRaises(TypeError, hash, c) + + class MappingTestCase(TestBase): COUNT = 10 @@ -1388,6 +1555,151 @@ class WeakKeyDictionaryTestCase(mapping_tests.BasicTestMappingProtocol): def _reference(self): return self.__ref.copy() + +class FinalizeTestCase(unittest.TestCase): + + class A: + pass + + def _collect_if_necessary(self): + # we create no ref-cycles so in CPython no gc should be needed + if sys.implementation.name != 'cpython': + support.gc_collect() + + def test_finalize(self): + def add(x,y,z): + res.append(x + y + z) + return x + y + z + + a = self.A() + + res = [] + f = weakref.finalize(a, add, 67, 43, z=89) + self.assertEqual(f.alive, True) + self.assertEqual(f.peek(), (a, add, (67,43), {'z':89})) + self.assertEqual(f(), 199) + self.assertEqual(f(), None) + self.assertEqual(f(), None) + self.assertEqual(f.peek(), None) + self.assertEqual(f.detach(), None) + self.assertEqual(f.alive, False) + self.assertEqual(res, [199]) + + res = [] + f = weakref.finalize(a, add, 67, 43, 89) + self.assertEqual(f.peek(), (a, add, (67,43,89), {})) + self.assertEqual(f.detach(), (a, add, (67,43,89), {})) + self.assertEqual(f(), None) + self.assertEqual(f(), None) + self.assertEqual(f.peek(), None) + self.assertEqual(f.detach(), None) + self.assertEqual(f.alive, False) + self.assertEqual(res, []) + + res = [] + f = weakref.finalize(a, add, x=67, y=43, z=89) + del a + self._collect_if_necessary() + self.assertEqual(f(), None) + self.assertEqual(f(), None) + self.assertEqual(f.peek(), None) + self.assertEqual(f.detach(), None) + self.assertEqual(f.alive, False) + self.assertEqual(res, [199]) + + def test_order(self): + a = self.A() + res = [] + + f1 = weakref.finalize(a, res.append, 'f1') + f2 = weakref.finalize(a, res.append, 'f2') + f3 = weakref.finalize(a, res.append, 'f3') + f4 = weakref.finalize(a, res.append, 'f4') + f5 = weakref.finalize(a, res.append, 'f5') + + # make sure finalizers can keep themselves alive + del f1, f4 + + self.assertTrue(f2.alive) + self.assertTrue(f3.alive) + self.assertTrue(f5.alive) + + self.assertTrue(f5.detach()) + self.assertFalse(f5.alive) + + f5() # nothing because previously unregistered + res.append('A') + f3() # => res.append('f3') + self.assertFalse(f3.alive) + res.append('B') + f3() # nothing because previously called + res.append('C') + del a + self._collect_if_necessary() + # => res.append('f4') + # => res.append('f2') + # => res.append('f1') + self.assertFalse(f2.alive) + res.append('D') + f2() # nothing because previously called by gc + + expected = ['A', 'f3', 'B', 'C', 'f4', 'f2', 'f1', 'D'] + self.assertEqual(res, expected) + + def test_all_freed(self): + # we want a weakrefable subclass of weakref.finalize + class MyFinalizer(weakref.finalize): + pass + + a = self.A() + res = [] + def callback(): + res.append(123) + f = MyFinalizer(a, callback) + + wr_callback = weakref.ref(callback) + wr_f = weakref.ref(f) + del callback, f + + self.assertIsNotNone(wr_callback()) + self.assertIsNotNone(wr_f()) + + del a + self._collect_if_necessary() + + self.assertIsNone(wr_callback()) + self.assertIsNone(wr_f()) + self.assertEqual(res, [123]) + + @classmethod + def run_in_child(cls): + def error(): + # Create an atexit finalizer from inside a finalizer called + # at exit. This should be the next to be run. + g1 = weakref.finalize(cls, print, 'g1') + print('f3 error') + 1/0 + + # cls should stay alive till atexit callbacks run + f1 = weakref.finalize(cls, print, 'f1', _global_var) + f2 = weakref.finalize(cls, print, 'f2', _global_var) + f3 = weakref.finalize(cls, error) + f4 = weakref.finalize(cls, print, 'f4', _global_var) + + assert f1.atexit == True + f2.atexit = False + assert f3.atexit == True + assert f4.atexit == True + + def test_atexit(self): + prog = ('from test.test_weakref import FinalizeTestCase;'+ + 'FinalizeTestCase.run_in_child()') + rc, out, err = script_helper.assert_python_ok('-c', prog) + out = out.decode('ascii').splitlines() + self.assertEqual(out, ['f4 foobar', 'f3 error', 'g1', 'f1 foobar']) + self.assertTrue(b'ZeroDivisionError' in err) + + libreftest = """ Doctest for examples in the library reference: weakref.rst >>> import weakref @@ -1476,10 +1788,12 @@ __test__ = {'libreftest' : libreftest} def test_main(): support.run_unittest( ReferencesTestCase, + WeakMethodTestCase, MappingTestCase, WeakValueDictionaryTestCase, WeakKeyDictionaryTestCase, SubclassableWeakrefTestCase, + FinalizeTestCase, ) support.run_doctest(sys.modules[__name__]) diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index cb4cde9bc6..ef4ce552f1 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -8,7 +8,7 @@ threading = support.import_module("threading") from platform import machine # Do this first so test will be skipped if module doesn't exist -support.import_module('winreg') +support.import_module('winreg', required_on=['win']) # Now import everything from winreg import * @@ -57,13 +57,13 @@ class BaseWinregTests(unittest.TestCase): def delete_tree(self, root, subkey): try: hkey = OpenKey(root, subkey, KEY_ALL_ACCESS) - except WindowsError: + except OSError: # subkey does not exist return while True: try: subsubkey = EnumKey(hkey, 0) - except WindowsError: + except OSError: # no more subkeys break self.delete_tree(hkey, subsubkey) @@ -100,7 +100,7 @@ class BaseWinregTests(unittest.TestCase): QueryInfoKey(int_sub_key) self.fail("It appears the CloseKey() function does " "not close the actual key!") - except EnvironmentError: + except OSError: pass # ... and close that key that way :-) int_key = int(key) @@ -109,7 +109,7 @@ class BaseWinregTests(unittest.TestCase): QueryInfoKey(int_key) self.fail("It appears the key.Close() function " "does not close the actual key!") - except EnvironmentError: + except OSError: pass def _read_test_data(self, root_key, subkeystr="sub_key", OpenKey=OpenKey): @@ -126,7 +126,7 @@ class BaseWinregTests(unittest.TestCase): while 1: try: data = EnumValue(sub_key, index) - except EnvironmentError: + except OSError: break self.assertEqual(data in test_data, True, "Didn't read back the correct test data") @@ -147,7 +147,7 @@ class BaseWinregTests(unittest.TestCase): try: EnumKey(key, 1) self.fail("Was able to get a second key when I only have one!") - except EnvironmentError: + except OSError: pass key.Close() @@ -171,7 +171,7 @@ class BaseWinregTests(unittest.TestCase): # Shouldnt be able to delete it twice! DeleteKey(key, subkeystr) self.fail("Deleting the key twice succeeded") - except EnvironmentError: + except OSError: pass key.Close() DeleteKey(root_key, test_key_name) @@ -179,7 +179,7 @@ class BaseWinregTests(unittest.TestCase): try: key = OpenKey(root_key, test_key_name) self.fail("Could open the non-existent key") - except WindowsError: # Use this error name this time + except OSError: # Use this error name this time pass def _test_all(self, root_key, subkeystr="sub_key"): @@ -230,7 +230,7 @@ class LocalWinregTests(BaseWinregTests): def test_inexistant_remote_registry(self): connect = lambda: ConnectRegistry("abcdefghijkl", HKEY_CURRENT_USER) - self.assertRaises(WindowsError, connect) + self.assertRaises(OSError, connect) def testExpandEnvironmentStrings(self): r = ExpandEnvironmentStrings("%windir%\\test") @@ -242,8 +242,8 @@ class LocalWinregTests(BaseWinregTests): try: with ConnectRegistry(None, HKEY_LOCAL_MACHINE) as h: self.assertNotEqual(h.handle, 0) - raise WindowsError - except WindowsError: + raise OSError + except OSError: self.assertEqual(h.handle, 0) def test_changing_value(self): @@ -407,7 +407,7 @@ class Win64WinregTests(BaseWinregTests): open_fail = lambda: OpenKey(HKEY_CURRENT_USER, test_reflect_key_name, 0, KEY_READ | KEY_WOW64_64KEY) - self.assertRaises(WindowsError, open_fail) + self.assertRaises(OSError, open_fail) # Now explicitly open the 64-bit version of the key with OpenKey(HKEY_CURRENT_USER, test_reflect_key_name, 0, @@ -447,7 +447,7 @@ class Win64WinregTests(BaseWinregTests): open_fail = lambda: OpenKeyEx(HKEY_CURRENT_USER, test_reflect_key_name, 0, KEY_READ | KEY_WOW64_64KEY) - self.assertRaises(WindowsError, open_fail) + self.assertRaises(OSError, open_fail) # Make sure the 32-bit key is actually there with OpenKeyEx(HKEY_CURRENT_USER, test_reflect_key_name, 0, diff --git a/Lib/test/test_winsound.py b/Lib/test/test_winsound.py index eb7f75f066..61d864a648 100644 --- a/Lib/test/test_winsound.py +++ b/Lib/test/test_winsound.py @@ -22,7 +22,7 @@ def has_sound(sound): key = winreg.OpenKeyEx(winreg.HKEY_CURRENT_USER, "AppEvents\Schemes\Apps\.Default\{0}\.Default".format(sound)) return winreg.EnumValue(key, 0)[1] != "" - except WindowsError: + except OSError: return False class BeepTest(unittest.TestCase): diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 54965345c7..614e598f6c 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -10,6 +10,7 @@ import io import operator import pickle import sys +import types import unittest import weakref @@ -240,7 +241,6 @@ class ElementTreeTest(unittest.TestCase): self.assertEqual(ET.XML, ET.fromstring) self.assertEqual(ET.PI, ET.ProcessingInstruction) - self.assertEqual(ET.XMLParser, ET.XMLTreeBuilder) def test_simpleops(self): # Basic method sanity checks. @@ -433,15 +433,6 @@ class ElementTreeTest(unittest.TestCase): ' <empty-element />\n' '</root>') - parser = ET.XMLTreeBuilder() # 1.2 compatibility - parser.feed(data) - self.serialize_check(parser.close(), - '<root>\n' - ' <element key="value">text</element>\n' - ' <element>text</element>tail\n' - ' <empty-element />\n' - '</root>') - target = ET.TreeBuilder() parser = ET.XMLParser(target=target) parser.feed(data) @@ -959,6 +950,160 @@ class ElementTreeTest(unittest.TestCase): self.assertEqual(serialized, expected) +class XMLPullParserTest(unittest.TestCase): + + def _feed(self, parser, data, chunk_size=None): + if chunk_size is None: + parser.feed(data) + else: + for i in range(0, len(data), chunk_size): + parser.feed(data[i:i+chunk_size]) + + def assert_event_tags(self, parser, expected): + events = parser.read_events() + self.assertEqual([(action, elem.tag) for action, elem in events], + expected) + + def test_simple_xml(self): + for chunk_size in (None, 1, 5): + with self.subTest(chunk_size=chunk_size): + parser = ET.XMLPullParser() + self.assert_event_tags(parser, []) + self._feed(parser, "<!-- comment -->\n", chunk_size) + self.assert_event_tags(parser, []) + self._feed(parser, + "<root>\n <element key='value'>text</element", + chunk_size) + self.assert_event_tags(parser, []) + self._feed(parser, ">\n", chunk_size) + self.assert_event_tags(parser, [('end', 'element')]) + self._feed(parser, "<element>text</element>tail\n", chunk_size) + self._feed(parser, "<empty-element/>\n", chunk_size) + self.assert_event_tags(parser, [ + ('end', 'element'), + ('end', 'empty-element'), + ]) + self._feed(parser, "</root>\n", chunk_size) + self.assert_event_tags(parser, [('end', 'root')]) + self.assertIsNone(parser.close()) + + def test_feed_while_iterating(self): + parser = ET.XMLPullParser() + it = parser.read_events() + self._feed(parser, "<root>\n <element key='value'>text</element>\n") + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + self._feed(parser, "</root>\n") + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'root')) + with self.assertRaises(StopIteration): + next(it) + + def test_simple_xml_with_ns(self): + parser = ET.XMLPullParser() + self.assert_event_tags(parser, []) + self._feed(parser, "<!-- comment -->\n") + self.assert_event_tags(parser, []) + self._feed(parser, "<root xmlns='namespace'>\n") + self.assert_event_tags(parser, []) + self._feed(parser, "<element key='value'>text</element") + self.assert_event_tags(parser, []) + self._feed(parser, ">\n") + self.assert_event_tags(parser, [('end', '{namespace}element')]) + self._feed(parser, "<element>text</element>tail\n") + self._feed(parser, "<empty-element/>\n") + self.assert_event_tags(parser, [ + ('end', '{namespace}element'), + ('end', '{namespace}empty-element'), + ]) + self._feed(parser, "</root>\n") + self.assert_event_tags(parser, [('end', '{namespace}root')]) + self.assertIsNone(parser.close()) + + def test_ns_events(self): + parser = ET.XMLPullParser(events=('start-ns', 'end-ns')) + self._feed(parser, "<!-- comment -->\n") + self._feed(parser, "<root xmlns='namespace'>\n") + self.assertEqual( + list(parser.read_events()), + [('start-ns', ('', 'namespace'))]) + self._feed(parser, "<element key='value'>text</element") + self._feed(parser, ">\n") + self._feed(parser, "<element>text</element>tail\n") + self._feed(parser, "<empty-element/>\n") + self._feed(parser, "</root>\n") + self.assertEqual(list(parser.read_events()), [('end-ns', None)]) + self.assertIsNone(parser.close()) + + def test_events(self): + parser = ET.XMLPullParser(events=()) + self._feed(parser, "<root/>\n") + self.assert_event_tags(parser, []) + + parser = ET.XMLPullParser(events=('start', 'end')) + self._feed(parser, "<!-- comment -->\n") + self.assert_event_tags(parser, []) + self._feed(parser, "<root>\n") + self.assert_event_tags(parser, [('start', 'root')]) + self._feed(parser, "<element key='value'>text</element") + self.assert_event_tags(parser, [('start', 'element')]) + self._feed(parser, ">\n") + self.assert_event_tags(parser, [('end', 'element')]) + self._feed(parser, + "<element xmlns='foo'>text<empty-element/></element>tail\n") + self.assert_event_tags(parser, [ + ('start', '{foo}element'), + ('start', '{foo}empty-element'), + ('end', '{foo}empty-element'), + ('end', '{foo}element'), + ]) + self._feed(parser, "</root>") + self.assertIsNone(parser.close()) + self.assert_event_tags(parser, [('end', 'root')]) + + parser = ET.XMLPullParser(events=('start',)) + self._feed(parser, "<!-- comment -->\n") + self.assert_event_tags(parser, []) + self._feed(parser, "<root>\n") + self.assert_event_tags(parser, [('start', 'root')]) + self._feed(parser, "<element key='value'>text</element") + self.assert_event_tags(parser, [('start', 'element')]) + self._feed(parser, ">\n") + self.assert_event_tags(parser, []) + self._feed(parser, + "<element xmlns='foo'>text<empty-element/></element>tail\n") + self.assert_event_tags(parser, [ + ('start', '{foo}element'), + ('start', '{foo}empty-element'), + ]) + self._feed(parser, "</root>") + self.assertIsNone(parser.close()) + + def test_events_sequence(self): + # Test that events can be some sequence that's not just a tuple or list + eventset = {'end', 'start'} + parser = ET.XMLPullParser(events=eventset) + self._feed(parser, "<foo>bar</foo>") + self.assert_event_tags(parser, [('start', 'foo'), ('end', 'foo')]) + + class DummyIter: + def __init__(self): + self.events = iter(['start', 'end', 'start-ns']) + def __iter__(self): + return self + def __next__(self): + return next(self.events) + + parser = ET.XMLPullParser(events=DummyIter()) + self._feed(parser, "<foo>bar</foo>") + self.assert_event_tags(parser, [('start', 'foo'), ('end', 'foo')]) + + + def test_unknown_event(self): + with self.assertRaises(ValueError): + ET.XMLPullParser(events=('start', 'end', 'bogus')) + + # # xinclude tests (samples from appendix C of the xinclude specification) @@ -1300,7 +1445,7 @@ class BugsTest(unittest.TestCase): # Don't crash when using custom entities. ENTITIES = {'rsquo': '\u2019', 'lsquo': '\u2018'} - parser = ET.XMLTreeBuilder() + parser = ET.XMLParser() parser.entity.update(ENTITIES) parser.feed("""<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE patent-application-publication SYSTEM "pap-v15-2001-01-31.dtd" []> @@ -1638,6 +1783,11 @@ class ElementFindTest(unittest.TestCase): self.assertEqual(e.find('./tag[last()-1]').attrib['class'], 'c') self.assertEqual(e.find('./tag[last()-2]').attrib['class'], 'b') + self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[0]') + self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[-1]') + self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[last()-0]') + self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[last()+1]') + def test_findall(self): e = ET.XML(SAMPLE_XML) e[2] = ET.XML(SAMPLE_SECTION) @@ -1897,7 +2047,7 @@ class TreeBuilderTest(unittest.TestCase): # Mimick SimpleTAL's behaviour (issue #16089): both versions of # TreeBuilder should be able to cope with a subclass of the # pure Python Element class. - base = ET._Element + base = ET._Element_Py # Not from a C extension self.assertEqual(base.__module__, 'xml.etree.ElementTree') # Force some multiple inheritance with a C class to make things @@ -2256,6 +2406,18 @@ class IOTest(unittest.TestCase): ET.tostring(root, 'utf-16'), b''.join(ET.tostringlist(root, 'utf-16'))) + def test_short_empty_elements(self): + root = ET.fromstring('<tag>a<x />b<y></y>c</tag>') + self.assertEqual( + ET.tostring(root, 'unicode'), + '<tag>a<x />b<y />c</tag>') + self.assertEqual( + ET.tostring(root, 'unicode', short_empty_elements=True), + '<tag>a<x />b<y />c</tag>') + self.assertEqual( + ET.tostring(root, 'unicode', short_empty_elements=False), + '<tag>a<x></x>b<y></y>c</tag>') + class ParseErrorTest(unittest.TestCase): def test_subclass(self): @@ -2321,8 +2483,11 @@ class NoAcceleratorTest(unittest.TestCase): # Test that the C accelerator was not imported for pyET def test_correct_import_pyET(self): - self.assertEqual(pyET.Element.__module__, 'xml.etree.ElementTree') - self.assertEqual(pyET.SubElement.__module__, 'xml.etree.ElementTree') + # The type of methods defined in Python code is types.FunctionType, + # while the type of methods defined inside _elementtree is + # <class 'wrapper_descriptor'> + self.assertIsInstance(pyET.Element.__init__, types.FunctionType) + self.assertIsInstance(pyET.XMLParser.__init__, types.FunctionType) # -------------------------------------------------------------------- @@ -2392,6 +2557,7 @@ def test_main(module=None): ElementIterTest, TreeBuilderTest, XMLParserTest, + XMLPullParserTest, BugsTest, ] diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index bcaa724344..b3ff7ae37d 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -2,6 +2,7 @@ import sys, struct from test import support from test.support import import_fresh_module +import types import unittest cET = import_fresh_module('xml.etree.ElementTree', @@ -33,14 +34,22 @@ class TestAliasWorking(unittest.TestCase): @unittest.skipUnless(cET, 'requires _elementtree') +@support.cpython_only class TestAcceleratorImported(unittest.TestCase): # Test that the C accelerator was imported, as expected def test_correct_import_cET(self): + # SubElement is a function so it retains _elementtree as its module. self.assertEqual(cET.SubElement.__module__, '_elementtree') def test_correct_import_cET_alias(self): self.assertEqual(cET_alias.SubElement.__module__, '_elementtree') + def test_parser_comes_from_C(self): + # The type of methods defined in Python code is types.FunctionType, + # while the type of methods defined inside _elementtree is + # <class 'wrapper_descriptor'> + self.assertNotIsInstance(cET.Element.__init__, types.FunctionType) + @unittest.skipUnless(cET, 'requires _elementtree') @support.cpython_only diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index cc52259c82..99b3eda13e 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -15,6 +15,10 @@ import contextlib from test import support try: + import gzip +except ImportError: + gzip = None +try: import threading except ImportError: threading = None @@ -216,7 +220,7 @@ class XMLRPCTestCase(unittest.TestCase): xmlrpc.client.ServerProxy('https://localhost:9999').bad_function() except NotImplementedError: self.assertFalse(has_ssl, "xmlrpc client's error with SSL support") - except socket.error: + except OSError: self.assertTrue(has_ssl) class HelperTestCase(unittest.TestCase): @@ -376,6 +380,11 @@ def http_server(evt, numrequests, requestHandler=None): if name == 'div': return 'This is the div function' + class Fixture: + @staticmethod + def getData(): + return '42' + def my_function(): '''This is my function''' return True @@ -407,7 +416,8 @@ def http_server(evt, numrequests, requestHandler=None): serv.register_function(pow) serv.register_function(lambda x,y: x+y, 'add') serv.register_function(my_function) - serv.register_instance(TestInstanceClass()) + testInstance = TestInstanceClass() + serv.register_instance(testInstance, allow_dotted_names=True) evt.set() # handle up to 'numrequests' requests @@ -500,7 +510,7 @@ def is_unavailable_exception(e): return True exc_mess = e.headers.get('X-exception') except AttributeError: - # Ignore socket.errors here. + # Ignore OSErrors here. exc_mess = str(e) if exc_mess and 'temporarily unavailable' in exc_mess.lower(): @@ -515,7 +525,7 @@ def make_request_and_skipIf(condition, reason): def make_request_and_skip(self): try: xmlrpclib.ServerProxy(URL).my_function() - except (xmlrpclib.ProtocolError, socket.error) as e: + except (xmlrpclib.ProtocolError, OSError) as e: if not is_unavailable_exception(e): raise raise unittest.SkipTest(reason) @@ -553,7 +563,7 @@ class SimpleServerTestCase(BaseServerTestCase): try: p = xmlrpclib.ServerProxy(URL) self.assertEqual(p.pow(6,8), 6**8) - except (xmlrpclib.ProtocolError, socket.error) as e: + except (xmlrpclib.ProtocolError, OSError) as e: # ignore failures due to non-blocking socket 'unavailable' errors if not is_unavailable_exception(e): # protocol error; provide additional information in test output @@ -566,7 +576,7 @@ class SimpleServerTestCase(BaseServerTestCase): p = xmlrpclib.ServerProxy(URL) self.assertEqual(p.add(start_string, end_string), start_string + end_string) - except (xmlrpclib.ProtocolError, socket.error) as e: + except (xmlrpclib.ProtocolError, OSError) as e: # ignore failures due to non-blocking socket 'unavailable' errors if not is_unavailable_exception(e): # protocol error; provide additional information in test output @@ -587,12 +597,13 @@ class SimpleServerTestCase(BaseServerTestCase): def test_introspection1(self): expected_methods = set(['pow', 'div', 'my_function', 'add', 'system.listMethods', 'system.methodHelp', - 'system.methodSignature', 'system.multicall']) + 'system.methodSignature', 'system.multicall', + 'Fixture']) try: p = xmlrpclib.ServerProxy(URL) meth = p.system.listMethods() self.assertEqual(set(meth), expected_methods) - except (xmlrpclib.ProtocolError, socket.error) as e: + except (xmlrpclib.ProtocolError, OSError) as e: # ignore failures due to non-blocking socket 'unavailable' errors if not is_unavailable_exception(e): # protocol error; provide additional information in test output @@ -605,7 +616,7 @@ class SimpleServerTestCase(BaseServerTestCase): p = xmlrpclib.ServerProxy(URL) divhelp = p.system.methodHelp('div') self.assertEqual(divhelp, 'This is the div function') - except (xmlrpclib.ProtocolError, socket.error) as e: + except (xmlrpclib.ProtocolError, OSError) as e: # ignore failures due to non-blocking socket 'unavailable' errors if not is_unavailable_exception(e): # protocol error; provide additional information in test output @@ -619,7 +630,7 @@ class SimpleServerTestCase(BaseServerTestCase): p = xmlrpclib.ServerProxy(URL) myfunction = p.system.methodHelp('my_function') self.assertEqual(myfunction, 'This is my function') - except (xmlrpclib.ProtocolError, socket.error) as e: + except (xmlrpclib.ProtocolError, OSError) as e: # ignore failures due to non-blocking socket 'unavailable' errors if not is_unavailable_exception(e): # protocol error; provide additional information in test output @@ -632,7 +643,7 @@ class SimpleServerTestCase(BaseServerTestCase): p = xmlrpclib.ServerProxy(URL) divsig = p.system.methodSignature('div') self.assertEqual(divsig, 'signatures not supported') - except (xmlrpclib.ProtocolError, socket.error) as e: + except (xmlrpclib.ProtocolError, OSError) as e: # ignore failures due to non-blocking socket 'unavailable' errors if not is_unavailable_exception(e): # protocol error; provide additional information in test output @@ -649,7 +660,7 @@ class SimpleServerTestCase(BaseServerTestCase): self.assertEqual(add_result, 2+3) self.assertEqual(pow_result, 6**8) self.assertEqual(div_result, 127//42) - except (xmlrpclib.ProtocolError, socket.error) as e: + except (xmlrpclib.ProtocolError, OSError) as e: # ignore failures due to non-blocking socket 'unavailable' errors if not is_unavailable_exception(e): # protocol error; provide additional information in test output @@ -670,7 +681,7 @@ class SimpleServerTestCase(BaseServerTestCase): self.assertEqual(result.results[0]['faultString'], '<class \'Exception\'>:method "this_is_not_exists" ' 'is not supported') - except (xmlrpclib.ProtocolError, socket.error) as e: + except (xmlrpclib.ProtocolError, OSError) as e: # ignore failures due to non-blocking socket 'unavailable' errors if not is_unavailable_exception(e): # protocol error; provide additional information in test output @@ -686,6 +697,12 @@ class SimpleServerTestCase(BaseServerTestCase): # This avoids waiting for the socket timeout. self.test_simple1() + def test_allow_dotted_names_true(self): + # XXX also need allow_dotted_names_false test. + server = xmlrpclib.ServerProxy("http://%s:%d/RPC2" % (ADDR, PORT)) + data = server.Fixture.getData() + self.assertEqual(data, '42') + def test_unicode_host(self): server = xmlrpclib.ServerProxy("http://%s:%d/RPC2" % (ADDR, PORT)) self.assertEqual(server.add("a", "\xe9"), "a\xe9") @@ -793,6 +810,7 @@ class KeepaliveServerTestCase2(BaseKeepaliveServerTestCase): #A test case that verifies that gzip encoding works in both directions #(for a request and the response) +@unittest.skipIf(gzip is None, 'requires gzip') class GzipServerTestCase(BaseServerTestCase): #a request handler that supports keep-alive and logs requests into a #class variable @@ -923,7 +941,7 @@ class FailingServerTestCase(unittest.TestCase): try: p = xmlrpclib.ServerProxy(URL) self.assertEqual(p.pow(6,8), 6**8) - except (xmlrpclib.ProtocolError, socket.error) as e: + except (xmlrpclib.ProtocolError, OSError) as e: # ignore failures due to non-blocking socket 'unavailable' errors if not is_unavailable_exception(e): # protocol error; provide additional information in test output @@ -936,7 +954,7 @@ class FailingServerTestCase(unittest.TestCase): try: p = xmlrpclib.ServerProxy(URL) p.pow(6,8) - except (xmlrpclib.ProtocolError, socket.error) as e: + except (xmlrpclib.ProtocolError, OSError) as e: # ignore failures due to non-blocking socket 'unavailable' errors if not is_unavailable_exception(e) and hasattr(e, "headers"): # The two server-side error headers shouldn't be sent back in this case @@ -956,7 +974,7 @@ class FailingServerTestCase(unittest.TestCase): try: p = xmlrpclib.ServerProxy(URL) p.pow(6,8) - except (xmlrpclib.ProtocolError, socket.error) as e: + except (xmlrpclib.ProtocolError, OSError) as e: # ignore failures due to non-blocking socket 'unavailable' errors if not is_unavailable_exception(e) and hasattr(e, "headers"): # We should get error info in the response @@ -1084,23 +1102,13 @@ class UseBuiltinTypesTestCase(unittest.TestCase): @support.reap_threads def test_main(): - xmlrpc_tests = [XMLRPCTestCase, HelperTestCase, DateTimeTestCase, - BinaryTestCase, FaultTestCase] - xmlrpc_tests.append(UseBuiltinTypesTestCase) - xmlrpc_tests.append(SimpleServerTestCase) - xmlrpc_tests.append(KeepaliveServerTestCase1) - xmlrpc_tests.append(KeepaliveServerTestCase2) - try: - import gzip - xmlrpc_tests.append(GzipServerTestCase) - except ImportError: - pass #gzip not supported in this build - xmlrpc_tests.append(MultiPathServerTestCase) - xmlrpc_tests.append(ServerProxyTestCase) - xmlrpc_tests.append(FailingServerTestCase) - xmlrpc_tests.append(CGIHandlerTestCase) - - support.run_unittest(*xmlrpc_tests) + support.run_unittest(XMLRPCTestCase, HelperTestCase, DateTimeTestCase, + BinaryTestCase, FaultTestCase, UseBuiltinTypesTestCase, + SimpleServerTestCase, KeepaliveServerTestCase1, + KeepaliveServerTestCase2, GzipServerTestCase, + MultiPathServerTestCase, ServerProxyTestCase, FailingServerTestCase, + CGIHandlerTestCase) + if __name__ == "__main__": test_main() diff --git a/Lib/test/test_xmlrpc_net.py b/Lib/test/test_xmlrpc_net.py index dfb5f9aa3d..00aca199df 100644 --- a/Lib/test/test_xmlrpc_net.py +++ b/Lib/test/test_xmlrpc_net.py @@ -9,32 +9,7 @@ from test import support import xmlrpc.client as xmlrpclib -class CurrentTimeTest(unittest.TestCase): - - def test_current_time(self): - # Get the current time from xmlrpc.com. This code exercises - # the minimal HTTP functionality in xmlrpclib. - self.skipTest("time.xmlrpc.com is unreliable") - server = xmlrpclib.ServerProxy("http://time.xmlrpc.com/RPC2") - try: - t0 = server.currentTime.getCurrentTime() - except socket.error as e: - self.skipTest("network error: %s" % e) - return - - # Perform a minimal sanity check on the result, just to be sure - # the request means what we think it means. - t1 = xmlrpclib.DateTime() - - dt0 = xmlrpclib._datetime_type(t0.value) - dt1 = xmlrpclib._datetime_type(t1.value) - if dt0 > dt1: - delta = dt0 - dt1 - else: - delta = dt1 - dt0 - # The difference between the system time here and the system - # time on the server should not be too big. - self.assertTrue(delta.days <= 1) +class PythonBuildersTest(unittest.TestCase): def test_python_builders(self): # Get the list of builders from the XMLRPC buildbot interface at @@ -42,7 +17,7 @@ class CurrentTimeTest(unittest.TestCase): server = xmlrpclib.ServerProxy("http://buildbot.python.org/all/xmlrpc/") try: builders = server.getAllBuilders() - except socket.error as e: + except OSError as e: self.skipTest("network error: %s" % e) return self.addCleanup(lambda: server('close')()) @@ -55,7 +30,7 @@ class CurrentTimeTest(unittest.TestCase): def test_main(): support.requires("network") - support.run_unittest(CurrentTimeTest) + support.run_unittest(PythonBuildersTest) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index ad0c0b7b41..899a3fbc40 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -1,7 +1,7 @@ import io import os import sys -import imp +import importlib.util import time import shutil import struct @@ -557,7 +557,7 @@ class PyZipFileTests(unittest.TestCase): if os.altsep is not None: path_split.extend(fn.split(os.altsep)) if '__pycache__' in path_split: - fn = imp.source_from_cache(fn) + fn = importlib.util.source_from_cache(fn) else: fn = fn[:-1] @@ -913,7 +913,7 @@ class OtherTests(unittest.TestCase): try: with zipfile.ZipFile(TESTFN, 'a') as zf: zf.writestr(filename, content) - except IOError: + except OSError: self.fail('Could not append data to a non-existent zip file.') self.assertTrue(os.path.exists(TESTFN)) @@ -985,7 +985,7 @@ class OtherTests(unittest.TestCase): fp.seek(0, 0) self.assertTrue(zipfile.is_zipfile(fp)) - def test_non_existent_file_raises_IOError(self): + def test_non_existent_file_raises_OSError(self): # make sure we don't raise an AttributeError when a partially-constructed # ZipFile instance is finalized; this tests for regression on SF tracker # bug #403871. @@ -997,7 +997,7 @@ class OtherTests(unittest.TestCase): # it is ignored, but the user should be sufficiently annoyed by # the message on the output that regression will be noticed # quickly. - self.assertRaises(IOError, zipfile.ZipFile, TESTFN) + self.assertRaises(OSError, zipfile.ZipFile, TESTFN) def test_empty_file_raises_BadZipFile(self): f = open(TESTFN, 'w') @@ -1184,7 +1184,7 @@ class OtherTests(unittest.TestCase): def test_open_empty_file(self): # Issue 1710703: Check that opening a file with less than 22 bytes # raises a BadZipFile exception (rather than the previously unhelpful - # IOError) + # OSError) f = open(TESTFN, 'w') f.close() self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, TESTFN, 'r') @@ -1668,6 +1668,5 @@ class LzmaUniversalNewlineTests(AbstractUniversalNewlineTests, unittest.TestCase): compression = zipfile.ZIP_LZMA - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index f7cb8b977f..3f16041f51 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -1,13 +1,12 @@ import sys import os import marshal -import imp +import importlib.util import struct import time import unittest from test import support -from test.test_importhooks import ImportHooksBaseTestCase, test_src, test_co from zipfile import ZipFile, ZipInfo, ZIP_STORED, ZIP_DEFLATED @@ -17,6 +16,14 @@ import doctest import inspect import io from traceback import extract_tb, extract_stack, print_tb + +test_src = """\ +def get_name(): + return __name__ +def get_file(): + return __file__ +""" +test_co = compile(test_src, "<???>", "exec") raise_src = 'def do_raise(): raise TypeError\n' def make_pyc(co, mtime, size): @@ -27,7 +34,8 @@ def make_pyc(co, mtime, size): mtime = int(mtime) else: mtime = int(-0x100000000 + int(mtime)) - pyc = imp.get_magic() + struct.pack("<ii", int(mtime), size & 0xFFFFFFFF) + data + pyc = (importlib.util.MAGIC_NUMBER + + struct.pack("<ii", int(mtime), size & 0xFFFFFFFF) + data) return pyc def module_path_to_dotted_name(path): @@ -42,10 +50,27 @@ TESTPACK = "ziptestpackage" TESTPACK2 = "ziptestpackage2" TEMP_ZIP = os.path.abspath("junk95142.zip") -pyc_file = imp.cache_from_source(TESTMOD + '.py') +pyc_file = importlib.util.cache_from_source(TESTMOD + '.py') pyc_ext = ('.pyc' if __debug__ else '.pyo') +class ImportHooksBaseTestCase(unittest.TestCase): + + def setUp(self): + self.path = sys.path[:] + self.meta_path = sys.meta_path[:] + self.path_hooks = sys.path_hooks[:] + sys.path_importer_cache.clear() + self.modules_before = support.modules_setup() + + def tearDown(self): + sys.path[:] = self.path + sys.meta_path[:] = self.meta_path + sys.path_hooks[:] = self.path_hooks + sys.path_importer_cache.clear() + support.modules_cleanup(*self.modules_before) + + class UncompressedZipImportTestCase(ImportHooksBaseTestCase): compression = ZIP_STORED @@ -196,6 +221,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): for name, (mtime, data) in files.items(): zinfo = ZipInfo(name, time.localtime(mtime)) zinfo.compress_type = self.compression + zinfo.comment = b"spam" z.writestr(zinfo, data) z.close() @@ -245,6 +271,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): for name, (mtime, data) in files.items(): zinfo = ZipInfo(name, time.localtime(mtime)) zinfo.compress_type = self.compression + zinfo.comment = b"eggs" z.writestr(zinfo, data) z.close() @@ -459,7 +486,7 @@ class BadFileZipImportTestCase(unittest.TestCase): self.assertRaises(error, z.load_module, 'abc') self.assertRaises(error, z.get_code, 'abc') - self.assertRaises(IOError, z.get_data, 'abc') + self.assertRaises(OSError, z.get_data, 'abc') self.assertRaises(error, z.get_source, 'abc') self.assertRaises(error, z.is_package, 'abc') finally: diff --git a/Lib/test/tf_inherit_check.py b/Lib/test/tf_inherit_check.py index 92ebd95e52..afe50d2325 100644 --- a/Lib/test/tf_inherit_check.py +++ b/Lib/test/tf_inherit_check.py @@ -11,7 +11,7 @@ try: try: os.write(fd, b"blat") - except os.error: + except OSError: # Success -- could not write to fd. sys.exit(0) else: |