diff options
Diffstat (limited to 'Lib/test')
362 files changed, 19364 insertions, 6003 deletions
diff --git a/Lib/test/__main__.py b/Lib/test/__main__.py index d5fbe159d7..19a6b2b890 100644 --- a/Lib/test/__main__.py +++ b/Lib/test/__main__.py @@ -1,3 +1,2 @@ -from test import regrtest - -regrtest.main_in_temp_cwd() +from test.libregrtest import main +main() diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 9b789c2a33..b5f4782546 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1628,13 +1628,33 @@ class _TestContainers(BaseTestCase): d = [a, b] e = self.list(d) self.assertEqual( - e[:], + [element[:] for element in e], [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [0, 1, 2, 3, 4, 0, 1, 2, 3, 4]] ) f = self.list([a]) a.append('hello') - self.assertEqual(f[:], [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 'hello']]) + self.assertEqual(f[0][:], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 'hello']) + + def test_list_proxy_in_list(self): + a = self.list([self.list(range(3)) for _i in range(3)]) + self.assertEqual([inner[:] for inner in a], [[0, 1, 2]] * 3) + + a[0][-1] = 55 + self.assertEqual(a[0][:], [0, 1, 55]) + for i in range(1, 3): + self.assertEqual(a[i][:], [0, 1, 2]) + + self.assertEqual(a[1].pop(), 2) + self.assertEqual(len(a[1]), 2) + for i in range(0, 3, 2): + self.assertEqual(len(a[i]), 3) + + del a + + b = self.list() + b.append(b) + del b def test_dict(self): d = self.dict() @@ -1646,6 +1666,52 @@ class _TestContainers(BaseTestCase): self.assertEqual(sorted(d.values()), [chr(i) for i in indices]) self.assertEqual(sorted(d.items()), [(i, chr(i)) for i in indices]) + def test_dict_proxy_nested(self): + pets = self.dict(ferrets=2, hamsters=4) + supplies = self.dict(water=10, feed=3) + d = self.dict(pets=pets, supplies=supplies) + + self.assertEqual(supplies['water'], 10) + self.assertEqual(d['supplies']['water'], 10) + + d['supplies']['blankets'] = 5 + self.assertEqual(supplies['blankets'], 5) + self.assertEqual(d['supplies']['blankets'], 5) + + d['supplies']['water'] = 7 + self.assertEqual(supplies['water'], 7) + self.assertEqual(d['supplies']['water'], 7) + + del pets + del supplies + self.assertEqual(d['pets']['ferrets'], 2) + d['supplies']['blankets'] = 11 + self.assertEqual(d['supplies']['blankets'], 11) + + pets = d['pets'] + supplies = d['supplies'] + supplies['water'] = 7 + self.assertEqual(supplies['water'], 7) + self.assertEqual(d['supplies']['water'], 7) + + d.clear() + self.assertEqual(len(d), 0) + self.assertEqual(supplies['water'], 7) + self.assertEqual(pets['hamsters'], 4) + + l = self.list([pets, supplies]) + l[0]['marmots'] = 1 + self.assertEqual(pets['marmots'], 1) + self.assertEqual(l[0]['marmots'], 1) + + del pets + del supplies + self.assertEqual(l[0]['marmots'], 1) + + outer = self.list([[88, 99], l]) + self.assertIsInstance(outer[0], list) # Not a ListProxy + self.assertEqual(outer[-1][-1]['feed'], 3) + def test_namespace(self): n = self.Namespace() n.name = 'Bob' @@ -1668,6 +1734,10 @@ def sqr(x, wait=0.0): def mul(x, y): return x*y +def raise_large_valuerror(wait): + time.sleep(wait) + raise ValueError("x" * 1024**2) + class SayWhenError(ValueError): pass def exception_throwing_generator(total, when): @@ -1910,6 +1980,26 @@ class _TestPool(BaseTestCase): with self.assertRaises(RuntimeError): p.apply(self._test_wrapped_exception) + def test_map_no_failfast(self): + # Issue #23992: the fail-fast behaviour when an exception is raised + # during map() would make Pool.join() deadlock, because a worker + # process would fill the result queue (after the result handler thread + # terminated, hence not draining it anymore). + + t_start = time.time() + + with self.assertRaises(ValueError): + with self.Pool(2) as p: + try: + p.map(raise_large_valuerror, [0, 1]) + finally: + time.sleep(0.5) + p.close() + p.join() + + # check that we indeed waited for all jobs + self.assertGreater(time.time() - t_start, 0.9) + def raising(): raise KeyError("key") @@ -3780,7 +3870,7 @@ class TestSemaphoreTracker(unittest.TestCase): p.stderr.close() expected = 'semaphore_tracker: There appear to be 2 leaked semaphores' self.assertRegex(err, expected) - self.assertRegex(err, 'semaphore_tracker: %r: \[Errno' % name1) + self.assertRegex(err, r'semaphore_tracker: %r: \[Errno' % name1) # # Mixins diff --git a/Lib/test/ann_module.py b/Lib/test/ann_module.py new file mode 100644 index 0000000000..9e6b87dac4 --- /dev/null +++ b/Lib/test/ann_module.py @@ -0,0 +1,53 @@ + + +""" +The module for testing variable annotations. +Empty lines above are for good reason (testing for correct line numbers) +""" + +from typing import Optional + +__annotations__[1] = 2 + +class C: + + x = 5; y: Optional['C'] = None + +from typing import Tuple +x: int = 5; y: str = x; f: Tuple[int, int] + +class M(type): + + __annotations__['123'] = 123 + o: type = object + +(pars): bool = True + +class D(C): + j: str = 'hi'; k: str= 'bye' + +from types import new_class +h_class = new_class('H', (C,)) +j_class = new_class('J') + +class F(): + z: int = 5 + def __init__(self, x): + pass + +class Y(F): + def __init__(self): + super(F, self).__init__(123) + +class Meta(type): + def __new__(meta, name, bases, namespace): + return super().__new__(meta, name, bases, namespace) + +class S(metaclass = Meta): + x: str = 'something' + y: str = 'something else' + +def foo(x: int = 10): + def bar(y: List[str]): + x: str = 'yes' + bar() diff --git a/Lib/test/ann_module2.py b/Lib/test/ann_module2.py new file mode 100644 index 0000000000..76cf5b3ad9 --- /dev/null +++ b/Lib/test/ann_module2.py @@ -0,0 +1,36 @@ +""" +Some correct syntax for variable annotation here. +More examples are in test_grammar and test_parser. +""" + +from typing import no_type_check, ClassVar + +i: int = 1 +j: int +x: float = i/10 + +def f(): + class C: ... + return C() + +f().new_attr: object = object() + +class C: + def __init__(self, x: int) -> None: + self.x = x + +c = C(5) +c.new_attr: int = 10 + +__annotations__ = {} + + +@no_type_check +class NTC: + def meth(self, param: complex) -> None: + ... + +class CV: + var: ClassVar['CV'] + +CV.var = CV() diff --git a/Lib/test/ann_module3.py b/Lib/test/ann_module3.py new file mode 100644 index 0000000000..eccd7be22d --- /dev/null +++ b/Lib/test/ann_module3.py @@ -0,0 +1,18 @@ +""" +Correct syntax for variable annotation that should fail at runtime +in a certain manner. More examples are in test_grammar and test_parser. +""" + +def f_bad_ann(): + __annotations__[1] = 2 + +class C_OK: + def __init__(self, x: int) -> None: + self.x: no_such_name = x # This one is OK as proposed by Guido + +class D_bad_ann: + def __init__(self, x: int) -> None: + sfel.y: int = 0 + +def g_bad_ann(): + no_such_name.attr: int = 0 diff --git a/Lib/test/audiotests.py b/Lib/test/audiotests.py index 0ae2242851..d3e8e9ee44 100644 --- a/Lib/test/audiotests.py +++ b/Lib/test/audiotests.py @@ -1,9 +1,8 @@ from test.support import findfile, TESTFN, unlink -import unittest import array import io import pickle -import sys + class UnseekableIO(io.FileIO): def tell(self): diff --git a/Lib/test/autotest.py b/Lib/test/autotest.py index 41c2088727..fa85cc153a 100644 --- a/Lib/test/autotest.py +++ b/Lib/test/autotest.py @@ -1,6 +1,5 @@ # This should be equivalent to running regrtest.py from the cmdline. # It can be especially handy if you're in an interactive shell, e.g., # from test import autotest. - -from test import regrtest -regrtest.main() +from test.libregrtest import main +main() diff --git a/Lib/test/badsyntax_async1.py b/Lib/test/badsyntax_async1.py deleted file mode 100644 index fb85e29052..0000000000 --- a/Lib/test/badsyntax_async1.py +++ /dev/null @@ -1,2 +0,0 @@ -async def foo(a=await something()): - pass diff --git a/Lib/test/badsyntax_async2.py b/Lib/test/badsyntax_async2.py deleted file mode 100644 index fb85e29052..0000000000 --- a/Lib/test/badsyntax_async2.py +++ /dev/null @@ -1,2 +0,0 @@ -async def foo(a=await something()): - pass diff --git a/Lib/test/badsyntax_async3.py b/Lib/test/badsyntax_async3.py deleted file mode 100644 index dde1bc5955..0000000000 --- a/Lib/test/badsyntax_async3.py +++ /dev/null @@ -1,2 +0,0 @@ -async def foo(): - [i async for i in els] diff --git a/Lib/test/badsyntax_async4.py b/Lib/test/badsyntax_async4.py deleted file mode 100644 index d033b28114..0000000000 --- a/Lib/test/badsyntax_async4.py +++ /dev/null @@ -1,2 +0,0 @@ -async def foo(): - await diff --git a/Lib/test/badsyntax_async5.py b/Lib/test/badsyntax_async5.py deleted file mode 100644 index 9d19af6109..0000000000 --- a/Lib/test/badsyntax_async5.py +++ /dev/null @@ -1,2 +0,0 @@ -def foo(): - await something() diff --git a/Lib/test/badsyntax_async6.py b/Lib/test/badsyntax_async6.py deleted file mode 100644 index cb0a23d5f0..0000000000 --- a/Lib/test/badsyntax_async6.py +++ /dev/null @@ -1,2 +0,0 @@ -async def foo(): - yield diff --git a/Lib/test/badsyntax_async7.py b/Lib/test/badsyntax_async7.py deleted file mode 100644 index 51e4bf9b5b..0000000000 --- a/Lib/test/badsyntax_async7.py +++ /dev/null @@ -1,2 +0,0 @@ -async def foo(): - yield from [] diff --git a/Lib/test/badsyntax_async8.py b/Lib/test/badsyntax_async8.py deleted file mode 100644 index 3c636f9ac4..0000000000 --- a/Lib/test/badsyntax_async8.py +++ /dev/null @@ -1,2 +0,0 @@ -async def foo(): - await await fut diff --git a/Lib/test/capath/0e4015b9.0 b/Lib/test/capath/0e4015b9.0 deleted file mode 100644 index b6d259bcb2..0000000000 --- a/Lib/test/capath/0e4015b9.0 +++ /dev/null @@ -1,16 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv -bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG -A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo -b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 -aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ -Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm -Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv -EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl -bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN -AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h -TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 -C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= ------END CERTIFICATE----- diff --git a/Lib/test/capath/b1930218.0 b/Lib/test/capath/b1930218.0 new file mode 100644 index 0000000000..373349cae0 --- /dev/null +++ b/Lib/test/capath/b1930218.0 @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/Lib/test/capath/ce7b8643.0 b/Lib/test/capath/ce7b8643.0 deleted file mode 100644 index b6d259bcb2..0000000000 --- a/Lib/test/capath/ce7b8643.0 +++ /dev/null @@ -1,16 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv -bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG -A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo -b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 -aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ -Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm -Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv -EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl -bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN -AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h -TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 -C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= ------END CERTIFICATE----- diff --git a/Lib/test/capath/ceff1710.0 b/Lib/test/capath/ceff1710.0 new file mode 100644 index 0000000000..373349cae0 --- /dev/null +++ b/Lib/test/capath/ceff1710.0 @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/Lib/test/cmath_testcases.txt b/Lib/test/cmath_testcases.txt index 9b0865302e..dd7e458ddc 100644 --- a/Lib/test/cmath_testcases.txt +++ b/Lib/test/cmath_testcases.txt @@ -53,6 +53,12 @@ -- MPFR homepage at http://www.mpfr.org for more information about the -- MPFR project. +-- A minority of the test cases were generated with the help of +-- mpmath 0.19 at 100 bit accuracy (http://mpmath.org) to improve +-- coverage of real functions with real-valued arguments. These are +-- used in test.test_math.MathTests.test_testfile, as well as in +-- test_cmath. + -------------------------- -- acos: Inverse cosine -- @@ -848,6 +854,18 @@ atan0302 atan 9.9999999999999999e-161 -1.0 -> 0.78539816339744828 -184.553381029 atan0303 atan -1e-165 1.0 -> -0.78539816339744828 190.30984376228875 atan0304 atan -9.9998886718268301e-321 -1.0 -> -0.78539816339744828 -368.76019403576692 +-- Additional real values (mpmath) +atan0400 atan 1.7976931348623157e+308 0.0 -> 1.5707963267948966192 0.0 +atan0401 atan -1.7976931348623157e+308 0.0 -> -1.5707963267948966192 0.0 +atan0402 atan 1e-17 0.0 -> 1.0000000000000000715e-17 0.0 +atan0403 atan -1e-17 0.0 -> -1.0000000000000000715e-17 0.0 +atan0404 atan 0.0001 0.0 -> 0.000099999999666666673459 0.0 +atan0405 atan -0.0001 0.0 -> -0.000099999999666666673459 0.0 +atan0406 atan 0.999999999999999 0.0 -> 0.78539816339744781002 0.0 +atan0407 atan 1.000000000000001 0.0 -> 0.78539816339744886473 0.0 +atan0408 atan 14.101419947171719 0.0 -> 1.4999999999999999969 0.0 +atan0409 atan 1255.7655915007897 0.0 -> 1.5700000000000000622 0.0 + -- special values atan1000 atan -0.0 0.0 -> -0.0 0.0 atan1001 atan nan 0.0 -> nan 0.0 @@ -1514,6 +1532,11 @@ sqrt0131 sqrt -1.5477066694467245e-310 -0.0 -> 0.0 -1.2440685951533077e-155 sqrt0140 sqrt 1.6999999999999999e+308 -1.6999999999999999e+308 -> 1.4325088230154573e+154 -5.9336458271212207e+153 sqrt0141 sqrt -1.797e+308 -9.9999999999999999e+306 -> 3.7284476432057307e+152 -1.3410406899802901e+154 +-- Additional real values (mpmath) +sqrt0150 sqrt 1.7976931348623157e+308 0.0 -> 1.3407807929942596355e+154 0.0 +sqrt0151 sqrt 2.2250738585072014e-308 0.0 -> 1.4916681462400413487e-154 0.0 +sqrt0152 sqrt 5e-324 0.0 -> 2.2227587494850774834e-162 0.0 + -- special values sqrt1000 sqrt 0.0 0.0 -> 0.0 0.0 sqrt1001 sqrt -0.0 0.0 -> 0.0 0.0 @@ -1616,6 +1639,20 @@ exp0052 exp 710.0 1.5 -> 1.5802653829857376e+307 inf overflow exp0053 exp 710.0 1.6 -> -6.5231579995501372e+306 inf overflow exp0054 exp 710.0 2.8 -> -inf 7.4836177417448528e+307 overflow +-- Additional real values (mpmath) +exp0070 exp 1e-08 0.0 -> 1.00000001000000005 0.0 +exp0071 exp 0.0003 0.0 -> 1.0003000450045003375 0.0 +exp0072 exp 0.2 0.0 -> 1.2214027581601698475 0.0 +exp0073 exp 1.0 0.0 -> 2.7182818284590452354 0.0 +exp0074 exp -1e-08 0.0 -> 0.99999999000000005 0.0 +exp0075 exp -0.0003 0.0 -> 0.99970004499550033751 0.0 +exp0076 exp -1.0 0.0 -> 0.3678794411714423216 0.0 +exp0077 exp 2.220446049250313e-16 0.0 -> 1.000000000000000222 0.0 +exp0078 exp -1.1102230246251565e-16 0.0 -> 0.99999999999999988898 0.0 +exp0079 exp 2.302585092994046 0.0 -> 10.000000000000002171 0.0 +exp0080 exp -2.302585092994046 0.0 -> 0.099999999999999978292 0.0 +exp0081 exp 709.7827 0.0 -> 1.7976699566638014654e+308 0.0 + -- special values exp1000 exp 0.0 0.0 -> 1.0 0.0 exp1001 exp -0.0 0.0 -> 1.0 0.0 @@ -1708,6 +1745,23 @@ cosh0023 cosh 2.218885944363501 2.0015727395883687 -> -1.94294321081968 4.129026 cosh0030 cosh 710.5 2.3519999999999999 -> -1.2967465239355998e+308 1.3076707908857333e+308 cosh0031 cosh -710.5 0.69999999999999996 -> 1.4085466381392499e+308 -1.1864024666450239e+308 +-- Additional real values (mpmath) +cosh0050 cosh 1e-150 0.0 -> 1.0 0.0 +cosh0051 cosh 1e-18 0.0 -> 1.0 0.0 +cosh0052 cosh 1e-09 0.0 -> 1.0000000000000000005 0.0 +cosh0053 cosh 0.0003 0.0 -> 1.0000000450000003375 0.0 +cosh0054 cosh 0.2 0.0 -> 1.0200667556190758485 0.0 +cosh0055 cosh 1.0 0.0 -> 1.5430806348152437785 0.0 +cosh0056 cosh -1e-18 0.0 -> 1.0 -0.0 +cosh0057 cosh -0.0003 0.0 -> 1.0000000450000003375 -0.0 +cosh0058 cosh -1.0 0.0 -> 1.5430806348152437785 -0.0 +cosh0059 cosh 1.3169578969248168 0.0 -> 2.0000000000000001504 0.0 +cosh0060 cosh -1.3169578969248168 0.0 -> 2.0000000000000001504 -0.0 +cosh0061 cosh 17.328679513998633 0.0 -> 16777216.000000021938 0.0 +cosh0062 cosh 18.714973875118524 0.0 -> 67108864.000000043662 0.0 +cosh0063 cosh 709.7827 0.0 -> 8.9883497833190073272e+307 0.0 +cosh0064 cosh -709.7827 0.0 -> 8.9883497833190073272e+307 -0.0 + -- special values cosh1000 cosh 0.0 0.0 -> 1.0 0.0 cosh1001 cosh 0.0 inf -> nan 0.0 invalid ignore-imag-sign @@ -1800,6 +1854,24 @@ sinh0023 sinh 0.043713693678420068 0.22512549887532657 -> 0.042624198673416713 0 sinh0030 sinh 710.5 -2.3999999999999999 -> -1.3579970564885919e+308 -1.24394470907798e+308 sinh0031 sinh -710.5 0.80000000000000004 -> -1.2830671601735164e+308 1.3210954193997678e+308 +-- Additional real values (mpmath) +sinh0050 sinh 1e-100 0.0 -> 1.00000000000000002e-100 0.0 +sinh0051 sinh 5e-17 0.0 -> 4.9999999999999998955e-17 0.0 +sinh0052 sinh 1e-16 0.0 -> 9.999999999999999791e-17 0.0 +sinh0053 sinh 3.7e-08 0.0 -> 3.7000000000000008885e-8 0.0 +sinh0054 sinh 0.001 0.0 -> 0.0010000001666666750208 0.0 +sinh0055 sinh 0.2 0.0 -> 0.20133600254109399895 0.0 +sinh0056 sinh 1.0 0.0 -> 1.1752011936438014569 0.0 +sinh0057 sinh -3.7e-08 0.0 -> -3.7000000000000008885e-8 0.0 +sinh0058 sinh -0.001 0.0 -> -0.0010000001666666750208 0.0 +sinh0059 sinh -1.0 0.0 -> -1.1752011936438014569 0.0 +sinh0060 sinh 1.4436354751788103 0.0 -> 1.9999999999999999078 0.0 +sinh0061 sinh -1.4436354751788103 0.0 -> -1.9999999999999999078 0.0 +sinh0062 sinh 17.328679513998633 0.0 -> 16777215.999999992136 0.0 +sinh0063 sinh 18.714973875118524 0.0 -> 67108864.000000036211 0.0 +sinh0064 sinh 709.7827 0.0 -> 8.9883497833190073272e+307 0.0 +sinh0065 sinh -709.7827 0.0 -> -8.9883497833190073272e+307 0.0 + -- special values sinh1000 sinh 0.0 0.0 -> 0.0 0.0 sinh1001 sinh 0.0 inf -> 0.0 nan invalid ignore-real-sign @@ -1897,6 +1969,24 @@ tanh0031 tanh -711 7.4000000000000004 -> -1.0 0.0 tanh0032 tanh 1000 -2.3199999999999998 -> 1.0 0.0 tanh0033 tanh -1.0000000000000001e+300 -9.6699999999999999 -> -1.0 -0.0 +-- Additional real values (mpmath) +tanh0050 tanh 1e-100 0.0 -> 1.00000000000000002e-100 0.0 +tanh0051 tanh 5e-17 0.0 -> 4.9999999999999998955e-17 0.0 +tanh0052 tanh 1e-16 0.0 -> 9.999999999999999791e-17 0.0 +tanh0053 tanh 3.7e-08 0.0 -> 3.6999999999999983559e-8 0.0 +tanh0054 tanh 0.001 0.0 -> 0.00099999966666680002076 0.0 +tanh0055 tanh 0.2 0.0 -> 0.19737532022490401141 0.0 +tanh0056 tanh 1.0 0.0 -> 0.76159415595576488812 0.0 +tanh0057 tanh -3.7e-08 0.0 -> -3.6999999999999983559e-8 0.0 +tanh0058 tanh -0.001 0.0 -> -0.00099999966666680002076 0.0 +tanh0059 tanh -1.0 0.0 -> -0.76159415595576488812 0.0 +tanh0060 tanh 0.5493061443340549 0.0 -> 0.50000000000000003402 0.0 +tanh0061 tanh -0.5493061443340549 0.0 -> -0.50000000000000003402 0.0 +tanh0062 tanh 17.328679513998633 0.0 -> 0.99999999999999822364 0.0 +tanh0063 tanh 18.714973875118524 0.0 -> 0.99999999999999988898 0.0 +tanh0064 tanh 711 0.0 -> 1.0 0.0 +tanh0065 tanh 1.797e+308 0.0 -> 1.0 0.0 + --special values tanh1000 tanh 0.0 0.0 -> 0.0 0.0 tanh1001 tanh 0.0 inf -> nan nan invalid @@ -1985,6 +2075,22 @@ cos0021 cos 4.8048375263775256 0.0062248852898515658 -> 0.092318702015846243 0.0 cos0022 cos 7.9914515433858515 0.71659966615501436 -> -0.17375439906936566 -0.77217043527294582 cos0023 cos 0.45124351152540226 1.6992693993812158 -> 2.543477948972237 -1.1528193694875477 +-- Additional real values (mpmath) +cos0050 cos 1e-150 0.0 -> 1.0 -0.0 +cos0051 cos 1e-18 0.0 -> 1.0 -0.0 +cos0052 cos 1e-09 0.0 -> 0.9999999999999999995 -0.0 +cos0053 cos 0.0003 0.0 -> 0.9999999550000003375 -0.0 +cos0054 cos 0.2 0.0 -> 0.98006657784124162892 -0.0 +cos0055 cos 1.0 0.0 -> 0.5403023058681397174 -0.0 +cos0056 cos -1e-18 0.0 -> 1.0 0.0 +cos0057 cos -0.0003 0.0 -> 0.9999999550000003375 0.0 +cos0058 cos -1.0 0.0 -> 0.5403023058681397174 0.0 +cos0059 cos 1.0471975511965976 0.0 -> 0.50000000000000009945 -0.0 +cos0060 cos 2.5707963267948966 0.0 -> -0.84147098480789647357 -0.0 +cos0061 cos -2.5707963267948966 0.0 -> -0.84147098480789647357 0.0 +cos0062 cos 7.225663103256523 0.0 -> 0.58778525229247407559 -0.0 +cos0063 cos -8.79645943005142 0.0 -> -0.80901699437494722255 0.0 + -- special values cos1000 cos -0.0 0.0 -> 1.0 0.0 cos1001 cos -inf 0.0 -> nan 0.0 invalid ignore-imag-sign @@ -2073,6 +2179,22 @@ sin0021 sin 1.4342727387492671 0.81361889790284347 -> 1.3370960060947923 0.12336 sin0022 sin 1.1518087354403725 4.8597235966150558 -> 58.919141989603041 26.237003403758852 sin0023 sin 0.00087773078406649192 34.792379211312095 -> 565548145569.38245 644329685822700.62 +-- Additional real values (mpmath) +sin0050 sin 1e-100 0.0 -> 1.00000000000000002e-100 0.0 +sin0051 sin 3.7e-08 0.0 -> 3.6999999999999992001e-8 0.0 +sin0052 sin 0.001 0.0 -> 0.00099999983333334168748 0.0 +sin0053 sin 0.2 0.0 -> 0.19866933079506122634 0.0 +sin0054 sin 1.0 0.0 -> 0.84147098480789650665 0.0 +sin0055 sin -3.7e-08 0.0 -> -3.6999999999999992001e-8 0.0 +sin0056 sin -0.001 0.0 -> -0.00099999983333334168748 0.0 +sin0057 sin -1.0 0.0 -> -0.84147098480789650665 0.0 +sin0058 sin 0.5235987755982989 0.0 -> 0.50000000000000004642 0.0 +sin0059 sin -0.5235987755982989 0.0 -> -0.50000000000000004642 0.0 +sin0060 sin 2.6179938779914944 0.0 -> 0.49999999999999996018 -0.0 +sin0061 sin -2.6179938779914944 0.0 -> -0.49999999999999996018 -0.0 +sin0062 sin 7.225663103256523 0.0 -> 0.80901699437494673648 0.0 +sin0063 sin -8.79645943005142 0.0 -> -0.58778525229247340658 -0.0 + -- special values sin1000 sin -0.0 0.0 -> -0.0 0.0 sin1001 sin -inf 0.0 -> nan 0.0 invalid ignore-imag-sign @@ -2161,6 +2283,25 @@ tan0021 tan 1.7809617968443272 1.5052381702853379 -> -0.044066222118946903 1.093 tan0022 tan 1.1615313900880577 1.7956298728647107 -> 0.041793186826390362 1.0375339546034792 tan0023 tan 0.067014779477908945 5.8517361577457097 -> 2.2088639754800034e-06 0.9999836182420061 +-- Additional real values (mpmath) +tan0050 tan 1e-100 0.0 -> 1.00000000000000002e-100 0.0 +tan0051 tan 3.7e-08 0.0 -> 3.7000000000000017328e-8 0.0 +tan0052 tan 0.001 0.0 -> 0.0010000003333334666875 0.0 +tan0053 tan 0.2 0.0 -> 0.20271003550867249488 0.0 +tan0054 tan 1.0 0.0 -> 1.5574077246549022305 0.0 +tan0055 tan -3.7e-08 0.0 -> -3.7000000000000017328e-8 0.0 +tan0056 tan -0.001 0.0 -> -0.0010000003333334666875 0.0 +tan0057 tan -1.0 0.0 -> -1.5574077246549022305 0.0 +tan0058 tan 0.4636476090008061 0.0 -> 0.49999999999999997163 0.0 +tan0059 tan -0.4636476090008061 0.0 -> -0.49999999999999997163 0.0 +tan0060 tan 1.1071487177940904 0.0 -> 1.9999999999999995298 0.0 +tan0061 tan -1.1071487177940904 0.0 -> -1.9999999999999995298 0.0 +tan0062 tan 1.5 0.0 -> 14.101419947171719388 0.0 +tan0063 tan 1.57 0.0 -> 1255.7655915007896475 0.0 +tan0064 tan 1.5707963267948961 0.0 -> 1978937966095219.0538 0.0 +tan0065 tan 7.225663103256523 0.0 -> 1.3763819204711701522 0.0 +tan0066 tan -8.79645943005142 0.0 -> 0.7265425280053614098 0.0 + -- special values tan1000 tan -0.0 0.0 -> -0.0 0.0 tan1001 tan -inf 0.0 -> nan nan invalid diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 0860db80f5..d65186db0c 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2,14 +2,22 @@ See http://www.zope.org/Members/fdrake/DateTimeWiki/TestCases """ +from test.support import is_resource_enabled + +import itertools +import bisect import copy import decimal import sys +import os import pickle import random +import struct import unittest +from array import array + from operator import lt, le, gt, ge, eq, ne, truediv, floordiv, mod from test import support @@ -284,7 +292,8 @@ class TestTimeZone(unittest.TestCase): with self.assertRaises(TypeError): self.EST.dst(5) def test_tzname(self): - self.assertEqual('UTC+00:00', timezone(ZERO).tzname(None)) + self.assertEqual('UTC', timezone.utc.tzname(None)) + self.assertEqual('UTC', timezone(ZERO).tzname(None)) self.assertEqual('UTC-05:00', timezone(-5 * HOUR).tzname(None)) self.assertEqual('UTC+09:30', timezone(9.5 * HOUR).tzname(None)) self.assertEqual('UTC-00:01', timezone(timedelta(minutes=-1)).tzname(None)) @@ -1320,6 +1329,7 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase): green = pickler.dumps(orig, proto) derived = unpickler.loads(green) self.assertEqual(orig, derived) + self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) def test_compare(self): t1 = self.theclass(2, 3, 4) @@ -1558,13 +1568,32 @@ class TestDateTime(TestDate): self.assertEqual(dt, dt2) def test_isoformat(self): - t = self.theclass(2, 3, 2, 4, 5, 1, 123) - self.assertEqual(t.isoformat(), "0002-03-02T04:05:01.000123") - self.assertEqual(t.isoformat('T'), "0002-03-02T04:05:01.000123") - self.assertEqual(t.isoformat(' '), "0002-03-02 04:05:01.000123") - self.assertEqual(t.isoformat('\x00'), "0002-03-02\x0004:05:01.000123") + t = self.theclass(1, 2, 3, 4, 5, 1, 123) + self.assertEqual(t.isoformat(), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat('T'), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat(' '), "0001-02-03 04:05:01.000123") + self.assertEqual(t.isoformat('\x00'), "0001-02-03\x0004:05:01.000123") + self.assertEqual(t.isoformat(timespec='hours'), "0001-02-03T04") + self.assertEqual(t.isoformat(timespec='minutes'), "0001-02-03T04:05") + self.assertEqual(t.isoformat(timespec='seconds'), "0001-02-03T04:05:01") + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.000") + self.assertEqual(t.isoformat(timespec='microseconds'), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat(timespec='auto'), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat(sep=' ', timespec='minutes'), "0001-02-03 04:05") + self.assertRaises(ValueError, t.isoformat, timespec='foo') # str is ISO format with the separator forced to a blank. - self.assertEqual(str(t), "0002-03-02 04:05:01.000123") + self.assertEqual(str(t), "0001-02-03 04:05:01.000123") + + t = self.theclass(1, 2, 3, 4, 5, 1, 999500, tzinfo=timezone.utc) + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.999+00:00") + + t = self.theclass(1, 2, 3, 4, 5, 1, 999500) + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.999") + + t = self.theclass(1, 2, 3, 4, 5, 1) + self.assertEqual(t.isoformat(timespec='auto'), "0001-02-03T04:05:01") + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.000") + self.assertEqual(t.isoformat(timespec='microseconds'), "0001-02-03T04:05:01.000000") t = self.theclass(2, 3, 2) self.assertEqual(t.isoformat(), "0002-03-02T00:00:00") @@ -1572,6 +1601,10 @@ class TestDateTime(TestDate): self.assertEqual(t.isoformat(' '), "0002-03-02 00:00:00") # str is ISO format with the separator forced to a blank. self.assertEqual(str(t), "0002-03-02 00:00:00") + # ISO format with timezone + tz = FixedOffset(timedelta(seconds=16), 'XXX') + t = self.theclass(2, 3, 2, tzinfo=tz) + self.assertEqual(t.isoformat(), "0002-03-02T00:00:00+00:00:16") def test_format(self): dt = self.theclass(2007, 9, 10, 4, 5, 1, 123) @@ -1691,6 +1724,14 @@ class TestDateTime(TestDate): self.assertRaises(ValueError, self.theclass, 2000, 1, 31, 23, 59, 59, 1000000) + # bad fold + self.assertRaises(ValueError, self.theclass, + 2000, 1, 31, fold=-1) + self.assertRaises(ValueError, self.theclass, + 2000, 1, 31, fold=2) + # Positional fold: + self.assertRaises(TypeError, self.theclass, + 2000, 1, 31, 23, 59, 59, 0, None, 1) def test_hash_equality(self): d = self.theclass(2000, 12, 31, 23, 30, 17) @@ -1790,6 +1831,7 @@ class TestDateTime(TestDate): green = pickler.dumps(orig, proto) derived = unpickler.loads(green) self.assertEqual(orig, derived) + self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) def test_more_pickling(self): a = self.theclass(2003, 2, 7, 16, 48, 37, 444116) @@ -1874,16 +1916,20 @@ class TestDateTime(TestDate): t = self.theclass(1970, 1, 1, 1, 2, 3, 4) self.assertEqual(t.timestamp(), 18000.0 + 3600 + 2*60 + 3 + 4*1e-6) - # Missing hour may produce platform-dependent result - t = self.theclass(2012, 3, 11, 2, 30) - self.assertIn(self.theclass.fromtimestamp(t.timestamp()), - [t - timedelta(hours=1), t + timedelta(hours=1)]) + # Missing hour + t0 = self.theclass(2012, 3, 11, 2, 30) + t1 = t0.replace(fold=1) + self.assertEqual(self.theclass.fromtimestamp(t1.timestamp()), + t0 - timedelta(hours=1)) + self.assertEqual(self.theclass.fromtimestamp(t0.timestamp()), + t1 + timedelta(hours=1)) # Ambiguous hour defaults to DST t = self.theclass(2012, 11, 4, 1, 30) self.assertEqual(self.theclass.fromtimestamp(t.timestamp()), t) # Timestamp may raise an overflow error on some platforms - for t in [self.theclass(1,1,1), self.theclass(9999,12,12)]: + # XXX: Do we care to support the first and last year? + for t in [self.theclass(2,1,1), self.theclass(9998,12,12)]: try: s = t.timestamp() except OverflowError: @@ -1902,6 +1948,7 @@ class TestDateTime(TestDate): self.assertEqual(t.timestamp(), 18000 + 3600 + 2*60 + 3 + 4*1e-6) + @support.run_with_tz('MSK-03') # Something east of Greenwich def test_microsecond_rounding(self): for fts in [self.theclass.fromtimestamp, self.theclass.utcfromtimestamp]: @@ -2076,11 +2123,22 @@ class TestDateTime(TestDate): self.assertRaises(TypeError, combine) # need an arg self.assertRaises(TypeError, combine, d) # need two args self.assertRaises(TypeError, combine, t, d) # args reversed - self.assertRaises(TypeError, combine, d, t, 1) # too many args + self.assertRaises(TypeError, combine, d, t, 1) # wrong tzinfo type + self.assertRaises(TypeError, combine, d, t, 1, 2) # too many args self.assertRaises(TypeError, combine, "date", "time") # wrong types self.assertRaises(TypeError, combine, d, "time") # wrong type self.assertRaises(TypeError, combine, "date", t) # wrong type + # tzinfo= argument + dt = combine(d, t, timezone.utc) + self.assertIs(dt.tzinfo, timezone.utc) + dt = combine(d, t, tzinfo=timezone.utc) + self.assertIs(dt.tzinfo, timezone.utc) + t = time() + dt = combine(dt, t) + self.assertEqual(dt.date(), d) + self.assertEqual(dt.time(), t) + def test_replace(self): cls = self.theclass args = [1, 2, 3, 4, 5, 6, 7] @@ -2107,6 +2165,7 @@ class TestDateTime(TestDate): self.assertRaises(ValueError, base.replace, year=2001) def test_astimezone(self): + return # The rest is no longer applicable # Pretty boring! The TZ test is more interesting here. astimezone() # simply can't be applied to a naive object. dt = self.theclass.now() @@ -2324,6 +2383,23 @@ class TestTime(HarmlessMixedComparison, unittest.TestCase): self.assertEqual(t.isoformat(), "00:00:00.100000") self.assertEqual(t.isoformat(), str(t)) + t = self.theclass(hour=12, minute=34, second=56, microsecond=123456) + self.assertEqual(t.isoformat(timespec='hours'), "12") + self.assertEqual(t.isoformat(timespec='minutes'), "12:34") + self.assertEqual(t.isoformat(timespec='seconds'), "12:34:56") + self.assertEqual(t.isoformat(timespec='milliseconds'), "12:34:56.123") + self.assertEqual(t.isoformat(timespec='microseconds'), "12:34:56.123456") + self.assertEqual(t.isoformat(timespec='auto'), "12:34:56.123456") + self.assertRaises(ValueError, t.isoformat, timespec='monkey') + + t = self.theclass(hour=12, minute=34, second=56, microsecond=999500) + self.assertEqual(t.isoformat(timespec='milliseconds'), "12:34:56.999") + + t = self.theclass(hour=12, minute=34, second=56, microsecond=0) + self.assertEqual(t.isoformat(timespec='milliseconds'), "12:34:56.000") + self.assertEqual(t.isoformat(timespec='microseconds'), "12:34:56.000000") + self.assertEqual(t.isoformat(timespec='auto'), "12:34:56") + def test_1653736(self): # verify it doesn't accept extra keyword arguments t = self.theclass(second=1) @@ -2395,6 +2471,7 @@ class TestTime(HarmlessMixedComparison, unittest.TestCase): green = pickler.dumps(orig, proto) derived = unpickler.loads(green) self.assertEqual(orig, derived) + self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) def test_pickling_subclass_time(self): args = 20, 59, 16, 64**2 @@ -2582,9 +2659,9 @@ class TZInfoBase: self.assertRaises(ValueError, t.utcoffset) self.assertRaises(ValueError, t.dst) - # Not a whole number of minutes. + # Not a whole number of seconds. class C7(tzinfo): - def utcoffset(self, dt): return timedelta(seconds=61) + def utcoffset(self, dt): return timedelta(microseconds=61) def dst(self, dt): return timedelta(microseconds=-81) t = cls(1, 1, 1, tzinfo=C7()) self.assertRaises(ValueError, t.utcoffset) @@ -2755,6 +2832,7 @@ class TestTimeTZ(TestTime, TZInfoBase, unittest.TestCase): green = pickler.dumps(orig, proto) derived = unpickler.loads(green) self.assertEqual(orig, derived) + self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) # Try one with a tzinfo. tinfo = PicklableFixedOffset(-300, 'cookie') @@ -2766,6 +2844,7 @@ class TestTimeTZ(TestTime, TZInfoBase, unittest.TestCase): self.assertIsInstance(derived.tzinfo, PicklableFixedOffset) self.assertEqual(derived.utcoffset(), timedelta(minutes=-300)) self.assertEqual(derived.tzname(), 'cookie') + self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) def test_more_bool(self): # time is always True. @@ -2969,6 +3048,7 @@ class TestDateTimeTZ(TestDateTime, TZInfoBase, unittest.TestCase): green = pickler.dumps(orig, proto) derived = unpickler.loads(green) self.assertEqual(orig, derived) + self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) # Try one with a tzinfo. tinfo = PicklableFixedOffset(-300, 'cookie') @@ -2981,6 +3061,7 @@ class TestDateTimeTZ(TestDateTime, TZInfoBase, unittest.TestCase): self.assertIsInstance(derived.tzinfo, PicklableFixedOffset) self.assertEqual(derived.utcoffset(), timedelta(minutes=-300)) self.assertEqual(derived.tzname(), 'cookie') + self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) def test_extreme_hashes(self): # If an attempt is made to hash these via subtracting the offset @@ -3927,12 +4008,12 @@ class Oddballs(unittest.TestCase): datetime(xx, xx, xx, xx, xx, xx, xx)) with self.assertRaisesRegex(TypeError, '^an integer is required ' - '\(got type str\)$'): + r'\(got type str\)$'): datetime(10, 10, '10') f10 = Number(10.9) with self.assertRaisesRegex(TypeError, '^__int__ returned non-int ' - '\(type float\)$'): + r'\(type float\)$'): datetime(10, 10, f10) class Float(float): @@ -3957,5 +4038,790 @@ class Oddballs(unittest.TestCase): with self.assertRaises(TypeError): datetime(10, 10, 10, 10, 10, 10, 10.) +############################################################################# +# Local Time Disambiguation + +# An experimental reimplementation of fromutc that respects the "fold" flag. + +class tzinfo2(tzinfo): + + def fromutc(self, dt): + "datetime in UTC -> datetime in local time." + + if not isinstance(dt, datetime): + raise TypeError("fromutc() requires a datetime argument") + if dt.tzinfo is not self: + raise ValueError("dt.tzinfo is not self") + # Returned value satisfies + # dt + ldt.utcoffset() = ldt + off0 = dt.replace(fold=0).utcoffset() + off1 = dt.replace(fold=1).utcoffset() + if off0 is None or off1 is None or dt.dst() is None: + raise ValueError + if off0 == off1: + ldt = dt + off0 + off1 = ldt.utcoffset() + if off0 == off1: + return ldt + # Now, we discovered both possible offsets, so + # we can just try four possible solutions: + for off in [off0, off1]: + ldt = dt + off + if ldt.utcoffset() == off: + return ldt + ldt = ldt.replace(fold=1) + if ldt.utcoffset() == off: + return ldt + + raise ValueError("No suitable local time found") + +# Reimplementing simplified US timezones to respect the "fold" flag: + +class USTimeZone2(tzinfo2): + + def __init__(self, hours, reprname, stdname, dstname): + self.stdoffset = timedelta(hours=hours) + self.reprname = reprname + self.stdname = stdname + self.dstname = dstname + + def __repr__(self): + return self.reprname + + def tzname(self, dt): + if self.dst(dt): + return self.dstname + else: + return self.stdname + + def utcoffset(self, dt): + return self.stdoffset + self.dst(dt) + + def dst(self, dt): + if dt is None or dt.tzinfo is None: + # An exception instead may be sensible here, in one or more of + # the cases. + return ZERO + assert dt.tzinfo is self + + # Find first Sunday in April. + start = first_sunday_on_or_after(DSTSTART.replace(year=dt.year)) + assert start.weekday() == 6 and start.month == 4 and start.day <= 7 + + # Find last Sunday in October. + end = first_sunday_on_or_after(DSTEND.replace(year=dt.year)) + assert end.weekday() == 6 and end.month == 10 and end.day >= 25 + + # Can't compare naive to aware objects, so strip the timezone from + # dt first. + dt = dt.replace(tzinfo=None) + if start + HOUR <= dt < end: + # DST is in effect. + return HOUR + elif end <= dt < end + HOUR: + # Fold (an ambiguous hour): use dt.fold to disambiguate. + return ZERO if dt.fold else HOUR + elif start <= dt < start + HOUR: + # Gap (a non-existent hour): reverse the fold rule. + return HOUR if dt.fold else ZERO + else: + # DST is off. + return ZERO + +Eastern2 = USTimeZone2(-5, "Eastern2", "EST", "EDT") +Central2 = USTimeZone2(-6, "Central2", "CST", "CDT") +Mountain2 = USTimeZone2(-7, "Mountain2", "MST", "MDT") +Pacific2 = USTimeZone2(-8, "Pacific2", "PST", "PDT") + +# Europe_Vilnius_1941 tzinfo implementation reproduces the following +# 1941 transition from Olson's tzdist: +# +# Zone NAME GMTOFF RULES FORMAT [UNTIL] +# ZoneEurope/Vilnius 1:00 - CET 1940 Aug 3 +# 3:00 - MSK 1941 Jun 24 +# 1:00 C-Eur CE%sT 1944 Aug +# +# $ zdump -v Europe/Vilnius | grep 1941 +# Europe/Vilnius Mon Jun 23 20:59:59 1941 UTC = Mon Jun 23 23:59:59 1941 MSK isdst=0 gmtoff=10800 +# Europe/Vilnius Mon Jun 23 21:00:00 1941 UTC = Mon Jun 23 23:00:00 1941 CEST isdst=1 gmtoff=7200 + +class Europe_Vilnius_1941(tzinfo): + def _utc_fold(self): + return [datetime(1941, 6, 23, 21, tzinfo=self), # Mon Jun 23 21:00:00 1941 UTC + datetime(1941, 6, 23, 22, tzinfo=self)] # Mon Jun 23 22:00:00 1941 UTC + + def _loc_fold(self): + return [datetime(1941, 6, 23, 23, tzinfo=self), # Mon Jun 23 23:00:00 1941 MSK / CEST + datetime(1941, 6, 24, 0, tzinfo=self)] # Mon Jun 24 00:00:00 1941 CEST + + def utcoffset(self, dt): + fold_start, fold_stop = self._loc_fold() + if dt < fold_start: + return 3 * HOUR + if dt < fold_stop: + return (2 if dt.fold else 3) * HOUR + # if dt >= fold_stop + return 2 * HOUR + + def dst(self, dt): + fold_start, fold_stop = self._loc_fold() + if dt < fold_start: + return 0 * HOUR + if dt < fold_stop: + return (1 if dt.fold else 0) * HOUR + # if dt >= fold_stop + return 1 * HOUR + + def tzname(self, dt): + fold_start, fold_stop = self._loc_fold() + if dt < fold_start: + return 'MSK' + if dt < fold_stop: + return ('MSK', 'CEST')[dt.fold] + # if dt >= fold_stop + return 'CEST' + + def fromutc(self, dt): + assert dt.fold == 0 + assert dt.tzinfo is self + if dt.year != 1941: + raise NotImplementedError + fold_start, fold_stop = self._utc_fold() + if dt < fold_start: + return dt + 3 * HOUR + if dt < fold_stop: + return (dt + 2 * HOUR).replace(fold=1) + # if dt >= fold_stop + return dt + 2 * HOUR + + +class TestLocalTimeDisambiguation(unittest.TestCase): + + def test_vilnius_1941_fromutc(self): + Vilnius = Europe_Vilnius_1941() + + gdt = datetime(1941, 6, 23, 20, 59, 59, tzinfo=timezone.utc) + ldt = gdt.astimezone(Vilnius) + self.assertEqual(ldt.strftime("%c %Z%z"), + 'Mon Jun 23 23:59:59 1941 MSK+0300') + self.assertEqual(ldt.fold, 0) + self.assertFalse(ldt.dst()) + + gdt = datetime(1941, 6, 23, 21, tzinfo=timezone.utc) + ldt = gdt.astimezone(Vilnius) + self.assertEqual(ldt.strftime("%c %Z%z"), + 'Mon Jun 23 23:00:00 1941 CEST+0200') + self.assertEqual(ldt.fold, 1) + self.assertTrue(ldt.dst()) + + gdt = datetime(1941, 6, 23, 22, tzinfo=timezone.utc) + ldt = gdt.astimezone(Vilnius) + self.assertEqual(ldt.strftime("%c %Z%z"), + 'Tue Jun 24 00:00:00 1941 CEST+0200') + self.assertEqual(ldt.fold, 0) + self.assertTrue(ldt.dst()) + + def test_vilnius_1941_toutc(self): + Vilnius = Europe_Vilnius_1941() + + ldt = datetime(1941, 6, 23, 22, 59, 59, tzinfo=Vilnius) + gdt = ldt.astimezone(timezone.utc) + self.assertEqual(gdt.strftime("%c %Z"), + 'Mon Jun 23 19:59:59 1941 UTC') + + ldt = datetime(1941, 6, 23, 23, 59, 59, tzinfo=Vilnius) + gdt = ldt.astimezone(timezone.utc) + self.assertEqual(gdt.strftime("%c %Z"), + 'Mon Jun 23 20:59:59 1941 UTC') + + ldt = datetime(1941, 6, 23, 23, 59, 59, tzinfo=Vilnius, fold=1) + gdt = ldt.astimezone(timezone.utc) + self.assertEqual(gdt.strftime("%c %Z"), + 'Mon Jun 23 21:59:59 1941 UTC') + + ldt = datetime(1941, 6, 24, 0, tzinfo=Vilnius) + gdt = ldt.astimezone(timezone.utc) + self.assertEqual(gdt.strftime("%c %Z"), + 'Mon Jun 23 22:00:00 1941 UTC') + + + def test_constructors(self): + t = time(0, fold=1) + dt = datetime(1, 1, 1, fold=1) + self.assertEqual(t.fold, 1) + self.assertEqual(dt.fold, 1) + with self.assertRaises(TypeError): + time(0, 0, 0, 0, None, 0) + + def test_member(self): + dt = datetime(1, 1, 1, fold=1) + t = dt.time() + self.assertEqual(t.fold, 1) + t = dt.timetz() + self.assertEqual(t.fold, 1) + + def test_replace(self): + t = time(0) + dt = datetime(1, 1, 1) + self.assertEqual(t.replace(fold=1).fold, 1) + self.assertEqual(dt.replace(fold=1).fold, 1) + self.assertEqual(t.replace(fold=0).fold, 0) + self.assertEqual(dt.replace(fold=0).fold, 0) + # Check that replacement of other fields does not change "fold". + t = t.replace(fold=1, tzinfo=Eastern) + dt = dt.replace(fold=1, tzinfo=Eastern) + self.assertEqual(t.replace(tzinfo=None).fold, 1) + self.assertEqual(dt.replace(tzinfo=None).fold, 1) + # Check that fold is a keyword-only argument + with self.assertRaises(TypeError): + t.replace(1, 1, 1, None, 1) + with self.assertRaises(TypeError): + dt.replace(1, 1, 1, 1, 1, 1, 1, None, 1) + + def test_comparison(self): + t = time(0) + dt = datetime(1, 1, 1) + self.assertEqual(t, t.replace(fold=1)) + self.assertEqual(dt, dt.replace(fold=1)) + + def test_hash(self): + t = time(0) + dt = datetime(1, 1, 1) + self.assertEqual(hash(t), hash(t.replace(fold=1))) + self.assertEqual(hash(dt), hash(dt.replace(fold=1))) + + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_fromtimestamp(self): + s = 1414906200 + dt0 = datetime.fromtimestamp(s) + dt1 = datetime.fromtimestamp(s + 3600) + self.assertEqual(dt0.fold, 0) + self.assertEqual(dt1.fold, 1) + + @support.run_with_tz('Australia/Lord_Howe') + def test_fromtimestamp_lord_howe(self): + tm = _time.localtime(1.4e9) + if _time.strftime('%Z%z', tm) != 'LHST+1030': + self.skipTest('Australia/Lord_Howe timezone is not supported on this platform') + # $ TZ=Australia/Lord_Howe date -r 1428158700 + # Sun Apr 5 01:45:00 LHDT 2015 + # $ TZ=Australia/Lord_Howe date -r 1428160500 + # Sun Apr 5 01:45:00 LHST 2015 + s = 1428158700 + t0 = datetime.fromtimestamp(s) + t1 = datetime.fromtimestamp(s + 1800) + self.assertEqual(t0, t1) + self.assertEqual(t0.fold, 0) + self.assertEqual(t1.fold, 1) + + + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_timestamp(self): + dt0 = datetime(2014, 11, 2, 1, 30) + dt1 = dt0.replace(fold=1) + self.assertEqual(dt0.timestamp() + 3600, + dt1.timestamp()) + + @support.run_with_tz('Australia/Lord_Howe') + def test_timestamp_lord_howe(self): + tm = _time.localtime(1.4e9) + if _time.strftime('%Z%z', tm) != 'LHST+1030': + self.skipTest('Australia/Lord_Howe timezone is not supported on this platform') + t = datetime(2015, 4, 5, 1, 45) + s0 = t.replace(fold=0).timestamp() + s1 = t.replace(fold=1).timestamp() + self.assertEqual(s0 + 1800, s1) + + + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_astimezone(self): + dt0 = datetime(2014, 11, 2, 1, 30) + dt1 = dt0.replace(fold=1) + # Convert both naive instances to aware. + adt0 = dt0.astimezone() + adt1 = dt1.astimezone() + # Check that the first instance in DST zone and the second in STD + self.assertEqual(adt0.tzname(), 'EDT') + self.assertEqual(adt1.tzname(), 'EST') + self.assertEqual(adt0 + HOUR, adt1) + # Aware instances with fixed offset tzinfo's always have fold=0 + self.assertEqual(adt0.fold, 0) + self.assertEqual(adt1.fold, 0) + + + def test_pickle_fold(self): + t = time(fold=1) + dt = datetime(1, 1, 1, fold=1) + for pickler, unpickler, proto in pickle_choices: + for x in [t, dt]: + s = pickler.dumps(x, proto) + y = unpickler.loads(s) + self.assertEqual(x, y) + self.assertEqual((0 if proto < 4 else x.fold), y.fold) + + def test_repr(self): + t = time(fold=1) + dt = datetime(1, 1, 1, fold=1) + self.assertEqual(repr(t), 'datetime.time(0, 0, fold=1)') + self.assertEqual(repr(dt), + 'datetime.datetime(1, 1, 1, 0, 0, fold=1)') + + def test_dst(self): + # Let's first establish that things work in regular times. + dt_summer = datetime(2002, 10, 27, 1, tzinfo=Eastern2) - timedelta.resolution + dt_winter = datetime(2002, 10, 27, 2, tzinfo=Eastern2) + self.assertEqual(dt_summer.dst(), HOUR) + self.assertEqual(dt_winter.dst(), ZERO) + # The disambiguation flag is ignored + self.assertEqual(dt_summer.replace(fold=1).dst(), HOUR) + self.assertEqual(dt_winter.replace(fold=1).dst(), ZERO) + + # Pick local time in the fold. + for minute in [0, 30, 59]: + dt = datetime(2002, 10, 27, 1, minute, tzinfo=Eastern2) + # With fold=0 (the default) it is in DST. + self.assertEqual(dt.dst(), HOUR) + # With fold=1 it is in STD. + self.assertEqual(dt.replace(fold=1).dst(), ZERO) + + # Pick local time in the gap. + for minute in [0, 30, 59]: + dt = datetime(2002, 4, 7, 2, minute, tzinfo=Eastern2) + # With fold=0 (the default) it is in STD. + self.assertEqual(dt.dst(), ZERO) + # With fold=1 it is in DST. + self.assertEqual(dt.replace(fold=1).dst(), HOUR) + + + def test_utcoffset(self): + # Let's first establish that things work in regular times. + dt_summer = datetime(2002, 10, 27, 1, tzinfo=Eastern2) - timedelta.resolution + dt_winter = datetime(2002, 10, 27, 2, tzinfo=Eastern2) + self.assertEqual(dt_summer.utcoffset(), -4 * HOUR) + self.assertEqual(dt_winter.utcoffset(), -5 * HOUR) + # The disambiguation flag is ignored + self.assertEqual(dt_summer.replace(fold=1).utcoffset(), -4 * HOUR) + self.assertEqual(dt_winter.replace(fold=1).utcoffset(), -5 * HOUR) + + def test_fromutc(self): + # Let's first establish that things work in regular times. + u_summer = datetime(2002, 10, 27, 6, tzinfo=Eastern2) - timedelta.resolution + u_winter = datetime(2002, 10, 27, 7, tzinfo=Eastern2) + t_summer = Eastern2.fromutc(u_summer) + t_winter = Eastern2.fromutc(u_winter) + self.assertEqual(t_summer, u_summer - 4 * HOUR) + self.assertEqual(t_winter, u_winter - 5 * HOUR) + self.assertEqual(t_summer.fold, 0) + self.assertEqual(t_winter.fold, 0) + + # What happens in the fall-back fold? + u = datetime(2002, 10, 27, 5, 30, tzinfo=Eastern2) + t0 = Eastern2.fromutc(u) + u += HOUR + t1 = Eastern2.fromutc(u) + self.assertEqual(t0, t1) + self.assertEqual(t0.fold, 0) + self.assertEqual(t1.fold, 1) + # The tricky part is when u is in the local fold: + u = datetime(2002, 10, 27, 1, 30, tzinfo=Eastern2) + t = Eastern2.fromutc(u) + self.assertEqual((t.day, t.hour), (26, 21)) + # .. or gets into the local fold after a standard time adjustment + u = datetime(2002, 10, 27, 6, 30, tzinfo=Eastern2) + t = Eastern2.fromutc(u) + self.assertEqual((t.day, t.hour), (27, 1)) + + # What happens in the spring-forward gap? + u = datetime(2002, 4, 7, 2, 0, tzinfo=Eastern2) + t = Eastern2.fromutc(u) + self.assertEqual((t.day, t.hour), (6, 21)) + + def test_mixed_compare_regular(self): + t = datetime(2000, 1, 1, tzinfo=Eastern2) + self.assertEqual(t, t.astimezone(timezone.utc)) + t = datetime(2000, 6, 1, tzinfo=Eastern2) + self.assertEqual(t, t.astimezone(timezone.utc)) + + def test_mixed_compare_fold(self): + t_fold = datetime(2002, 10, 27, 1, 45, tzinfo=Eastern2) + t_fold_utc = t_fold.astimezone(timezone.utc) + self.assertNotEqual(t_fold, t_fold_utc) + + def test_mixed_compare_gap(self): + t_gap = datetime(2002, 4, 7, 2, 45, tzinfo=Eastern2) + t_gap_utc = t_gap.astimezone(timezone.utc) + self.assertNotEqual(t_gap, t_gap_utc) + + def test_hash_aware(self): + t = datetime(2000, 1, 1, tzinfo=Eastern2) + self.assertEqual(hash(t), hash(t.replace(fold=1))) + t_fold = datetime(2002, 10, 27, 1, 45, tzinfo=Eastern2) + t_gap = datetime(2002, 4, 7, 2, 45, tzinfo=Eastern2) + self.assertEqual(hash(t_fold), hash(t_fold.replace(fold=1))) + self.assertEqual(hash(t_gap), hash(t_gap.replace(fold=1))) + +SEC = timedelta(0, 1) + +def pairs(iterable): + a, b = itertools.tee(iterable) + next(b, None) + return zip(a, b) + +class ZoneInfo(tzinfo): + zoneroot = '/usr/share/zoneinfo' + def __init__(self, ut, ti): + """ + + :param ut: array + Array of transition point timestamps + :param ti: list + A list of (offset, isdst, abbr) tuples + :return: None + """ + self.ut = ut + self.ti = ti + self.lt = self.invert(ut, ti) + + @staticmethod + def invert(ut, ti): + lt = (array('q', ut), array('q', ut)) + if ut: + offset = ti[0][0] // SEC + lt[0][0] += offset + lt[1][0] += offset + for i in range(1, len(ut)): + lt[0][i] += ti[i-1][0] // SEC + lt[1][i] += ti[i][0] // SEC + return lt + + @classmethod + def fromfile(cls, fileobj): + if fileobj.read(4).decode() != "TZif": + raise ValueError("not a zoneinfo file") + fileobj.seek(32) + counts = array('i') + counts.fromfile(fileobj, 3) + if sys.byteorder != 'big': + counts.byteswap() + + ut = array('i') + ut.fromfile(fileobj, counts[0]) + if sys.byteorder != 'big': + ut.byteswap() + + type_indices = array('B') + type_indices.fromfile(fileobj, counts[0]) + + ttis = [] + for i in range(counts[1]): + ttis.append(struct.unpack(">lbb", fileobj.read(6))) + + abbrs = fileobj.read(counts[2]) + + # Convert ttis + for i, (gmtoff, isdst, abbrind) in enumerate(ttis): + abbr = abbrs[abbrind:abbrs.find(0, abbrind)].decode() + ttis[i] = (timedelta(0, gmtoff), isdst, abbr) + + ti = [None] * len(ut) + for i, idx in enumerate(type_indices): + ti[i] = ttis[idx] + + self = cls(ut, ti) + + return self + + @classmethod + def fromname(cls, name): + path = os.path.join(cls.zoneroot, name) + with open(path, 'rb') as f: + return cls.fromfile(f) + + EPOCHORDINAL = date(1970, 1, 1).toordinal() + + def fromutc(self, dt): + """datetime in UTC -> datetime in local time.""" + + if not isinstance(dt, datetime): + raise TypeError("fromutc() requires a datetime argument") + if dt.tzinfo is not self: + raise ValueError("dt.tzinfo is not self") + + timestamp = ((dt.toordinal() - self.EPOCHORDINAL) * 86400 + + dt.hour * 3600 + + dt.minute * 60 + + dt.second) + + if timestamp < self.ut[1]: + tti = self.ti[0] + fold = 0 + else: + idx = bisect.bisect_right(self.ut, timestamp) + assert self.ut[idx-1] <= timestamp + assert idx == len(self.ut) or timestamp < self.ut[idx] + tti_prev, tti = self.ti[idx-2:idx] + # Detect fold + shift = tti_prev[0] - tti[0] + fold = (shift > timedelta(0, timestamp - self.ut[idx-1])) + dt += tti[0] + if fold: + return dt.replace(fold=1) + else: + return dt + + def _find_ti(self, dt, i): + timestamp = ((dt.toordinal() - self.EPOCHORDINAL) * 86400 + + dt.hour * 3600 + + dt.minute * 60 + + dt.second) + lt = self.lt[dt.fold] + idx = bisect.bisect_right(lt, timestamp) + + return self.ti[max(0, idx - 1)][i] + + def utcoffset(self, dt): + return self._find_ti(dt, 0) + + def dst(self, dt): + isdst = self._find_ti(dt, 1) + # XXX: We cannot accurately determine the "save" value, + # so let's return 1h whenever DST is in effect. Since + # we don't use dst() in fromutc(), it is unlikely that + # it will be needed for anything more than bool(dst()). + return ZERO if isdst else HOUR + + def tzname(self, dt): + return self._find_ti(dt, 2) + + @classmethod + def zonenames(cls, zonedir=None): + if zonedir is None: + zonedir = cls.zoneroot + zone_tab = os.path.join(zonedir, 'zone.tab') + try: + f = open(zone_tab) + except OSError: + return + with f: + for line in f: + line = line.strip() + if line and not line.startswith('#'): + yield line.split()[2] + + @classmethod + def stats(cls, start_year=1): + count = gap_count = fold_count = zeros_count = 0 + min_gap = min_fold = timedelta.max + max_gap = max_fold = ZERO + min_gap_datetime = max_gap_datetime = datetime.min + min_gap_zone = max_gap_zone = None + min_fold_datetime = max_fold_datetime = datetime.min + min_fold_zone = max_fold_zone = None + stats_since = datetime(start_year, 1, 1) # Starting from 1970 eliminates a lot of noise + for zonename in cls.zonenames(): + count += 1 + tz = cls.fromname(zonename) + for dt, shift in tz.transitions(): + if dt < stats_since: + continue + if shift > ZERO: + gap_count += 1 + if (shift, dt) > (max_gap, max_gap_datetime): + max_gap = shift + max_gap_zone = zonename + max_gap_datetime = dt + if (shift, datetime.max - dt) < (min_gap, datetime.max - min_gap_datetime): + min_gap = shift + min_gap_zone = zonename + min_gap_datetime = dt + elif shift < ZERO: + fold_count += 1 + shift = -shift + if (shift, dt) > (max_fold, max_fold_datetime): + max_fold = shift + max_fold_zone = zonename + max_fold_datetime = dt + if (shift, datetime.max - dt) < (min_fold, datetime.max - min_fold_datetime): + min_fold = shift + min_fold_zone = zonename + min_fold_datetime = dt + else: + zeros_count += 1 + trans_counts = (gap_count, fold_count, zeros_count) + print("Number of zones: %5d" % count) + print("Number of transitions: %5d = %d (gaps) + %d (folds) + %d (zeros)" % + ((sum(trans_counts),) + trans_counts)) + print("Min gap: %16s at %s in %s" % (min_gap, min_gap_datetime, min_gap_zone)) + print("Max gap: %16s at %s in %s" % (max_gap, max_gap_datetime, max_gap_zone)) + print("Min fold: %16s at %s in %s" % (min_fold, min_fold_datetime, min_fold_zone)) + print("Max fold: %16s at %s in %s" % (max_fold, max_fold_datetime, max_fold_zone)) + + + def transitions(self): + for (_, prev_ti), (t, ti) in pairs(zip(self.ut, self.ti)): + shift = ti[0] - prev_ti[0] + yield datetime.utcfromtimestamp(t), shift + + def nondst_folds(self): + """Find all folds with the same value of isdst on both sides of the transition.""" + for (_, prev_ti), (t, ti) in pairs(zip(self.ut, self.ti)): + shift = ti[0] - prev_ti[0] + if shift < ZERO and ti[1] == prev_ti[1]: + yield datetime.utcfromtimestamp(t), -shift, prev_ti[2], ti[2] + + @classmethod + def print_all_nondst_folds(cls, same_abbr=False, start_year=1): + count = 0 + for zonename in cls.zonenames(): + tz = cls.fromname(zonename) + for dt, shift, prev_abbr, abbr in tz.nondst_folds(): + if dt.year < start_year or same_abbr and prev_abbr != abbr: + continue + count += 1 + print("%3d) %-30s %s %10s %5s -> %s" % + (count, zonename, dt, shift, prev_abbr, abbr)) + + def folds(self): + for t, shift in self.transitions(): + if shift < ZERO: + yield t, -shift + + def gaps(self): + for t, shift in self.transitions(): + if shift > ZERO: + yield t, shift + + def zeros(self): + for t, shift in self.transitions(): + if not shift: + yield t + + +class ZoneInfoTest(unittest.TestCase): + zonename = 'America/New_York' + + def setUp(self): + if sys.platform == "win32": + self.skipTest("Skipping zoneinfo tests on Windows") + try: + self.tz = ZoneInfo.fromname(self.zonename) + except FileNotFoundError as err: + self.skipTest("Skipping %s: %s" % (self.zonename, err)) + + def assertEquivDatetimes(self, a, b): + self.assertEqual((a.replace(tzinfo=None), a.fold, id(a.tzinfo)), + (b.replace(tzinfo=None), b.fold, id(b.tzinfo))) + + def test_folds(self): + tz = self.tz + for dt, shift in tz.folds(): + for x in [0 * shift, 0.5 * shift, shift - timedelta.resolution]: + udt = dt + x + ldt = tz.fromutc(udt.replace(tzinfo=tz)) + self.assertEqual(ldt.fold, 1) + adt = udt.replace(tzinfo=timezone.utc).astimezone(tz) + self.assertEquivDatetimes(adt, ldt) + utcoffset = ldt.utcoffset() + self.assertEqual(ldt.replace(tzinfo=None), udt + utcoffset) + # Round trip + self.assertEquivDatetimes(ldt.astimezone(timezone.utc), + udt.replace(tzinfo=timezone.utc)) + + + for x in [-timedelta.resolution, shift]: + udt = dt + x + udt = udt.replace(tzinfo=tz) + ldt = tz.fromutc(udt) + self.assertEqual(ldt.fold, 0) + + def test_gaps(self): + tz = self.tz + for dt, shift in tz.gaps(): + for x in [0 * shift, 0.5 * shift, shift - timedelta.resolution]: + udt = dt + x + udt = udt.replace(tzinfo=tz) + ldt = tz.fromutc(udt) + self.assertEqual(ldt.fold, 0) + adt = udt.replace(tzinfo=timezone.utc).astimezone(tz) + self.assertEquivDatetimes(adt, ldt) + utcoffset = ldt.utcoffset() + self.assertEqual(ldt.replace(tzinfo=None), udt.replace(tzinfo=None) + utcoffset) + # Create a local time inside the gap + ldt = tz.fromutc(dt.replace(tzinfo=tz)) - shift + x + self.assertLess(ldt.replace(fold=1).utcoffset(), + ldt.replace(fold=0).utcoffset(), + "At %s." % ldt) + + for x in [-timedelta.resolution, shift]: + udt = dt + x + ldt = tz.fromutc(udt.replace(tzinfo=tz)) + self.assertEqual(ldt.fold, 0) + + def test_system_transitions(self): + if ('Riyadh8' in self.zonename or + # From tzdata NEWS file: + # The files solar87, solar88, and solar89 are no longer distributed. + # They were a negative experiment - that is, a demonstration that + # tz data can represent solar time only with some difficulty and error. + # Their presence in the distribution caused confusion, as Riyadh + # civil time was generally not solar time in those years. + self.zonename.startswith('right/')): + self.skipTest("Skipping %s" % self.zonename) + tz = self.tz + TZ = os.environ.get('TZ') + os.environ['TZ'] = self.zonename + try: + _time.tzset() + for udt, shift in tz.transitions(): + if udt.year >= 2037: + # System support for times around the end of 32-bit time_t + # and later is flaky on many systems. + break + s0 = (udt - datetime(1970, 1, 1)) // SEC + ss = shift // SEC # shift seconds + for x in [-40 * 3600, -20*3600, -1, 0, + ss - 1, ss + 20 * 3600, ss + 40 * 3600]: + s = s0 + x + sdt = datetime.fromtimestamp(s) + tzdt = datetime.fromtimestamp(s, tz).replace(tzinfo=None) + self.assertEquivDatetimes(sdt, tzdt) + s1 = sdt.timestamp() + self.assertEqual(s, s1) + if ss > 0: # gap + # Create local time inside the gap + dt = datetime.fromtimestamp(s0) - shift / 2 + ts0 = dt.timestamp() + ts1 = dt.replace(fold=1).timestamp() + self.assertEqual(ts0, s0 + ss / 2) + self.assertEqual(ts1, s0 - ss / 2) + finally: + if TZ is None: + del os.environ['TZ'] + else: + os.environ['TZ'] = TZ + _time.tzset() + + +class ZoneInfoCompleteTest(unittest.TestSuite): + def __init__(self): + tests = [] + if is_resource_enabled('tzdata'): + for name in ZoneInfo.zonenames(): + Test = type('ZoneInfoTest[%s]' % name, (ZoneInfoTest,), {}) + Test.zonename = name + for method in dir(Test): + if method.startswith('test_'): + tests.append(Test(method)) + super().__init__(tests) + +# Iran had a sub-minute UTC offset before 1946. +class IranTest(ZoneInfoTest): + zonename = 'Asia/Tehran' + +def load_tests(loader, standard_tests, pattern): + standard_tests.addTest(ZoneInfoCompleteTest()) + return standard_tests + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/dtracedata/assert_usable.d b/Lib/test/dtracedata/assert_usable.d new file mode 100644 index 0000000000..0b2d4da66e --- /dev/null +++ b/Lib/test/dtracedata/assert_usable.d @@ -0,0 +1,5 @@ +BEGIN +{ + printf("probe: success\n"); + exit(0); +} diff --git a/Lib/test/dtracedata/assert_usable.stp b/Lib/test/dtracedata/assert_usable.stp new file mode 100644 index 0000000000..88e7e68e2c --- /dev/null +++ b/Lib/test/dtracedata/assert_usable.stp @@ -0,0 +1,5 @@ +probe begin +{ + println("probe: success") + exit () +} diff --git a/Lib/test/dtracedata/call_stack.d b/Lib/test/dtracedata/call_stack.d new file mode 100644 index 0000000000..450e939d4f --- /dev/null +++ b/Lib/test/dtracedata/call_stack.d @@ -0,0 +1,31 @@ +self int indent; + +python$target:::function-entry +/copyinstr(arg1) == "start"/ +{ + self->trace = 1; +} + +python$target:::function-entry +/self->trace/ +{ + printf("%d\t%*s:", timestamp, 15, probename); + printf("%*s", self->indent, ""); + printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2); + self->indent++; +} + +python$target:::function-return +/self->trace/ +{ + self->indent--; + printf("%d\t%*s:", timestamp, 15, probename); + printf("%*s", self->indent, ""); + printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2); +} + +python$target:::function-return +/copyinstr(arg1) == "start"/ +{ + self->trace = 0; +} diff --git a/Lib/test/dtracedata/call_stack.d.expected b/Lib/test/dtracedata/call_stack.d.expected new file mode 100644 index 0000000000..27849d1549 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.d.expected @@ -0,0 +1,18 @@ + function-entry:call_stack.py:start:23 + function-entry: call_stack.py:function_1:1 + function-entry: call_stack.py:function_3:9 +function-return: call_stack.py:function_3:10 +function-return: call_stack.py:function_1:2 + function-entry: call_stack.py:function_2:5 + function-entry: call_stack.py:function_1:1 + function-entry: call_stack.py:function_3:9 +function-return: call_stack.py:function_3:10 +function-return: call_stack.py:function_1:2 +function-return: call_stack.py:function_2:6 + function-entry: call_stack.py:function_3:9 +function-return: call_stack.py:function_3:10 + function-entry: call_stack.py:function_4:13 +function-return: call_stack.py:function_4:14 + function-entry: call_stack.py:function_5:18 +function-return: call_stack.py:function_5:21 +function-return:call_stack.py:start:28 diff --git a/Lib/test/dtracedata/call_stack.py b/Lib/test/dtracedata/call_stack.py new file mode 100644 index 0000000000..ee9f3ae8d6 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.py @@ -0,0 +1,30 @@ +def function_1(): + function_3(1, 2) + +# Check stacktrace +def function_2(): + function_1() + +# CALL_FUNCTION_VAR +def function_3(dummy, dummy2): + pass + +# CALL_FUNCTION_KW +def function_4(**dummy): + return 1 + return 2 # unreachable + +# CALL_FUNCTION_VAR_KW +def function_5(dummy, dummy2, **dummy3): + if False: + return 7 + return 8 + +def start(): + function_1() + function_2() + function_3(1, 2) + function_4(test=42) + function_5(*(1, 2), **{"test": 42}) + +start() diff --git a/Lib/test/dtracedata/call_stack.stp b/Lib/test/dtracedata/call_stack.stp new file mode 100644 index 0000000000..54082c202f --- /dev/null +++ b/Lib/test/dtracedata/call_stack.stp @@ -0,0 +1,41 @@ +global tracing + +function basename:string(path:string) +{ + last_token = token = tokenize(path, "/"); + while (token != "") { + last_token = token; + token = tokenize("", "/"); + } + return last_token; +} + +probe process.mark("function__entry") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 1; + } +} + +probe process.mark("function__entry"), process.mark("function__return") +{ + filename = user_string($arg1); + funcname = user_string($arg2); + lineno = $arg3; + + if (tracing) { + printf("%d\t%s:%s:%s:%d\n", gettimeofday_us(), $$name, + basename(filename), funcname, lineno); + } +} + +probe process.mark("function__return") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 0; + } +} diff --git a/Lib/test/dtracedata/call_stack.stp.expected b/Lib/test/dtracedata/call_stack.stp.expected new file mode 100644 index 0000000000..32cf396f82 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.stp.expected @@ -0,0 +1,14 @@ +function__entry:call_stack.py:start:23 +function__entry:call_stack.py:function_1:1 +function__return:call_stack.py:function_1:2 +function__entry:call_stack.py:function_2:5 +function__entry:call_stack.py:function_1:1 +function__return:call_stack.py:function_1:2 +function__return:call_stack.py:function_2:6 +function__entry:call_stack.py:function_3:9 +function__return:call_stack.py:function_3:10 +function__entry:call_stack.py:function_4:13 +function__return:call_stack.py:function_4:14 +function__entry:call_stack.py:function_5:18 +function__return:call_stack.py:function_5:21 +function__return:call_stack.py:start:28 diff --git a/Lib/test/dtracedata/gc.d b/Lib/test/dtracedata/gc.d new file mode 100644 index 0000000000..4d91487b7e --- /dev/null +++ b/Lib/test/dtracedata/gc.d @@ -0,0 +1,18 @@ +python$target:::function-entry +/copyinstr(arg1) == "start"/ +{ + self->trace = 1; +} + +python$target:::gc-start, +python$target:::gc-done +/self->trace/ +{ + printf("%d\t%s:%ld\n", timestamp, probename, arg0); +} + +python$target:::function-return +/copyinstr(arg1) == "start"/ +{ + self->trace = 0; +} diff --git a/Lib/test/dtracedata/gc.d.expected b/Lib/test/dtracedata/gc.d.expected new file mode 100644 index 0000000000..8e5ac2a6d5 --- /dev/null +++ b/Lib/test/dtracedata/gc.d.expected @@ -0,0 +1,8 @@ +gc-start:0 +gc-done:0 +gc-start:1 +gc-done:0 +gc-start:2 +gc-done:0 +gc-start:2 +gc-done:1 diff --git a/Lib/test/dtracedata/gc.py b/Lib/test/dtracedata/gc.py new file mode 100644 index 0000000000..144a783b7b --- /dev/null +++ b/Lib/test/dtracedata/gc.py @@ -0,0 +1,13 @@ +import gc + +def start(): + gc.collect(0) + gc.collect(1) + gc.collect(2) + l = [] + l.append(l) + del l + gc.collect(2) + +gc.collect() +start() diff --git a/Lib/test/dtracedata/gc.stp b/Lib/test/dtracedata/gc.stp new file mode 100644 index 0000000000..162c6d3a22 --- /dev/null +++ b/Lib/test/dtracedata/gc.stp @@ -0,0 +1,26 @@ +global tracing + +probe process.mark("function__entry") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 1; + } +} + +probe process.mark("gc__start"), process.mark("gc__done") +{ + if (tracing) { + printf("%d\t%s:%ld\n", gettimeofday_us(), $$name, $arg1); + } +} + +probe process.mark("function__return") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 0; + } +} diff --git a/Lib/test/dtracedata/gc.stp.expected b/Lib/test/dtracedata/gc.stp.expected new file mode 100644 index 0000000000..7e6e6227fb --- /dev/null +++ b/Lib/test/dtracedata/gc.stp.expected @@ -0,0 +1,8 @@ +gc__start:0 +gc__done:0 +gc__start:1 +gc__done:0 +gc__start:2 +gc__done:0 +gc__start:2 +gc__done:1 diff --git a/Lib/test/dtracedata/instance.py b/Lib/test/dtracedata/instance.py new file mode 100644 index 0000000000..f1421378b0 --- /dev/null +++ b/Lib/test/dtracedata/instance.py @@ -0,0 +1,24 @@ +import gc + +class old_style_class(): + pass +class new_style_class(object): + pass + +a = old_style_class() +del a +gc.collect() +b = new_style_class() +del b +gc.collect() + +a = old_style_class() +del old_style_class +gc.collect() +b = new_style_class() +del new_style_class +gc.collect() +del a +gc.collect() +del b +gc.collect() diff --git a/Lib/test/dtracedata/line.d b/Lib/test/dtracedata/line.d new file mode 100644 index 0000000000..03f22db6fc --- /dev/null +++ b/Lib/test/dtracedata/line.d @@ -0,0 +1,7 @@ +python$target:::line +/(copyinstr(arg1)=="test_line")/ +{ + printf("%d\t%s:%s:%s:%d\n", timestamp, + probename, basename(copyinstr(arg0)), + copyinstr(arg1), arg2); +} diff --git a/Lib/test/dtracedata/line.d.expected b/Lib/test/dtracedata/line.d.expected new file mode 100644 index 0000000000..9b16ce76ee --- /dev/null +++ b/Lib/test/dtracedata/line.d.expected @@ -0,0 +1,20 @@ +line:line.py:test_line:2 +line:line.py:test_line:3 +line:line.py:test_line:4 +line:line.py:test_line:5 +line:line.py:test_line:6 +line:line.py:test_line:7 +line:line.py:test_line:8 +line:line.py:test_line:9 +line:line.py:test_line:10 +line:line.py:test_line:11 +line:line.py:test_line:4 +line:line.py:test_line:5 +line:line.py:test_line:6 +line:line.py:test_line:7 +line:line.py:test_line:8 +line:line.py:test_line:10 +line:line.py:test_line:11 +line:line.py:test_line:4 +line:line.py:test_line:12 +line:line.py:test_line:13 diff --git a/Lib/test/dtracedata/line.py b/Lib/test/dtracedata/line.py new file mode 100644 index 0000000000..0930ff391f --- /dev/null +++ b/Lib/test/dtracedata/line.py @@ -0,0 +1,17 @@ +def test_line(): + a = 1 + print('# Preamble', a) + for i in range(2): + a = i + b = i+2 + c = i+3 + if c < 4: + a = c + d = a + b +c + print('#', a, b, c, d) + a = 1 + print('# Epilogue', a) + + +if __name__ == '__main__': + test_line() diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index e3f1aa519e..d194e775ea 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -9,7 +9,7 @@ sub-second periodicity (contrarily to signal()). """ import contextlib -import io +import faulthandler import os import select import signal @@ -20,6 +20,7 @@ import time import unittest from test import support +android_not_root = support.android_not_root @contextlib.contextmanager def kill_on_error(proc): @@ -50,6 +51,10 @@ class EINTRBaseTest(unittest.TestCase): signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) + # Issue #25277: Use faulthandler to try to debug a hang on FreeBSD + if hasattr(faulthandler, 'dump_traceback_later'): + faulthandler.dump_traceback_later(10 * 60, exit=True) + @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) @@ -58,6 +63,8 @@ class EINTRBaseTest(unittest.TestCase): def tearDownClass(cls): cls.stop_alarm() signal.signal(signal.SIGALRM, cls.orig_handler) + if hasattr(faulthandler, 'cancel_dump_traceback_later'): + faulthandler.cancel_dump_traceback_later() def subprocess(self, *args, **kw): cmd_args = (sys.executable, '-c') + args @@ -77,6 +84,9 @@ class OSEINTRTest(EINTRBaseTest): processes = [self.new_sleep_process() for _ in range(num)] for _ in range(num): wait_func() + # Call the Popen method to avoid a ResourceWarning + for proc in processes: + proc.wait() def test_wait(self): self._test_wait_multiple(os.wait) @@ -88,6 +98,8 @@ class OSEINTRTest(EINTRBaseTest): def _test_wait_single(self, wait_func): proc = self.new_sleep_process() wait_func(proc.pid) + # Call the Popen method to avoid a ResourceWarning + proc.wait() def test_waitpid(self): self._test_wait_single(lambda pid: os.waitpid(pid, 0)) @@ -300,6 +312,7 @@ class SocketEINTRTest(EINTRBaseTest): # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=203162 @support.requires_freebsd_version(10, 3) @unittest.skipUnless(hasattr(os, 'mkfifo'), 'needs mkfifo()') + @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") def _test_open(self, do_open_close_reader, do_open_close_writer): filename = support.TESTFN diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index 05137654de..7333b2aa62 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -14,6 +14,7 @@ BaseException +-- BufferError +-- EOFError +-- ImportError + +-- ModuleNotFoundError +-- LookupError | +-- IndexError | +-- KeyError diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py new file mode 100644 index 0000000000..7ba0e6e535 --- /dev/null +++ b/Lib/test/libregrtest/__init__.py @@ -0,0 +1,5 @@ +# We import importlib *ASAP* in order to test #15386 +import importlib + +from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES +from test.libregrtest.main import main diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py new file mode 100644 index 0000000000..d621f5f9f3 --- /dev/null +++ b/Lib/test/libregrtest/cmdline.py @@ -0,0 +1,347 @@ +import argparse +import os +import sys +from test import support + + +USAGE = """\ +python -m test [options] [test_name1 [test_name2 ...]] +python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] +""" + +DESCRIPTION = """\ +Run Python regression tests. + +If no arguments or options are provided, finds all files matching +the pattern "test_*" in the Lib/test subdirectory and runs +them in alphabetical order (but see -M and -u, below, for exceptions). + +For more rigorous testing, it is useful to use the following +command line: + +python -E -Wd -m test [options] [test_name1 ...] +""" + +EPILOG = """\ +Additional option details: + +-r randomizes test execution order. You can use --randseed=int to provide an +int seed value for the randomizer; this is useful for reproducing troublesome +test orders. + +-s On the first invocation of regrtest using -s, the first test file found +or the first test file given on the command line is run, and the name of +the next test is recorded in a file named pynexttest. If run from the +Python build directory, pynexttest is located in the 'build' subdirectory, +otherwise it is located in tempfile.gettempdir(). On subsequent runs, +the test in pynexttest is run, and the next test is written to pynexttest. +When the last test has been run, pynexttest is deleted. In this way it +is possible to single step through the test files. This is useful when +doing memory analysis on the Python interpreter, which process tends to +consume too many resources to run the full regression test non-stop. + +-S is used to continue running tests after an aborted run. It will +maintain the order a standard run (ie, this assumes -r is not used). +This is useful after the tests have prematurely stopped for some external +reason and you want to start running from where you left off rather +than starting from the beginning. + +-f reads the names of tests from the file given as f's argument, one +or more test names per line. Whitespace is ignored. Blank lines and +lines beginning with '#' are ignored. This is especially useful for +whittling down failures involving interactions among tests. + +-L causes the leaks(1) command to be run just before exit if it exists. +leaks(1) is available on Mac OS X and presumably on some other +FreeBSD-derived systems. + +-R runs each test several times and examines sys.gettotalrefcount() to +see if the test appears to be leaking references. The argument should +be of the form stab:run:fname where 'stab' is the number of times the +test is run to let gettotalrefcount settle down, 'run' is the number +of times further it is run and 'fname' is the name of the file the +reports are written to. These parameters all have defaults (5, 4 and +"reflog.txt" respectively), and the minimal invocation is '-R :'. + +-M runs tests that require an exorbitant amount of memory. These tests +typically try to ascertain containers keep working when containing more than +2 billion objects, which only works on 64-bit systems. There are also some +tests that try to exhaust the address space of the process, which only makes +sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, +which is a string in the form of '2.5Gb', determines howmuch memory the +tests will limit themselves to (but they may go slightly over.) The number +shouldn't be more memory than the machine has (including swap memory). You +should also keep in mind that swap memory is generally much, much slower +than RAM, and setting memlimit to all available RAM or higher will heavily +tax the machine. On the other hand, it is no use running these tests with a +limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect +to use more than memlimit memory will be skipped. The big-memory tests +generally run very, very long. + +-u is used to specify which special resource intensive tests to run, +such as those requiring large file support or network connectivity. +The argument is a comma-separated list of words indicating the +resources to test. Currently only the following are defined: + + all - Enable all special resources. + + none - Disable all special resources (this is the default). + + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) + + curses - Tests that use curses and will modify the terminal's + state and output modes. + + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2GB of disk space temporarily. + + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. + + decimal - Test the decimal module against a large suite that + verifies compliance with standards. + + cpu - Used for certain CPU-heavy tests. + + subprocess Run all tests for the subprocess module. + + urlfetch - It is okay to download files required on testing. + + gui - Run tests that require a running GUI. + + tzdata - Run tests that require timezone data. + +To enable all resources except one, use '-uall,-<resource>'. For +example, to run all the tests except for the gui tests, give the +option '-uall,-gui'. +""" + + +RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'tzdata') + +class _ArgParser(argparse.ArgumentParser): + + def error(self, message): + super().error(message + "\nPass -h or --help for complete help.") + + +def _create_parser(): + # Set prog to prevent the uninformative "__main__.py" from displaying in + # error messages when using "python -m test ...". + parser = _ArgParser(prog='regrtest.py', + usage=USAGE, + description=DESCRIPTION, + epilog=EPILOG, + add_help=False, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # Arguments with this clause added to its help are described further in + # the epilog's "Additional option details" section. + more_details = ' See the section at bottom for more details.' + + group = parser.add_argument_group('General options') + # We add help explicitly to control what argument group it renders under. + group.add_argument('-h', '--help', action='help', + help='show this help message and exit') + group.add_argument('--timeout', metavar='TIMEOUT', type=float, + help='dump the traceback and exit if a test takes ' + 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'is negative or equals to zero') + group.add_argument('--wait', action='store_true', + help='wait for user input, e.g., allow a debugger ' + 'to be attached') + group.add_argument('--slaveargs', metavar='ARGS') + group.add_argument('-S', '--start', metavar='START', + help='the name of the test at which to start.' + + more_details) + + group = parser.add_argument_group('Verbosity') + group.add_argument('-v', '--verbose', action='count', + help='run tests in verbose mode with output to stdout') + group.add_argument('-w', '--verbose2', action='store_true', + help='re-run failed tests in verbose mode') + group.add_argument('-W', '--verbose3', action='store_true', + help='display test output on failure') + group.add_argument('-q', '--quiet', action='store_true', + help='no output unless one or more tests fail') + group.add_argument('-o', '--slowest', action='store_true', dest='print_slow', + help='print the slowest 10 tests') + group.add_argument('--header', action='store_true', + help='print header with interpreter info') + + group = parser.add_argument_group('Selecting tests') + group.add_argument('-r', '--randomize', action='store_true', + help='randomize test execution order.' + more_details) + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a random seed to reproduce a previous ' + 'random run') + group.add_argument('-f', '--fromfile', metavar='FILE', + help='read names of tests to run from a file.' + + more_details) + group.add_argument('-x', '--exclude', action='store_true', + help='arguments are tests to *exclude*') + group.add_argument('-s', '--single', action='store_true', + help='single step through a set of tests.' + + more_details) + group.add_argument('-m', '--match', metavar='PAT', + dest='match_tests', + help='match test cases and methods with glob pattern PAT') + group.add_argument('-G', '--failfast', action='store_true', + help='fail as soon as a test fails (only with -v or -W)') + group.add_argument('-u', '--use', metavar='RES1,RES2,...', + action='append', type=resources_list, + help='specify which special resource intensive tests ' + 'to run.' + more_details) + group.add_argument('-M', '--memlimit', metavar='LIMIT', + help='run very large memory-consuming tests.' + + more_details) + group.add_argument('--testdir', metavar='DIR', + type=relative_filename, + help='execute test files in the specified directory ' + '(instead of the Python stdlib test suite)') + + group = parser.add_argument_group('Special runs') + group.add_argument('-l', '--findleaks', action='store_true', + help='if GC is available detect tests that leak memory') + group.add_argument('-L', '--runleaks', action='store_true', + help='run the leaks(1) command just before exit.' + + more_details) + group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', + type=huntrleaks, + help='search for reference leaks (needs debug build, ' + 'very slow).' + more_details) + group.add_argument('-j', '--multiprocess', metavar='PROCESSES', + dest='use_mp', type=int, + help='run PROCESSES processes at once') + group.add_argument('-T', '--coverage', action='store_true', + dest='trace', + help='turn on code coverage tracing using the trace ' + 'module') + group.add_argument('-D', '--coverdir', metavar='DIR', + type=relative_filename, + help='directory where coverage files are put') + group.add_argument('-N', '--nocoverdir', + action='store_const', const=None, dest='coverdir', + help='put coverage files alongside modules') + group.add_argument('-t', '--threshold', metavar='THRESHOLD', + type=int, + help='call gc.set_threshold(THRESHOLD)') + group.add_argument('-n', '--nowindows', action='store_true', + help='suppress error message boxes on Windows') + group.add_argument('-F', '--forever', action='store_true', + help='run the specified tests in a loop, until an ' + 'error happens') + group.add_argument('--list-tests', action='store_true', + help="only write the name of tests that will be run, " + "don't execute them") + group.add_argument('-P', '--pgo', dest='pgo', action='store_true', + help='enable Profile Guided Optimization training') + + return parser + + +def relative_filename(string): + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + return os.path.join(support.SAVEDCWD, string) + + +def huntrleaks(string): + args = string.split(':') + if len(args) not in (2, 3): + raise argparse.ArgumentTypeError( + 'needs 2 or 3 colon-separated arguments') + nwarmup = int(args[0]) if args[0] else 5 + ntracked = int(args[1]) if args[1] else 4 + fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' + return nwarmup, ntracked, fname + + +def resources_list(string): + u = [x.lower() for x in string.split(',')] + for r in u: + if r == 'all' or r == 'none': + continue + if r[0] == '-': + r = r[1:] + if r not in RESOURCE_NAMES: + raise argparse.ArgumentTypeError('invalid resource: ' + r) + return u + + +def _parse_args(args, **kwargs): + # Defaults + ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, + exclude=False, single=False, randomize=False, fromfile=None, + findleaks=False, use_resources=None, trace=False, coverdir='coverage', + runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, + random_seed=None, use_mp=None, verbose3=False, forever=False, + header=False, failfast=False, match_tests=None, pgo=False) + for k, v in kwargs.items(): + if not hasattr(ns, k): + raise TypeError('%r is an invalid keyword argument ' + 'for this function' % k) + setattr(ns, k, v) + if ns.use_resources is None: + ns.use_resources = [] + + parser = _create_parser() + # Issue #14191: argparse doesn't support "intermixed" positional and + # optional arguments. Use parse_known_args() as workaround. + ns.args = parser.parse_known_args(args=args, namespace=ns)[1] + for arg in ns.args: + if arg.startswith('-'): + parser.error("unrecognized arguments: %s" % arg) + sys.exit(1) + + if ns.single and ns.fromfile: + parser.error("-s and -f don't go together!") + if ns.use_mp is not None and ns.trace: + parser.error("-T and -j don't go together!") + if ns.use_mp is not None and ns.findleaks: + parser.error("-l and -j don't go together!") + if ns.failfast and not (ns.verbose or ns.verbose3): + parser.error("-G/--failfast needs either -v or -W") + if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3): + parser.error("--pgo/-v don't go together!") + + if ns.nowindows: + print("Warning: the --nowindows (-n) option is deprecated. " + "Use -vv to display assertions in stderr.", file=sys.stderr) + + if ns.quiet: + ns.verbose = 0 + if ns.timeout is not None: + if ns.timeout <= 0: + ns.timeout = None + if ns.use_mp is not None: + if ns.use_mp <= 0: + # Use all cores + extras for tests that like to sleep + ns.use_mp = 2 + (os.cpu_count() or 1) + if ns.use: + for a in ns.use: + for r in a: + if r == 'all': + ns.use_resources[:] = RESOURCE_NAMES + continue + if r == 'none': + del ns.use_resources[:] + continue + remove = False + if r[0] == '-': + remove = True + r = r[1:] + if remove: + if r in ns.use_resources: + ns.use_resources.remove(r) + elif r not in ns.use_resources: + ns.use_resources.append(r) + if ns.random_seed is not None: + ns.randomize = True + + return ns diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py new file mode 100644 index 0000000000..f0effc973c --- /dev/null +++ b/Lib/test/libregrtest/main.py @@ -0,0 +1,532 @@ +import datetime +import faulthandler +import locale +import os +import platform +import random +import re +import sys +import sysconfig +import tempfile +import textwrap +import time +from test.libregrtest.cmdline import _parse_args +from test.libregrtest.runtest import ( + findtests, runtest, + STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, + INTERRUPTED, CHILD_ERROR, + PROGRESS_MIN_TIME, format_test_result) +from test.libregrtest.setup import setup_tests +from test import support +try: + import gc +except ImportError: + gc = None + + +# When tests are run from the Python build directory, it is best practice +# to keep the test files in a subfolder. This eases the cleanup of leftover +# files using the "make distclean" command. +if sysconfig.is_python_build(): + TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') +else: + TEMPDIR = tempfile.gettempdir() +TEMPDIR = os.path.abspath(TEMPDIR) + + +def format_duration(seconds): + if seconds < 1.0: + return '%.0f ms' % (seconds * 1e3) + if seconds < 60.0: + return '%.0f sec' % seconds + + minutes, seconds = divmod(seconds, 60.0) + return '%.0f min %.0f sec' % (minutes, seconds) + + +class Regrtest: + """Execute a test suite. + + This also parses command-line options and modifies its behavior + accordingly. + + tests -- a list of strings containing test names (optional) + testdir -- the directory in which to look for tests (optional) + + Users other than the Python test suite will certainly want to + specify testdir; if it's omitted, the directory containing the + Python test suite is searched for. + + If the tests argument is omitted, the tests listed on the + command-line will be used. If that's empty, too, then all *.py + files beginning with test_ will be used. + + The other default arguments (verbose, quiet, exclude, + single, randomize, findleaks, use_resources, trace, coverdir, + print_slow, and random_seed) allow programmers calling main() + directly to set the values that would normally be set by flags + on the command line. + """ + def __init__(self): + # Namespace of command line options + self.ns = None + + # tests + self.tests = [] + self.selected = [] + + # test results + self.good = [] + self.bad = [] + self.skipped = [] + self.resource_denieds = [] + self.environment_changed = [] + self.interrupted = False + + # used by --slow + self.test_times = [] + + # used by --coverage, trace.Trace instance + self.tracer = None + + # used by --findleaks, store for gc.garbage + self.found_garbage = [] + + # used to display the progress bar "[ 3/100]" + self.start_time = time.monotonic() + self.test_count = '' + self.test_count_width = 1 + + # used by --single + self.next_single_test = None + self.next_single_filename = None + + def accumulate_result(self, test, result): + ok, test_time = result + if ok not in (CHILD_ERROR, INTERRUPTED): + self.test_times.append((test_time, test)) + if ok == PASSED: + self.good.append(test) + elif ok == FAILED: + self.bad.append(test) + elif ok == ENV_CHANGED: + self.environment_changed.append(test) + elif ok == SKIPPED: + self.skipped.append(test) + elif ok == RESOURCE_DENIED: + self.skipped.append(test) + self.resource_denieds.append(test) + + def display_progress(self, test_index, test): + if self.ns.quiet: + return + if self.bad and not self.ns.pgo: + fmt = "{time} [{test_index:{count_width}}{test_count}/{nbad}] {test_name}" + else: + fmt = "{time} [{test_index:{count_width}}{test_count}] {test_name}" + test_time = time.monotonic() - self.start_time + test_time = datetime.timedelta(seconds=int(test_time)) + line = fmt.format(count_width=self.test_count_width, + test_index=test_index, + test_count=self.test_count, + nbad=len(self.bad), + test_name=test, + time=test_time) + print(line, flush=True) + + def parse_args(self, kwargs): + ns = _parse_args(sys.argv[1:], **kwargs) + + if ns.timeout and not hasattr(faulthandler, 'dump_traceback_later'): + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later", file=sys.stderr) + ns.timeout = None + + if ns.threshold is not None and gc is None: + print('No GC available, ignore --threshold.', file=sys.stderr) + ns.threshold = None + + if ns.findleaks: + if gc is not None: + # Uncomment the line below to report garbage that is not + # freeable by reference counting alone. By default only + # garbage that is not collectable by the GC is reported. + pass + #gc.set_debug(gc.DEBUG_SAVEALL) + else: + print('No GC available, disabling --findleaks', + file=sys.stderr) + ns.findleaks = False + + # Strip .py extensions. + removepy(ns.args) + + return ns + + def find_tests(self, tests): + self.tests = tests + + if self.ns.single: + self.next_single_filename = os.path.join(TEMPDIR, 'pynexttest') + try: + with open(self.next_single_filename, 'r') as fp: + next_test = fp.read().strip() + self.tests = [next_test] + except OSError: + pass + + if self.ns.fromfile: + self.tests = [] + # regex to match 'test_builtin' in line: + # '0:00:00 [ 4/400] test_builtin -- test_dict took 1 sec' + regex = (r'^(?:[0-9]+:[0-9]+:[0-9]+ *)?' + r'(?:\[[0-9/ ]+\] *)?' + r'(test_[a-zA-Z0-9_]+)') + regex = re.compile(regex) + with open(os.path.join(support.SAVEDCWD, self.ns.fromfile)) as fp: + for line in fp: + line = line.strip() + if line.startswith('#'): + continue + match = regex.match(line) + if match is None: + continue + self.tests.append(match.group(1)) + + removepy(self.tests) + + stdtests = STDTESTS[:] + nottests = NOTTESTS.copy() + if self.ns.exclude: + for arg in self.ns.args: + if arg in stdtests: + stdtests.remove(arg) + nottests.add(arg) + self.ns.args = [] + + # if testdir is set, then we are not running the python tests suite, so + # don't add default tests to be executed or skipped (pass empty values) + if self.ns.testdir: + alltests = findtests(self.ns.testdir, list(), set()) + else: + alltests = findtests(self.ns.testdir, stdtests, nottests) + + if not self.ns.fromfile: + self.selected = self.tests or self.ns.args or alltests + else: + self.selected = self.tests + if self.ns.single: + self.selected = self.selected[:1] + try: + pos = alltests.index(self.selected[0]) + self.next_single_test = alltests[pos + 1] + except IndexError: + pass + + # Remove all the selected tests that precede start if it's set. + if self.ns.start: + try: + del self.selected[:self.selected.index(self.ns.start)] + except ValueError: + print("Couldn't find starting test (%s), using all tests" + % self.ns.start, file=sys.stderr) + + if self.ns.randomize: + if self.ns.random_seed is None: + self.ns.random_seed = random.randrange(10000000) + random.seed(self.ns.random_seed) + random.shuffle(self.selected) + + def list_tests(self): + for name in self.selected: + print(name) + + def rerun_failed_tests(self): + self.ns.verbose = True + self.ns.failfast = False + self.ns.verbose3 = False + self.ns.match_tests = None + + print("Re-running failed tests in verbose mode") + for test in self.bad[:]: + print("Re-running test %r in verbose mode" % test, flush=True) + try: + self.ns.verbose = True + ok = runtest(self.ns, test) + except KeyboardInterrupt: + self.interrupted = True + # print a newline separate from the ^C + print() + break + else: + if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: + self.bad.remove(test) + else: + if self.bad: + print(count(len(self.bad), 'test'), "failed again:") + printlist(self.bad) + + def display_result(self): + if self.interrupted: + # print a newline after ^C + print() + print("Test suite interrupted by signal SIGINT.") + executed = set(self.good) | set(self.bad) | set(self.skipped) + omitted = set(self.selected) - executed + print(count(len(omitted), "test"), "omitted:") + printlist(omitted) + + # If running the test suite for PGO then no one cares about + # results. + if self.ns.pgo: + return + + if self.good and not self.ns.quiet: + if (not self.bad + and not self.skipped + and not self.interrupted + and len(self.good) > 1): + print("All", end=' ') + print(count(len(self.good), "test"), "OK.") + + if self.ns.print_slow: + self.test_times.sort(reverse=True) + print() + print("10 slowest tests:") + for time, test in self.test_times[:10]: + print("- %s: %s" % (test, format_duration(time))) + + if self.bad: + print() + print(count(len(self.bad), "test"), "failed:") + printlist(self.bad) + + if self.environment_changed: + print() + print("{} altered the execution environment:".format( + count(len(self.environment_changed), "test"))) + printlist(self.environment_changed) + + if self.skipped and not self.ns.quiet: + print() + print(count(len(self.skipped), "test"), "skipped:") + printlist(self.skipped) + + def run_tests_sequential(self): + if self.ns.trace: + import trace + self.tracer = trace.Trace(trace=False, count=True) + + save_modules = sys.modules.keys() + + print("Run tests sequentially") + + previous_test = None + for test_index, test in enumerate(self.tests, 1): + start_time = time.monotonic() + + text = test + if previous_test: + text = '%s -- %s' % (text, previous_test) + self.display_progress(test_index, text) + + if self.tracer: + # If we're tracing code coverage, then we don't exit with status + # if on a false return value from main. + cmd = ('result = runtest(self.ns, test); ' + 'self.accumulate_result(test, result)') + ns = dict(locals()) + self.tracer.runctx(cmd, globals=globals(), locals=ns) + result = ns['result'] + else: + try: + result = runtest(self.ns, test) + except KeyboardInterrupt: + self.interrupted = True + self.accumulate_result(test, (INTERRUPTED, None)) + break + else: + self.accumulate_result(test, result) + + previous_test = format_test_result(test, result[0]) + test_time = time.monotonic() - start_time + if test_time >= PROGRESS_MIN_TIME: + previous_test = "%s in %s" % (previous_test, format_duration(test_time)) + elif result[0] == PASSED: + # be quiet: say nothing if the test passed shortly + previous_test = None + + if self.ns.findleaks: + gc.collect() + if gc.garbage: + print("Warning: test created", len(gc.garbage), end=' ') + print("uncollectable object(s).") + # move the uncollectable objects somewhere so we don't see + # them again + self.found_garbage.extend(gc.garbage) + del gc.garbage[:] + + # Unload the newly imported modules (best effort finalization) + for module in sys.modules.keys(): + if module not in save_modules and module.startswith("test."): + support.unload(module) + + if previous_test: + print(previous_test) + + def _test_forever(self, tests): + while True: + for test in tests: + yield test + if self.bad: + return + + def run_tests(self): + # For a partial run, we do not need to clutter the output. + if (self.ns.verbose + or self.ns.header + or not (self.ns.pgo or self.ns.quiet or self.ns.single + or self.tests or self.ns.args)): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("== ", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== ", "hash algorithm:", sys.hash_info.algorithm, + "64bit" if sys.maxsize > 2**32 else "32bit") + print("== cwd:", os.getcwd()) + print("== encodings: locale=%s, FS=%s" + % (locale.getpreferredencoding(False), + sys.getfilesystemencoding())) + print("Testing with flags:", sys.flags) + + if self.ns.randomize: + print("Using random seed", self.ns.random_seed) + + if self.ns.forever: + self.tests = self._test_forever(list(self.selected)) + self.test_count = '' + self.test_count_width = 3 + else: + self.tests = iter(self.selected) + self.test_count = '/{}'.format(len(self.selected)) + self.test_count_width = len(self.test_count) - 1 + + if self.ns.use_mp: + from test.libregrtest.runtest_mp import run_tests_multiprocess + run_tests_multiprocess(self) + else: + self.run_tests_sequential() + + def finalize(self): + if self.next_single_filename: + if self.next_single_test: + with open(self.next_single_filename, 'w') as fp: + fp.write(self.next_single_test + '\n') + else: + os.unlink(self.next_single_filename) + + if self.tracer: + r = self.tracer.results() + r.write_results(show_missing=True, summary=True, + coverdir=self.ns.coverdir) + + print() + duration = time.monotonic() - self.start_time + print("Total duration: %s" % format_duration(duration)) + + if self.bad: + result = "FAILURE" + elif self.interrupted: + result = "INTERRUPTED" + else: + result = "SUCCESS" + print("Tests result: %s" % result) + + if self.ns.runleaks: + os.system("leaks %d" % os.getpid()) + + def main(self, tests=None, **kwargs): + global TEMPDIR + + if sysconfig.is_python_build(): + try: + os.mkdir(TEMPDIR) + except FileExistsError: + pass + + # Define a writable temp dir that will be used as cwd while running + # the tests. The name of the dir includes the pid to allow parallel + # testing (see the -j option). + test_cwd = 'test_python_{}'.format(os.getpid()) + test_cwd = os.path.join(TEMPDIR, test_cwd) + + # Run the tests in a context manager that temporarily changes the CWD to a + # temporary and writable directory. If it's not possible to create or + # change the CWD, the original CWD will be used. The original CWD is + # available from support.SAVEDCWD. + with support.temp_cwd(test_cwd, quiet=True): + self._main(tests, kwargs) + + def _main(self, tests, kwargs): + self.ns = self.parse_args(kwargs) + + if self.ns.slaveargs is not None: + from test.libregrtest.runtest_mp import run_tests_slave + run_tests_slave(self.ns.slaveargs) + + if self.ns.wait: + input("Press any key to continue...") + + support.PGO = self.ns.pgo + + setup_tests(self.ns) + + self.find_tests(tests) + + if self.ns.list_tests: + self.list_tests() + sys.exit(0) + + self.run_tests() + self.display_result() + + if self.ns.verbose2 and self.bad: + self.rerun_failed_tests() + + self.finalize() + sys.exit(len(self.bad) > 0 or self.interrupted) + + +def removepy(names): + if not names: + return + for idx, name in enumerate(names): + basename, ext = os.path.splitext(name) + if ext == '.py': + names[idx] = basename + + +def count(n, word): + if n == 1: + return "%d %s" % (n, word) + else: + return "%d %ss" % (n, word) + + +def printlist(x, width=70, indent=4): + """Print the elements of iterable x to stdout. + + Optional arg width (default 70) is the maximum line length. + Optional arg indent (default 4) is the number of blanks with which to + begin each line. + """ + + blanks = ' ' * indent + # Print the sorted list: 'x' may be a '--random' list or a set() + print(textwrap.fill(' '.join(str(elt) for elt in sorted(x)), width, + initial_indent=blanks, subsequent_indent=blanks)) + + +def main(tests=None, **kwargs): + """Run the Python suite.""" + Regrtest().main(tests=tests, **kwargs) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py new file mode 100644 index 0000000000..3b3d2458b1 --- /dev/null +++ b/Lib/test/libregrtest/refleak.py @@ -0,0 +1,269 @@ +import errno +import os +import re +import sys +import warnings +from inspect import isabstract +from test import support + + +try: + MAXFD = os.sysconf("SC_OPEN_MAX") +except Exception: + MAXFD = 256 + + +def fd_count(): + """Count the number of open file descriptors""" + if sys.platform.startswith(('linux', 'freebsd')): + try: + names = os.listdir("/proc/self/fd") + return len(names) + except FileNotFoundError: + pass + + count = 0 + for fd in range(MAXFD): + try: + # Prefer dup() over fstat(). fstat() can require input/output + # whereas dup() doesn't. + fd2 = os.dup(fd) + except OSError as e: + if e.errno != errno.EBADF: + raise + else: + os.close(fd2) + count += 1 + return count + + +def dash_R(the_module, test, indirect_test, huntrleaks): + """Run a test multiple times, looking for reference leaks. + + Returns: + False if the test didn't leak references; True if we detected refleaks. + """ + # This code is hackish and inelegant, but it seems to do the job. + import copyreg + import collections.abc + + if not hasattr(sys, 'gettotalrefcount'): + raise Exception("Tracking reference leaks requires a debug build " + "of Python") + + # Save current values for dash_R_cleanup() to restore. + fs = warnings.filters[:] + ps = copyreg.dispatch_table.copy() + pic = sys.path_importer_cache.copy() + try: + import zipimport + except ImportError: + zdc = None # Run unmodified on platforms without zipimport support + else: + zdc = zipimport._zip_directory_cache.copy() + abcs = {} + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: + if not isabstract(abc): + continue + for obj in abc.__subclasses__() + [abc]: + abcs[obj] = obj._abc_registry.copy() + + nwarmup, ntracked, fname = huntrleaks + fname = os.path.join(support.SAVEDCWD, fname) + repcount = nwarmup + ntracked + rc_deltas = [0] * repcount + alloc_deltas = [0] * repcount + fd_deltas = [0] * repcount + + print("beginning", repcount, "repetitions", file=sys.stderr) + print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, + flush=True) + # initialize variables to make pyflakes quiet + rc_before = alloc_before = fd_before = 0 + for i in range(repcount): + indirect_test() + alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc, + abcs) + print('.', end='', flush=True) + if i >= nwarmup: + rc_deltas[i] = rc_after - rc_before + alloc_deltas[i] = alloc_after - alloc_before + fd_deltas[i] = fd_after - fd_before + alloc_before = alloc_after + rc_before = rc_after + fd_before = fd_after + print(file=sys.stderr) + # These checkers return False on success, True on failure + def check_rc_deltas(deltas): + return any(deltas) + def check_alloc_deltas(deltas): + # At least 1/3rd of 0s + if 3 * deltas.count(0) < len(deltas): + return True + # Nothing else than 1s, 0s and -1s + if not set(deltas) <= {1,0,-1}: + return True + return False + failed = False + for deltas, item_name, checker in [ + (rc_deltas, 'references', check_rc_deltas), + (alloc_deltas, 'memory blocks', check_alloc_deltas), + (fd_deltas, 'file descriptors', check_rc_deltas)]: + if checker(deltas): + msg = '%s leaked %s %s, sum=%s' % ( + test, deltas[nwarmup:], item_name, sum(deltas)) + print(msg, file=sys.stderr, flush=True) + with open(fname, "a") as refrep: + print(msg, file=refrep) + refrep.flush() + failed = True + return failed + + +def dash_R_cleanup(fs, ps, pic, zdc, abcs): + import gc, copyreg + import collections.abc + from weakref import WeakSet + + # Restore some original values. + warnings.filters[:] = fs + copyreg.dispatch_table.clear() + copyreg.dispatch_table.update(ps) + sys.path_importer_cache.clear() + sys.path_importer_cache.update(pic) + try: + import zipimport + except ImportError: + pass # Run unmodified on platforms without zipimport support + else: + zipimport._zip_directory_cache.clear() + zipimport._zip_directory_cache.update(zdc) + + # clear type cache + sys._clear_type_cache() + + # Clear ABC registries, restoring previously saved ABC registries. + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: + if not isabstract(abc): + continue + for obj in abc.__subclasses__() + [abc]: + obj._abc_registry = abcs.get(obj, WeakSet()).copy() + obj._abc_cache.clear() + obj._abc_negative_cache.clear() + + clear_caches() + + # Collect cyclic trash and read memory statistics immediately after. + func1 = sys.getallocatedblocks + func2 = sys.gettotalrefcount + gc.collect() + return func1(), func2(), fd_count() + + +def clear_caches(): + import gc + + # Clear the warnings registry, so they can be displayed again + for mod in sys.modules.values(): + if hasattr(mod, '__warningregistry__'): + del mod.__warningregistry__ + + # Flush standard output, so that buffered data is sent to the OS and + # associated Python objects are reclaimed. + for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): + if stream is not None: + stream.flush() + + # Clear assorted module caches. + # Don't worry about resetting the cache if the module is not loaded + try: + distutils_dir_util = sys.modules['distutils.dir_util'] + except KeyError: + pass + else: + distutils_dir_util._path_created.clear() + re.purge() + + try: + _strptime = sys.modules['_strptime'] + except KeyError: + pass + else: + _strptime._regex_cache.clear() + + try: + urllib_parse = sys.modules['urllib.parse'] + except KeyError: + pass + else: + urllib_parse.clear_cache() + + try: + urllib_request = sys.modules['urllib.request'] + except KeyError: + pass + else: + urllib_request.urlcleanup() + + try: + linecache = sys.modules['linecache'] + except KeyError: + pass + else: + linecache.clearcache() + + try: + mimetypes = sys.modules['mimetypes'] + except KeyError: + pass + else: + mimetypes._default_mime_types() + + try: + filecmp = sys.modules['filecmp'] + except KeyError: + pass + else: + filecmp._cache.clear() + + try: + struct = sys.modules['struct'] + except KeyError: + pass + else: + struct._clearcache() + + try: + doctest = sys.modules['doctest'] + except KeyError: + pass + else: + doctest.master = None + + try: + ctypes = sys.modules['ctypes'] + except KeyError: + pass + else: + ctypes._reset_cache() + + try: + typing = sys.modules['typing'] + except KeyError: + pass + else: + for f in typing._cleanups: + f() + + gc.collect() + + +def warm_caches(): + # char cache + s = bytes(range(256)) + for i in range(256): + s[i:i+1] + # unicode cache + [chr(i) for i in range(256)] + # int cache + list(range(-5, 257)) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py new file mode 100644 index 0000000000..ba0df0a317 --- /dev/null +++ b/Lib/test/libregrtest/runtest.py @@ -0,0 +1,245 @@ +import faulthandler +import importlib +import io +import os +import sys +import time +import traceback +import unittest +from test import support +from test.libregrtest.refleak import dash_R, clear_caches +from test.libregrtest.save_env import saved_test_environment + + +# Test result constants. +PASSED = 1 +FAILED = 0 +ENV_CHANGED = -1 +SKIPPED = -2 +RESOURCE_DENIED = -3 +INTERRUPTED = -4 +CHILD_ERROR = -5 # error in a child process + +_FORMAT_TEST_RESULT = { + PASSED: '%s passed', + FAILED: '%s failed', + ENV_CHANGED: '%s failed (env changed)', + SKIPPED: '%s skipped', + RESOURCE_DENIED: '%s skipped (resource denied)', + INTERRUPTED: '%s interrupted', + CHILD_ERROR: '%s crashed', +} + +# Minimum duration of a test to display its duration or to mention that +# the test is running in background +PROGRESS_MIN_TIME = 30.0 # seconds + +# small set of tests to determine if we have a basically functioning interpreter +# (i.e. if any of these fail, then anything else is likely to follow) +STDTESTS = [ + 'test_grammar', + 'test_opcodes', + 'test_dict', + 'test_builtin', + 'test_exceptions', + 'test_types', + 'test_unittest', + 'test_doctest', + 'test_doctest2', + 'test_support' +] + +# set of tests that we don't want to be executed when using regrtest +NOTTESTS = set() + + +def format_test_result(test_name, result): + fmt = _FORMAT_TEST_RESULT.get(result, "%s") + return fmt % test_name + + +def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): + """Return a list of all applicable test modules.""" + testdir = findtestdir(testdir) + names = os.listdir(testdir) + tests = [] + others = set(stdtests) | nottests + for name in names: + mod, ext = os.path.splitext(name) + if mod[:5] == "test_" and ext in (".py", "") and mod not in others: + tests.append(mod) + return stdtests + sorted(tests) + + +def runtest(ns, test): + """Run a single test. + + ns -- regrtest namespace of options + test -- the name of the test + + Returns the tuple (result, test_time), where result is one of the + constants: + + INTERRUPTED KeyboardInterrupt when run under -j + RESOURCE_DENIED test skipped because resource denied + SKIPPED test skipped for some other reason + ENV_CHANGED test failed because it changed the execution environment + FAILED test failed + PASSED test passed + """ + + output_on_failure = ns.verbose3 + + use_timeout = (ns.timeout is not None) + if use_timeout: + faulthandler.dump_traceback_later(ns.timeout, exit=True) + try: + support.match_tests = ns.match_tests + if ns.failfast: + support.failfast = True + if output_on_failure: + support.verbose = True + + # Reuse the same instance to all calls to runtest(). Some + # tests keep a reference to sys.stdout or sys.stderr + # (eg. test_argparse). + if runtest.stringio is None: + stream = io.StringIO() + runtest.stringio = stream + else: + stream = runtest.stringio + stream.seek(0) + stream.truncate() + + orig_stdout = sys.stdout + orig_stderr = sys.stderr + try: + sys.stdout = stream + sys.stderr = stream + result = runtest_inner(ns, test, display_failure=False) + if result[0] != PASSED: + output = stream.getvalue() + orig_stderr.write(output) + orig_stderr.flush() + finally: + sys.stdout = orig_stdout + sys.stderr = orig_stderr + else: + support.verbose = ns.verbose # Tell tests to be moderately quiet + result = runtest_inner(ns, test, display_failure=not ns.verbose) + return result + finally: + if use_timeout: + faulthandler.cancel_dump_traceback_later() + cleanup_test_droppings(test, ns.verbose) +runtest.stringio = None + + +def runtest_inner(ns, test, display_failure=True): + support.unload(test) + + test_time = 0.0 + refleak = False # True if the test leaked references. + try: + if test.startswith('test.') or ns.testdir: + abstest = test + else: + # Always import it from the test package + abstest = 'test.' + test + clear_caches() + with saved_test_environment(test, ns.verbose, ns.quiet, pgo=ns.pgo) as environment: + start_time = time.time() + the_module = importlib.import_module(abstest) + # If the test has a test_main, that will run the appropriate + # tests. If not, use normal unittest test loading. + test_runner = getattr(the_module, "test_main", None) + if test_runner is None: + def test_runner(): + loader = unittest.TestLoader() + tests = loader.loadTestsFromModule(the_module) + for error in loader.errors: + print(error, file=sys.stderr) + if loader.errors: + raise Exception("errors while loading tests") + support.run_unittest(tests) + test_runner() + if ns.huntrleaks: + refleak = dash_R(the_module, test, test_runner, ns.huntrleaks) + test_time = time.time() - start_time + except support.ResourceDenied as msg: + if not ns.quiet and not ns.pgo: + print(test, "skipped --", msg, flush=True) + return RESOURCE_DENIED, test_time + except unittest.SkipTest as msg: + if not ns.quiet and not ns.pgo: + print(test, "skipped --", msg, flush=True) + return SKIPPED, test_time + except KeyboardInterrupt: + raise + except support.TestFailed as msg: + if not ns.pgo: + if display_failure: + print("test", test, "failed --", msg, file=sys.stderr, + flush=True) + else: + print("test", test, "failed", file=sys.stderr, flush=True) + return FAILED, test_time + except: + msg = traceback.format_exc() + if not ns.pgo: + print("test", test, "crashed --", msg, file=sys.stderr, + flush=True) + return FAILED, test_time + else: + if refleak: + return FAILED, test_time + if environment.changed: + return ENV_CHANGED, test_time + return PASSED, test_time + + +def cleanup_test_droppings(testname, verbose): + import shutil + import stat + import gc + + # First kill any dangling references to open files etc. + # This can also issue some ResourceWarnings which would otherwise get + # triggered during the following test run, and possibly produce failures. + gc.collect() + + # Try to clean up junk commonly left behind. While tests shouldn't leave + # any files or directories behind, when a test fails that can be tedious + # for it to arrange. The consequences can be especially nasty on Windows, + # since if a test leaves a file open, it cannot be deleted by name (while + # there's nothing we can do about that here either, we can display the + # name of the offending test, which is a real help). + for name in (support.TESTFN, + "db_home", + ): + if not os.path.exists(name): + continue + + if os.path.isdir(name): + kind, nuker = "directory", shutil.rmtree + elif os.path.isfile(name): + kind, nuker = "file", os.unlink + else: + raise SystemError("os.path says %r exists but is neither " + "directory nor file" % name) + + if verbose: + print("%r left behind %s %r" % (testname, kind, name)) + try: + # if we have chmod, fix possible permissions problems + # that might prevent cleanup + if (hasattr(os, 'chmod')): + os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + nuker(name) + except Exception as msg: + print(("%r left behind %s %r and it couldn't be " + "removed: %s" % (testname, kind, name, msg)), file=sys.stderr) + + +def findtestdir(path=None): + return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py new file mode 100644 index 0000000000..9604c16600 --- /dev/null +++ b/Lib/test/libregrtest/runtest_mp.py @@ -0,0 +1,245 @@ +import faulthandler +import json +import os +import queue +import sys +import time +import traceback +import types +from test import support +try: + import threading +except ImportError: + print("Multiprocess option requires thread support") + sys.exit(2) + +from test.libregrtest.runtest import ( + runtest, INTERRUPTED, CHILD_ERROR, PROGRESS_MIN_TIME, + format_test_result) +from test.libregrtest.setup import setup_tests + + +# Display the running tests if nothing happened last N seconds +PROGRESS_UPDATE = 30.0 # seconds + +# If interrupted, display the wait progress every N seconds +WAIT_PROGRESS = 2.0 # seconds + + +def run_test_in_subprocess(testname, ns): + """Run the given test in a subprocess with --slaveargs. + + ns is the option Namespace parsed from command-line arguments. regrtest + is invoked in a subprocess with the --slaveargs argument; when the + subprocess exits, its return code, stdout and stderr are returned as a + 3-tuple. + """ + from subprocess import Popen, PIPE + + ns_dict = vars(ns) + slaveargs = (ns_dict, testname) + slaveargs = json.dumps(slaveargs) + + cmd = [sys.executable, *support.args_from_interpreter_flags(), + '-X', 'faulthandler', + '-m', 'test.regrtest', + '--slaveargs', slaveargs] + if ns.pgo: + cmd += ['--pgo'] + + # Running the child from the same working directory as regrtest's original + # invocation ensures that TEMPDIR for the child is the same when + # sysconfig.is_python_build() is true. See issue 15300. + popen = Popen(cmd, + stdout=PIPE, stderr=PIPE, + universal_newlines=True, + close_fds=(os.name != 'nt'), + cwd=support.SAVEDCWD) + with popen: + stdout, stderr = popen.communicate() + retcode = popen.wait() + return retcode, stdout, stderr + + +def run_tests_slave(slaveargs): + ns_dict, testname = json.loads(slaveargs) + ns = types.SimpleNamespace(**ns_dict) + + setup_tests(ns) + + try: + result = runtest(ns, testname) + except KeyboardInterrupt: + result = INTERRUPTED, '' + except BaseException as e: + traceback.print_exc() + result = CHILD_ERROR, str(e) + + print() # Force a newline (just in case) + print(json.dumps(result), flush=True) + sys.exit(0) + + +# We do not use a generator so multiple threads can call next(). +class MultiprocessIterator: + + """A thread-safe iterator over tests for multiprocess mode.""" + + def __init__(self, tests): + self.interrupted = False + self.lock = threading.Lock() + self.tests = tests + + def __iter__(self): + return self + + def __next__(self): + with self.lock: + if self.interrupted: + raise StopIteration('tests interrupted') + return next(self.tests) + + +class MultiprocessThread(threading.Thread): + def __init__(self, pending, output, ns): + super().__init__() + self.pending = pending + self.output = output + self.ns = ns + self.current_test = None + self.start_time = None + + def _runtest(self): + try: + test = next(self.pending) + except StopIteration: + self.output.put((None, None, None, None)) + return True + + try: + self.start_time = time.monotonic() + self.current_test = test + + retcode, stdout, stderr = run_test_in_subprocess(test, self.ns) + finally: + self.current_test = None + + stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) + return True + + if not result: + self.output.put((None, None, None, None)) + return True + + result = json.loads(result) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) + return False + + def run(self): + try: + stop = False + while not stop: + stop = self._runtest() + except BaseException: + self.output.put((None, None, None, None)) + raise + + +def run_tests_multiprocess(regrtest): + output = queue.Queue() + pending = MultiprocessIterator(regrtest.tests) + test_timeout = regrtest.ns.timeout + use_timeout = (test_timeout is not None) + + workers = [MultiprocessThread(pending, output, regrtest.ns) + for i in range(regrtest.ns.use_mp)] + print("Run tests in parallel using %s child processes" + % len(workers)) + for worker in workers: + worker.start() + + def get_running(workers): + running = [] + for worker in workers: + current_test = worker.current_test + if not current_test: + continue + dt = time.monotonic() - worker.start_time + if dt >= PROGRESS_MIN_TIME: + running.append('%s (%.0f sec)' % (current_test, dt)) + return running + + finished = 0 + test_index = 1 + get_timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME) + try: + while finished < regrtest.ns.use_mp: + if use_timeout: + faulthandler.dump_traceback_later(test_timeout, exit=True) + + try: + item = output.get(timeout=get_timeout) + except queue.Empty: + running = get_running(workers) + if running and not regrtest.ns.pgo: + print('running: %s' % ', '.join(running)) + continue + + test, stdout, stderr, result = item + if test is None: + finished += 1 + continue + regrtest.accumulate_result(test, result) + + # Display progress + ok, test_time = result + text = format_test_result(test, ok) + if (ok not in (CHILD_ERROR, INTERRUPTED) + and test_time >= PROGRESS_MIN_TIME + and not regrtest.ns.pgo): + text += ' (%.0f sec)' % test_time + running = get_running(workers) + if running and not regrtest.ns.pgo: + text += ' -- running: %s' % ', '.join(running) + regrtest.display_progress(test_index, text) + + # Copy stdout and stderr from the child process + if stdout: + print(stdout, flush=True) + if stderr and not regrtest.ns.pgo: + print(stderr, file=sys.stderr, flush=True) + + if result[0] == INTERRUPTED: + raise KeyboardInterrupt + if result[0] == CHILD_ERROR: + msg = "Child error on {}: {}".format(test, result[1]) + raise Exception(msg) + test_index += 1 + except KeyboardInterrupt: + regrtest.interrupted = True + pending.interrupted = True + print() + finally: + if use_timeout: + faulthandler.cancel_dump_traceback_later() + + # If tests are interrupted, wait until tests complete + wait_start = time.monotonic() + while True: + running = [worker.current_test for worker in workers] + running = list(filter(bool, running)) + if not running: + break + + dt = time.monotonic() - wait_start + line = "Waiting for %s (%s tests)" % (', '.join(running), len(running)) + if dt >= WAIT_PROGRESS: + line = "%s since %.0f sec" % (line, dt) + print(line) + for worker in workers: + worker.join(WAIT_PROGRESS) diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py new file mode 100644 index 0000000000..96ad3af8df --- /dev/null +++ b/Lib/test/libregrtest/save_env.py @@ -0,0 +1,285 @@ +import builtins +import locale +import logging +import os +import shutil +import sys +import sysconfig +import warnings +from test import support +try: + import threading +except ImportError: + threading = None +try: + import _multiprocessing, multiprocessing.process +except ImportError: + multiprocessing = None + + +# Unit tests are supposed to leave the execution environment unchanged +# once they complete. But sometimes tests have bugs, especially when +# tests fail, and the changes to environment go on to mess up other +# tests. This can cause issues with buildbot stability, since tests +# are run in random order and so problems may appear to come and go. +# There are a few things we can save and restore to mitigate this, and +# the following context manager handles this task. + +class saved_test_environment: + """Save bits of the test environment and restore them at block exit. + + with saved_test_environment(testname, verbose, quiet): + #stuff + + Unless quiet is True, a warning is printed to stderr if any of + the saved items was changed by the test. The attribute 'changed' + is initially False, but is set to True if a change is detected. + + If verbose is more than 1, the before and after state of changed + items is also printed. + """ + + changed = False + + def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): + self.testname = testname + self.verbose = verbose + self.quiet = quiet + self.pgo = pgo + + # To add things to save and restore, add a name XXX to the resources list + # and add corresponding get_XXX/restore_XXX functions. get_XXX should + # return the value to be saved and compared against a second call to the + # get function when test execution completes. restore_XXX should accept + # the saved value and restore the resource using it. It will be called if + # and only if a change in the value is detected. + # + # Note: XXX will have any '.' replaced with '_' characters when determining + # the corresponding method names. + + resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', + 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', + 'warnings.filters', 'asyncore.socket_map', + 'logging._handlers', 'logging._handlerList', 'sys.gettrace', + 'sys.warnoptions', + # multiprocessing.process._cleanup() may release ref + # to a thread, so check processes first. + 'multiprocessing.process._dangling', 'threading._dangling', + 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', + 'files', 'locale', 'warnings.showwarning', + 'shutil_archive_formats', 'shutil_unpack_formats', + ) + + def get_sys_argv(self): + return id(sys.argv), sys.argv, sys.argv[:] + def restore_sys_argv(self, saved_argv): + sys.argv = saved_argv[1] + sys.argv[:] = saved_argv[2] + + def get_cwd(self): + return os.getcwd() + def restore_cwd(self, saved_cwd): + os.chdir(saved_cwd) + + def get_sys_stdout(self): + return sys.stdout + def restore_sys_stdout(self, saved_stdout): + sys.stdout = saved_stdout + + def get_sys_stderr(self): + return sys.stderr + def restore_sys_stderr(self, saved_stderr): + sys.stderr = saved_stderr + + def get_sys_stdin(self): + return sys.stdin + def restore_sys_stdin(self, saved_stdin): + sys.stdin = saved_stdin + + def get_os_environ(self): + return id(os.environ), os.environ, dict(os.environ) + def restore_os_environ(self, saved_environ): + os.environ = saved_environ[1] + os.environ.clear() + os.environ.update(saved_environ[2]) + + def get_sys_path(self): + return id(sys.path), sys.path, sys.path[:] + def restore_sys_path(self, saved_path): + sys.path = saved_path[1] + sys.path[:] = saved_path[2] + + def get_sys_path_hooks(self): + return id(sys.path_hooks), sys.path_hooks, sys.path_hooks[:] + def restore_sys_path_hooks(self, saved_hooks): + sys.path_hooks = saved_hooks[1] + sys.path_hooks[:] = saved_hooks[2] + + def get_sys_gettrace(self): + return sys.gettrace() + def restore_sys_gettrace(self, trace_fxn): + sys.settrace(trace_fxn) + + def get___import__(self): + return builtins.__import__ + def restore___import__(self, import_): + builtins.__import__ = import_ + + def get_warnings_filters(self): + return id(warnings.filters), warnings.filters, warnings.filters[:] + def restore_warnings_filters(self, saved_filters): + warnings.filters = saved_filters[1] + warnings.filters[:] = saved_filters[2] + + def get_asyncore_socket_map(self): + asyncore = sys.modules.get('asyncore') + # XXX Making a copy keeps objects alive until __exit__ gets called. + return asyncore and asyncore.socket_map.copy() or {} + def restore_asyncore_socket_map(self, saved_map): + asyncore = sys.modules.get('asyncore') + if asyncore is not None: + asyncore.close_all(ignore_all=True) + asyncore.socket_map.update(saved_map) + + def get_shutil_archive_formats(self): + # we could call get_archives_formats() but that only returns the + # registry keys; we want to check the values too (the functions that + # are registered) + return shutil._ARCHIVE_FORMATS, shutil._ARCHIVE_FORMATS.copy() + def restore_shutil_archive_formats(self, saved): + shutil._ARCHIVE_FORMATS = saved[0] + shutil._ARCHIVE_FORMATS.clear() + shutil._ARCHIVE_FORMATS.update(saved[1]) + + def get_shutil_unpack_formats(self): + return shutil._UNPACK_FORMATS, shutil._UNPACK_FORMATS.copy() + def restore_shutil_unpack_formats(self, saved): + shutil._UNPACK_FORMATS = saved[0] + shutil._UNPACK_FORMATS.clear() + shutil._UNPACK_FORMATS.update(saved[1]) + + def get_logging__handlers(self): + # _handlers is a WeakValueDictionary + return id(logging._handlers), logging._handlers, logging._handlers.copy() + def restore_logging__handlers(self, saved_handlers): + # Can't easily revert the logging state + pass + + def get_logging__handlerList(self): + # _handlerList is a list of weakrefs to handlers + return id(logging._handlerList), logging._handlerList, logging._handlerList[:] + def restore_logging__handlerList(self, saved_handlerList): + # Can't easily revert the logging state + pass + + def get_sys_warnoptions(self): + return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] + def restore_sys_warnoptions(self, saved_options): + sys.warnoptions = saved_options[1] + sys.warnoptions[:] = saved_options[2] + + # Controlling dangling references to Thread objects can make it easier + # to track reference leaks. + def get_threading__dangling(self): + if not threading: + return None + # This copies the weakrefs without making any strong reference + return threading._dangling.copy() + def restore_threading__dangling(self, saved): + if not threading: + return + threading._dangling.clear() + threading._dangling.update(saved) + + # Same for Process objects + def get_multiprocessing_process__dangling(self): + if not multiprocessing: + return None + # Unjoined process objects can survive after process exits + multiprocessing.process._cleanup() + # This copies the weakrefs without making any strong reference + return multiprocessing.process._dangling.copy() + def restore_multiprocessing_process__dangling(self, saved): + if not multiprocessing: + return + multiprocessing.process._dangling.clear() + multiprocessing.process._dangling.update(saved) + + def get_sysconfig__CONFIG_VARS(self): + # make sure the dict is initialized + sysconfig.get_config_var('prefix') + return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, + dict(sysconfig._CONFIG_VARS)) + def restore_sysconfig__CONFIG_VARS(self, saved): + sysconfig._CONFIG_VARS = saved[1] + sysconfig._CONFIG_VARS.clear() + sysconfig._CONFIG_VARS.update(saved[2]) + + def get_sysconfig__INSTALL_SCHEMES(self): + return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, + sysconfig._INSTALL_SCHEMES.copy()) + def restore_sysconfig__INSTALL_SCHEMES(self, saved): + sysconfig._INSTALL_SCHEMES = saved[1] + sysconfig._INSTALL_SCHEMES.clear() + sysconfig._INSTALL_SCHEMES.update(saved[2]) + + def get_files(self): + return sorted(fn + ('/' if os.path.isdir(fn) else '') + for fn in os.listdir()) + def restore_files(self, saved_value): + fn = support.TESTFN + if fn not in saved_value and (fn + '/') not in saved_value: + if os.path.isfile(fn): + support.unlink(fn) + elif os.path.isdir(fn): + support.rmtree(fn) + + _lc = [getattr(locale, lc) for lc in dir(locale) + if lc.startswith('LC_')] + def get_locale(self): + pairings = [] + for lc in self._lc: + try: + pairings.append((lc, locale.setlocale(lc, None))) + except (TypeError, ValueError): + continue + return pairings + def restore_locale(self, saved): + for lc, setting in saved: + locale.setlocale(lc, setting) + + def get_warnings_showwarning(self): + return warnings.showwarning + def restore_warnings_showwarning(self, fxn): + warnings.showwarning = fxn + + def resource_info(self): + for name in self.resources: + method_suffix = name.replace('.', '_') + get_name = 'get_' + method_suffix + restore_name = 'restore_' + method_suffix + yield name, getattr(self, get_name), getattr(self, restore_name) + + def __enter__(self): + self.saved_values = dict((name, get()) for name, get, restore + in self.resource_info()) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + saved_values = self.saved_values + del self.saved_values + support.gc_collect() # Some resources use weak references + for name, get, restore in self.resource_info(): + current = get() + original = saved_values.pop(name) + # Check for changes to the resource's value + if current != original: + self.changed = True + restore(original) + if not self.quiet and not self.pgo: + print(f"Warning -- {name} was modified by {self.testname}", + file=sys.stderr, flush=True) + if self.verbose > 1: + print(f" Before: {original}\n After: {current} ", + file=sys.stderr, flush=True) + return False diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py new file mode 100644 index 0000000000..1d24531fc2 --- /dev/null +++ b/Lib/test/libregrtest/setup.py @@ -0,0 +1,121 @@ +import atexit +import faulthandler +import os +import signal +import sys +import unittest +from test import support +try: + import gc +except ImportError: + gc = None + +from test.libregrtest.refleak import warm_caches + + +def setup_tests(ns): + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + + replace_stdout() + support.record_original_stdout(sys.stdout) + + if ns.testdir: + # Prepend test directory to sys.path, so runtest() will be able + # to locate tests + sys.path.insert(0, os.path.abspath(ns.testdir)) + + # Some times __path__ and __file__ are not absolute (e.g. while running from + # Lib/) and, if we change the CWD to run the tests in a temporary dir, some + # imports might fail. This affects only the modules imported before os.chdir(). + # These modules are searched first in sys.path[0] (so '' -- the CWD) and if + # they are found in the CWD their __file__ and __path__ will be relative (this + # happens before the chdir). All the modules imported after the chdir, are + # not found in the CWD, and since the other paths in sys.path[1:] are absolute + # (site.py absolutize them), the __file__ and __path__ will be absolute too. + # Therefore it is necessary to absolutize manually the __file__ and __path__ of + # the packages to prevent later imports to fail when the CWD is different. + for module in sys.modules.values(): + if hasattr(module, '__path__'): + for index, path in enumerate(module.__path__): + module.__path__[index] = os.path.abspath(path) + if hasattr(module, '__file__'): + module.__file__ = os.path.abspath(module.__file__) + + # MacOSX (a.k.a. Darwin) has a default stack size that is too small + # for deeply recursive regular expressions. We see this as crashes in + # the Python test suite when running test_re.py and test_sre.py. The + # fix is to set the stack limit to 2048. + # This approach may also be useful for other Unixy platforms that + # suffer from small default stack limits. + if sys.platform == 'darwin': + try: + import resource + except ImportError: + pass + else: + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + newsoft = min(hard, max(soft, 1024*2048)) + resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + + if ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() + + if ns.memlimit is not None: + support.set_memlimit(ns.memlimit) + + if ns.threshold is not None: + gc.set_threshold(ns.threshold) + + try: + import msvcrt + except ImportError: + pass + else: + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + if ns.verbose and ns.verbose >= 2: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + else: + msvcrt.CrtSetReportMode(m, 0) + + support.use_resources = ns.use_resources + + +def replace_stdout(): + """Set stdout encoder error handler to backslashreplace (as stderr error + handler) to avoid UnicodeEncodeError when printing a traceback""" + stdout = sys.stdout + sys.stdout = open(stdout.fileno(), 'w', + encoding=stdout.encoding, + errors="backslashreplace", + closefd=False, + newline='\n') + + def restore_stdout(): + sys.stdout.close() + sys.stdout = stdout + atexit.register(restore_stdout) diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index f20fdc0a5f..26e93687f4 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -266,9 +266,21 @@ class CommonTest(seq_tests.CommonTest): self.assertEqual(a, list("spameggs")) self.assertRaises(TypeError, a.extend, None) - self.assertRaises(TypeError, a.extend) + # overflow test. issue1621 + class CustomIter: + def __iter__(self): + return self + def __next__(self): + raise StopIteration + def __length_hint__(self): + return sys.maxsize + a = self.type2test([1,2,3,4]) + a.extend(CustomIter()) + self.assertEqual(a, [1,2,3,4]) + + def test_insert(self): a = self.type2test([0, 1, 2]) a.insert(0, -2) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index a64aa18cd3..a6cb3b169b 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -7,6 +7,7 @@ import time from _thread import start_new_thread, TIMEOUT_MAX import threading import unittest +import weakref from test import support @@ -198,6 +199,17 @@ class BaseLockTests(BaseTestCase): self.assertFalse(results[0]) self.assertTimeout(results[1], 0.5) + def test_weakref_exists(self): + lock = self.locktype() + ref = weakref.ref(lock) + self.assertIsNotNone(ref()) + + def test_weakref_deleted(self): + lock = self.locktype() + ref = weakref.ref(lock) + del lock + self.assertIsNone(ref()) + class LockTests(BaseLockTests): """ diff --git a/Lib/test/make_ssl_certs.py b/Lib/test/make_ssl_certs.py index a1f298de34..4d9f01ba6a 100644 --- a/Lib/test/make_ssl_certs.py +++ b/Lib/test/make_ssl_certs.py @@ -3,7 +3,6 @@ and friends.""" import os import shutil -import sys import tempfile from subprocess import * diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 7922b54f03..5c83361027 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1000,7 +1000,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'0', # POP b'1', # POP_MARK b'2', # DUP - # b'(2', # PyUnpickler doesn't raise + b'(2', b'R', # REDUCE b')R', b'a', # APPEND @@ -1009,7 +1009,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'Nb', b'd', # DICT b'e', # APPENDS - # b'(e', # PyUnpickler raises AttributeError + b'(e', b'ibuiltins\nlist\n', # INST b'l', # LIST b'o', # OBJ @@ -1022,7 +1022,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'NNs', b't', # TUPLE b'u', # SETITEMS - # b'(u', # PyUnpickler doesn't raise + b'(u', b'}(Nu', b'\x81', # NEWOBJ b')\x81', @@ -1033,7 +1033,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'N\x87', b'NN\x87', b'\x90', # ADDITEMS - # b'(\x90', # PyUnpickler raises AttributeError + b'(\x90', b'\x91', # FROZENSET b'\x92', # NEWOBJ_EX b')}\x92', @@ -1046,7 +1046,7 @@ class AbstractUnpickleTests(unittest.TestCase): def test_bad_mark(self): badpickles = [ - # b'N(.', # STOP + b'N(.', # STOP b'N(2', # DUP b'cbuiltins\nlist\n)(R', # REDUCE b'cbuiltins\nlist\n()R', @@ -1081,7 +1081,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'N(\x94', # MEMOIZE ] for p in badpickles: - self.check_unpickling_error(self.bad_mark_errors, p) + self.check_unpickling_error(self.bad_stack_errors, p) def test_truncated_data(self): self.check_unpickling_error(EOFError, b'') @@ -1855,16 +1855,14 @@ class AbstractPickleTests(unittest.TestCase): x.abc = 666 for proto in protocols: with self.subTest(proto=proto): - if 2 <= proto < 4: - self.assertRaises(ValueError, self.dumps, x, proto) - continue s = self.dumps(x, proto) if proto < 1: self.assertIn(b'\nL64206', s) # LONG elif proto < 2: self.assertIn(b'M\xce\xfa', s) # BININT2 + elif proto < 4: + self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE else: - assert proto >= 4 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE self.assertFalse(opcode_in_pickle(pickle.NEWOBJ, s)) self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s), @@ -2583,11 +2581,6 @@ class AbstractPickleModuleTests(unittest.TestCase): self.assertRaises(pickle.PicklingError, BadPickler().dump, 0) self.assertRaises(pickle.UnpicklingError, BadUnpickler().load) - def test_bad_input(self): - # Test issue4298 - s = bytes([0x58, 0, 0, 0, 0x54]) - self.assertRaises(EOFError, pickle.loads, s) - class AbstractPersistentPicklerTests(unittest.TestCase): diff --git a/Lib/test/pydoc_mod.py b/Lib/test/pydoc_mod.py index cda1c9e231..9c1fff5c2f 100644 --- a/Lib/test/pydoc_mod.py +++ b/Lib/test/pydoc_mod.py @@ -12,7 +12,7 @@ class A: pass class B(object): - NO_MEANING = "eggs" + NO_MEANING: str = "eggs" pass class C(object): diff --git a/Lib/test/re_tests.py b/Lib/test/re_tests.py index 8c158f883b..a379d33aec 100755 --- a/Lib/test/re_tests.py +++ b/Lib/test/re_tests.py @@ -106,8 +106,8 @@ tests = [ ('a.*b', 'acc\nccb', FAIL), ('a.{4,5}b', 'acc\nccb', FAIL), ('a.b', 'a\rb', SUCCEED, 'found', 'a\rb'), - ('a.b(?s)', 'a\nb', SUCCEED, 'found', 'a\nb'), - ('a.*(?s)b', 'acc\nccb', SUCCEED, 'found', 'acc\nccb'), + ('(?s)a.b', 'a\nb', SUCCEED, 'found', 'a\nb'), + ('(?s)a.*b', 'acc\nccb', SUCCEED, 'found', 'acc\nccb'), ('(?s)a.{4,5}b', 'acc\nccb', SUCCEED, 'found', 'acc\nccb'), ('(?s)a.b', 'a\nb', SUCCEED, 'found', 'a\nb'), @@ -158,7 +158,7 @@ tests = [ ('(abc', '-', SYNTAX_ERROR), ('a]', 'a]', SUCCEED, 'found', 'a]'), ('a[]]b', 'a]b', SUCCEED, 'found', 'a]b'), - ('a[\]]b', 'a]b', SUCCEED, 'found', 'a]b'), + ('a[\\]]b', 'a]b', SUCCEED, 'found', 'a]b'), ('a[^bc]d', 'aed', SUCCEED, 'found', 'aed'), ('a[^bc]d', 'abd', FAIL), ('a[^-b]c', 'adc', SUCCEED, 'found', 'adc'), @@ -551,7 +551,7 @@ tests = [ # lookbehind: split by : but not if it is escaped by -. ('(?<!-):(.*?)(?<!-):', 'a:bc-:de:f', SUCCEED, 'g1', 'bc-:de' ), # escaping with \ as we know it - ('(?<!\\\):(.*?)(?<!\\\):', 'a:bc\\:de:f', SUCCEED, 'g1', 'bc\\:de' ), + ('(?<!\\\\):(.*?)(?<!\\\\):', 'a:bc\\:de:f', SUCCEED, 'g1', 'bc\\:de' ), # terminating with ' and escaping with ? as in edifact ("(?<!\\?)'(.*?)(?<!\\?)'", "a'bc?'de'f", SUCCEED, 'g1', "bc?'de" ), @@ -563,7 +563,7 @@ tests = [ # Check odd placement of embedded pattern modifiers # not an error under PCRE/PRE: - ('w(?i)', 'W', SUCCEED, 'found', 'W'), + ('(?i)w', 'W', SUCCEED, 'found', 'W'), # ('w(?i)', 'W', SYNTAX_ERROR), # Comments using the x embedded pattern modifier @@ -627,7 +627,7 @@ xyzabc # bug 114033: nothing to repeat (r'(x?)?', 'x', SUCCEED, 'found', 'x'), # bug 115040: rescan if flags are modified inside pattern - (r' (?x)foo ', 'foo', SUCCEED, 'found', 'foo'), + (r'(?x) foo ', 'foo', SUCCEED, 'found', 'foo'), # bug 115618: negative lookahead (r'(?<!abc)(d.f)', 'abcdefdof', SUCCEED, 'found', 'dof'), # bug 116251: character class bug diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index c1d85f6a18..21b0edfd07 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -6,1670 +6,33 @@ Script to run Python regression tests. Run this script with -h or --help for documentation. """ -USAGE = """\ -python -m test [options] [test_name1 [test_name2 ...]] -python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] -""" - -DESCRIPTION = """\ -Run Python regression tests. - -If no arguments or options are provided, finds all files matching -the pattern "test_*" in the Lib/test subdirectory and runs -them in alphabetical order (but see -M and -u, below, for exceptions). - -For more rigorous testing, it is useful to use the following -command line: - -python -E -Wd -m test [options] [test_name1 ...] -""" - -EPILOG = """\ -Additional option details: - --r randomizes test execution order. You can use --randseed=int to provide an -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. - --s On the first invocation of regrtest using -s, the first test file found -or the first test file given on the command line is run, and the name of -the next test is recorded in a file named pynexttest. If run from the -Python build directory, pynexttest is located in the 'build' subdirectory, -otherwise it is located in tempfile.gettempdir(). On subsequent runs, -the test in pynexttest is run, and the next test is written to pynexttest. -When the last test has been run, pynexttest is deleted. In this way it -is possible to single step through the test files. This is useful when -doing memory analysis on the Python interpreter, which process tends to -consume too many resources to run the full regression test non-stop. - --S is used to continue running tests after an aborted run. It will -maintain the order a standard run (ie, this assumes -r is not used). -This is useful after the tests have prematurely stopped for some external -reason and you want to start running from where you left off rather -than starting from the beginning. - --f reads the names of tests from the file given as f's argument, one -or more test names per line. Whitespace is ignored. Blank lines and -lines beginning with '#' are ignored. This is especially useful for -whittling down failures involving interactions among tests. - --L causes the leaks(1) command to be run just before exit if it exists. -leaks(1) is available on Mac OS X and presumably on some other -FreeBSD-derived systems. - --R runs each test several times and examines sys.gettotalrefcount() to -see if the test appears to be leaking references. The argument should -be of the form stab:run:fname where 'stab' is the number of times the -test is run to let gettotalrefcount settle down, 'run' is the number -of times further it is run and 'fname' is the name of the file the -reports are written to. These parameters all have defaults (5, 4 and -"reflog.txt" respectively), and the minimal invocation is '-R :'. - --M runs tests that require an exorbitant amount of memory. These tests -typically try to ascertain containers keep working when containing more than -2 billion objects, which only works on 64-bit systems. There are also some -tests that try to exhaust the address space of the process, which only makes -sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, -which is a string in the form of '2.5Gb', determines howmuch memory the -tests will limit themselves to (but they may go slightly over.) The number -shouldn't be more memory than the machine has (including swap memory). You -should also keep in mind that swap memory is generally much, much slower -than RAM, and setting memlimit to all available RAM or higher will heavily -tax the machine. On the other hand, it is no use running these tests with a -limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect -to use more than memlimit memory will be skipped. The big-memory tests -generally run very, very long. - --u is used to specify which special resource intensive tests to run, -such as those requiring large file support or network connectivity. -The argument is a comma-separated list of words indicating the -resources to test. Currently only the following are defined: - - all - Enable all special resources. - - none - Disable all special resources (this is the default). - - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) - - curses - Tests that use curses and will modify the terminal's - state and output modes. - - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2GB of disk space temporarily. - - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. - - decimal - Test the decimal module against a large suite that - verifies compliance with standards. - - cpu - Used for certain CPU-heavy tests. - - subprocess Run all tests for the subprocess module. - - urlfetch - It is okay to download files required on testing. - - gui - Run tests that require a running GUI. - -To enable all resources except one, use '-uall,-<resource>'. For -example, to run all the tests except for the gui tests, give the -option '-uall,-gui'. -""" - # We import importlib *ASAP* in order to test #15386 import importlib -import argparse -import builtins -import faulthandler -import io -import json -import locale -import logging import os -import platform -import random -import re -import shutil -import signal import sys -import sysconfig -import tempfile -import time -import traceback -import unittest -import warnings -from inspect import isabstract - -try: - import threading -except ImportError: - threading = None -try: - import _multiprocessing, multiprocessing.process -except ImportError: - multiprocessing = None - - -# Some times __path__ and __file__ are not absolute (e.g. while running from -# Lib/) and, if we change the CWD to run the tests in a temporary dir, some -# imports might fail. This affects only the modules imported before os.chdir(). -# These modules are searched first in sys.path[0] (so '' -- the CWD) and if -# they are found in the CWD their __file__ and __path__ will be relative (this -# happens before the chdir). All the modules imported after the chdir, are -# not found in the CWD, and since the other paths in sys.path[1:] are absolute -# (site.py absolutize them), the __file__ and __path__ will be absolute too. -# Therefore it is necessary to absolutize manually the __file__ and __path__ of -# the packages to prevent later imports to fail when the CWD is different. -for module in sys.modules.values(): - if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) for path in module.__path__] - if hasattr(module, '__file__'): - module.__file__ = os.path.abspath(module.__file__) - - -# MacOSX (a.k.a. Darwin) has a default stack size that is too small -# for deeply recursive regular expressions. We see this as crashes in -# the Python test suite when running test_re.py and test_sre.py. The -# fix is to set the stack limit to 2048. -# This approach may also be useful for other Unixy platforms that -# suffer from small default stack limits. -if sys.platform == 'darwin': - try: - import resource - except ImportError: - pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) - -# Test result constants. -PASSED = 1 -FAILED = 0 -ENV_CHANGED = -1 -SKIPPED = -2 -RESOURCE_DENIED = -3 -INTERRUPTED = -4 -CHILD_ERROR = -5 # error in a child process - -from test import support - -RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') - -# When tests are run from the Python build directory, it is best practice -# to keep the test files in a subfolder. This eases the cleanup of leftover -# files using the "make distclean" command. -if sysconfig.is_python_build(): - TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') -else: - TEMPDIR = tempfile.gettempdir() -TEMPDIR = os.path.abspath(TEMPDIR) - -class _ArgParser(argparse.ArgumentParser): - - def error(self, message): - super().error(message + "\nPass -h or --help for complete help.") - -def _create_parser(): - # Set prog to prevent the uninformative "__main__.py" from displaying in - # error messages when using "python -m test ...". - parser = _ArgParser(prog='regrtest.py', - usage=USAGE, - description=DESCRIPTION, - epilog=EPILOG, - add_help=False, - formatter_class=argparse.RawDescriptionHelpFormatter) - - # Arguments with this clause added to its help are described further in - # the epilog's "Additional option details" section. - more_details = ' See the section at bottom for more details.' - - group = parser.add_argument_group('General options') - # We add help explicitly to control what argument group it renders under. - group.add_argument('-h', '--help', action='help', - help='show this help message and exit') - group.add_argument('--timeout', metavar='TIMEOUT', type=float, - help='dump the traceback and exit if a test takes ' - 'more than TIMEOUT seconds; disabled if TIMEOUT ' - 'is negative or equals to zero') - group.add_argument('--wait', action='store_true', - help='wait for user input, e.g., allow a debugger ' - 'to be attached') - group.add_argument('--slaveargs', metavar='ARGS') - group.add_argument('-S', '--start', metavar='START', - help='the name of the test at which to start.' + - more_details) - - group = parser.add_argument_group('Verbosity') - group.add_argument('-v', '--verbose', action='count', - help='run tests in verbose mode with output to stdout') - group.add_argument('-w', '--verbose2', action='store_true', - help='re-run failed tests in verbose mode') - group.add_argument('-W', '--verbose3', action='store_true', - help='display test output on failure') - group.add_argument('-q', '--quiet', action='store_true', - help='no output unless one or more tests fail') - group.add_argument('-o', '--slow', action='store_true', dest='print_slow', - help='print the slowest 10 tests') - group.add_argument('--header', action='store_true', - help='print header with interpreter info') - - group = parser.add_argument_group('Selecting tests') - group.add_argument('-r', '--randomize', action='store_true', - help='randomize test execution order.' + more_details) - group.add_argument('--randseed', metavar='SEED', - dest='random_seed', type=int, - help='pass a random seed to reproduce a previous ' - 'random run') - group.add_argument('-f', '--fromfile', metavar='FILE', - help='read names of tests to run from a file.' + - more_details) - group.add_argument('-x', '--exclude', action='store_true', - help='arguments are tests to *exclude*') - group.add_argument('-s', '--single', action='store_true', - help='single step through a set of tests.' + - more_details) - group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', - help='match test cases and methods with glob pattern PAT') - group.add_argument('-G', '--failfast', action='store_true', - help='fail as soon as a test fails (only with -v or -W)') - group.add_argument('-u', '--use', metavar='RES1,RES2,...', - action='append', type=resources_list, - help='specify which special resource intensive tests ' - 'to run.' + more_details) - group.add_argument('-M', '--memlimit', metavar='LIMIT', - help='run very large memory-consuming tests.' + - more_details) - group.add_argument('--testdir', metavar='DIR', - type=relative_filename, - help='execute test files in the specified directory ' - '(instead of the Python stdlib test suite)') - - group = parser.add_argument_group('Special runs') - group.add_argument('-l', '--findleaks', action='store_true', - help='if GC is available detect tests that leak memory') - group.add_argument('-L', '--runleaks', action='store_true', - help='run the leaks(1) command just before exit.' + - more_details) - group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', - type=huntrleaks, - help='search for reference leaks (needs debug build, ' - 'very slow).' + more_details) - group.add_argument('-j', '--multiprocess', metavar='PROCESSES', - dest='use_mp', type=int, - help='run PROCESSES processes at once') - group.add_argument('-T', '--coverage', action='store_true', - dest='trace', - help='turn on code coverage tracing using the trace ' - 'module') - group.add_argument('-D', '--coverdir', metavar='DIR', - type=relative_filename, - help='directory where coverage files are put') - group.add_argument('-N', '--nocoverdir', - action='store_const', const=None, dest='coverdir', - help='put coverage files alongside modules') - group.add_argument('-t', '--threshold', metavar='THRESHOLD', - type=int, - help='call gc.set_threshold(THRESHOLD)') - group.add_argument('-n', '--nowindows', action='store_true', - help='suppress error message boxes on Windows') - group.add_argument('-F', '--forever', action='store_true', - help='run the specified tests in a loop, until an ' - 'error happens') - group.add_argument('-P', '--pgo', dest='pgo', action='store_true', - help='enable Profile Guided Optimization training') - - return parser - -def relative_filename(string): - # CWD is replaced with a temporary dir before calling main(), so we - # join it with the saved CWD so it ends up where the user expects. - return os.path.join(support.SAVEDCWD, string) - -def huntrleaks(string): - args = string.split(':') - if len(args) not in (2, 3): - raise argparse.ArgumentTypeError( - 'needs 2 or 3 colon-separated arguments') - nwarmup = int(args[0]) if args[0] else 5 - ntracked = int(args[1]) if args[1] else 4 - fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' - return nwarmup, ntracked, fname - -def resources_list(string): - u = [x.lower() for x in string.split(',')] - for r in u: - if r == 'all' or r == 'none': - continue - if r[0] == '-': - r = r[1:] - if r not in RESOURCE_NAMES: - raise argparse.ArgumentTypeError('invalid resource: ' + r) - return u - -def _parse_args(args, **kwargs): - # Defaults - ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, - exclude=False, single=False, randomize=False, fromfile=None, - findleaks=False, use_resources=None, trace=False, coverdir='coverage', - runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, - random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None, pgo=False) - for k, v in kwargs.items(): - if not hasattr(ns, k): - raise TypeError('%r is an invalid keyword argument ' - 'for this function' % k) - setattr(ns, k, v) - if ns.use_resources is None: - ns.use_resources = [] - - parser = _create_parser() - # Issue #14191: argparse doesn't support "intermixed" positional and - # optional arguments. Use parse_known_args() as workaround. - ns.args = parser.parse_known_args(args=args, namespace=ns)[1] - for arg in ns.args: - if arg.startswith('-'): - parser.error("unrecognized arguments: %s" % arg) - sys.exit(1) - - if ns.single and ns.fromfile: - parser.error("-s and -f don't go together!") - if ns.use_mp is not None and ns.trace: - parser.error("-T and -j don't go together!") - if ns.use_mp is not None and ns.findleaks: - parser.error("-l and -j don't go together!") - if ns.use_mp is not None and ns.memlimit: - parser.error("-M and -j don't go together!") - if ns.failfast and not (ns.verbose or ns.verbose3): - parser.error("-G/--failfast needs either -v or -W") - - if ns.quiet: - ns.verbose = 0 - if ns.timeout is not None: - if hasattr(faulthandler, 'dump_traceback_later'): - if ns.timeout <= 0: - ns.timeout = None - else: - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later") - ns.timeout = None - if ns.use_mp is not None: - if ns.use_mp <= 0: - # Use all cores + extras for tests that like to sleep - ns.use_mp = 2 + (os.cpu_count() or 1) - if ns.use_mp == 1: - ns.use_mp = None - if ns.use: - for a in ns.use: - for r in a: - if r == 'all': - ns.use_resources[:] = RESOURCE_NAMES - continue - if r == 'none': - del ns.use_resources[:] - continue - remove = False - if r[0] == '-': - remove = True - r = r[1:] - if remove: - if r in ns.use_resources: - ns.use_resources.remove(r) - elif r not in ns.use_resources: - ns.use_resources.append(r) - if ns.random_seed is not None: - ns.randomize = True - - return ns - - -def run_test_in_subprocess(testname, ns): - """Run the given test in a subprocess with --slaveargs. - - ns is the option Namespace parsed from command-line arguments. regrtest - is invoked in a subprocess with the --slaveargs argument; when the - subprocess exits, its return code, stdout and stderr are returned as a - 3-tuple. - """ - from subprocess import Popen, PIPE - base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + - ['-X', 'faulthandler', '-m', 'test.regrtest']) - # required to spawn a new process with PGO flag on/off - if ns.pgo: - base_cmd = base_cmd + ['--pgo'] - slaveargs = ( - (testname, ns.verbose, ns.quiet), - dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests, pgo=ns.pgo)) - # Running the child from the same working directory as regrtest's original - # invocation ensures that TEMPDIR for the child is the same when - # sysconfig.is_python_build() is true. See issue 15300. - popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], - stdout=PIPE, stderr=PIPE, - universal_newlines=True, - close_fds=(os.name != 'nt'), - cwd=support.SAVEDCWD) - stdout, stderr = popen.communicate() - retcode = popen.wait() - return retcode, stdout, stderr - - -def main(tests=None, **kwargs): - """Execute a test suite. - - This also parses command-line options and modifies its behavior - accordingly. - - tests -- a list of strings containing test names (optional) - testdir -- the directory in which to look for tests (optional) - - Users other than the Python test suite will certainly want to - specify testdir; if it's omitted, the directory containing the - Python test suite is searched for. - - If the tests argument is omitted, the tests listed on the - command-line will be used. If that's empty, too, then all *.py - files beginning with test_ will be used. - - The other default arguments (verbose, quiet, exclude, - single, randomize, findleaks, use_resources, trace, coverdir, - print_slow, and random_seed) allow programmers calling main() - directly to set the values that would normally be set by flags - on the command line. - """ - # Display the Python traceback on fatal errors (e.g. segfault) - faulthandler.enable(all_threads=True) - - # Display the Python traceback on SIGALRM or SIGUSR1 signal - signals = [] - if hasattr(signal, 'SIGALRM'): - signals.append(signal.SIGALRM) - if hasattr(signal, 'SIGUSR1'): - signals.append(signal.SIGUSR1) - for signum in signals: - faulthandler.register(signum, chain=True) - - replace_stdout() - - support.record_original_stdout(sys.stdout) - - ns = _parse_args(sys.argv[1:], **kwargs) - - if ns.huntrleaks: - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) - if ns.threshold is not None: - import gc - gc.set_threshold(ns.threshold) - if ns.nowindows: - print('The --nowindows (-n) option is deprecated. ' - 'Use -vv to display assertions in stderr.') - try: - import msvcrt - except ImportError: - pass - else: - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - if ns.verbose and ns.verbose >= 2: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - else: - msvcrt.CrtSetReportMode(m, 0) - if ns.wait: - input("Press any key to continue...") - - if ns.slaveargs is not None: - args, kwargs = json.loads(ns.slaveargs) - if kwargs.get('huntrleaks'): - unittest.BaseTestSuite._cleanup = False - try: - result = runtest(*args, **kwargs) - except KeyboardInterrupt: - result = INTERRUPTED, '' - except BaseException as e: - traceback.print_exc() - result = CHILD_ERROR, str(e) - sys.stdout.flush() - print() # Force a newline (just in case) - print(json.dumps(result)) - sys.exit(0) - - good = [] - bad = [] - skipped = [] - resource_denieds = [] - environment_changed = [] - interrupted = False - - if ns.findleaks: - try: - import gc - except ImportError: - print('No GC available, disabling findleaks.') - ns.findleaks = False - else: - # Uncomment the line below to report garbage that is not - # freeable by reference counting alone. By default only - # garbage that is not collectable by the GC is reported. - #gc.set_debug(gc.DEBUG_SAVEALL) - found_garbage = [] - - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False - - if ns.single: - filename = os.path.join(TEMPDIR, 'pynexttest') - try: - with open(filename, 'r') as fp: - next_test = fp.read().strip() - tests = [next_test] - except OSError: - pass - - if ns.fromfile: - tests = [] - with open(os.path.join(support.SAVEDCWD, ns.fromfile)) as fp: - count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') - for line in fp: - line = count_pat.sub('', line) - guts = line.split() # assuming no test has whitespace in its name - if guts and not guts[0].startswith('#'): - tests.extend(guts) - - # Strip .py extensions. - removepy(ns.args) - removepy(tests) - - stdtests = STDTESTS[:] - nottests = NOTTESTS.copy() - if ns.exclude: - for arg in ns.args: - if arg in stdtests: - stdtests.remove(arg) - nottests.add(arg) - ns.args = [] - - # For a partial run, we do not need to clutter the output. - if (ns.verbose or ns.header or - not (ns.pgo or ns.quiet or ns.single or tests or ns.args)): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("== ", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== ", "hash algorithm:", sys.hash_info.algorithm, - "64bit" if sys.maxsize > 2**32 else "32bit") - print("== ", os.getcwd()) - print("Testing with flags:", sys.flags) - - # if testdir is set, then we are not running the python tests suite, so - # don't add default tests to be executed or skipped (pass empty values) - if ns.testdir: - alltests = findtests(ns.testdir, list(), set()) - else: - alltests = findtests(ns.testdir, stdtests, nottests) - - selected = tests or ns.args or alltests - if ns.single: - selected = selected[:1] - try: - next_single_test = alltests[alltests.index(selected[0])+1] - except IndexError: - next_single_test = None - # Remove all the selected tests that precede start if it's set. - if ns.start: - try: - del selected[:selected.index(ns.start)] - except ValueError: - print("Couldn't find starting test (%s), using all tests" % ns.start) - if ns.randomize: - if ns.random_seed is None: - ns.random_seed = random.randrange(10000000) - random.seed(ns.random_seed) - print("Using random seed", ns.random_seed) - random.shuffle(selected) - if ns.trace: - import trace, tempfile - tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) - - test_times = [] - support.verbose = ns.verbose # Tell tests to be moderately quiet - support.use_resources = ns.use_resources - save_modules = sys.modules.keys() - - def accumulate_result(test, result): - ok, test_time = result - if ok not in (CHILD_ERROR, INTERRUPTED): - test_times.append((test_time, test)) - if ok == PASSED: - good.append(test) - elif ok == FAILED: - bad.append(test) - elif ok == ENV_CHANGED: - environment_changed.append(test) - elif ok == SKIPPED: - skipped.append(test) - elif ok == RESOURCE_DENIED: - skipped.append(test) - resource_denieds.append(test) - - if ns.forever: - def test_forever(tests=list(selected)): - while True: - for test in tests: - yield test - if bad: - return - tests = test_forever() - test_count = '' - test_count_width = 3 - else: - tests = iter(selected) - test_count = '/{}'.format(len(selected)) - test_count_width = len(test_count) - 1 - - if ns.use_mp: - try: - from threading import Thread - except ImportError: - print("Multiprocess option requires thread support") - sys.exit(2) - from queue import Queue - debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") - output = Queue() - pending = MultiprocessTests(tests) - def work(): - # A worker thread. - try: - while True: - try: - test = next(pending) - except StopIteration: - output.put((None, None, None, None)) - return - retcode, stdout, stderr = run_test_in_subprocess(test, ns) - # Strip last refcount output line if it exists, since it - # comes from the shutdown of the interpreter in the subcommand. - stderr = debug_output_pat.sub("", stderr) - stdout, _, result = stdout.strip().rpartition("\n") - if retcode != 0: - result = (CHILD_ERROR, "Exit code %s" % retcode) - output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - return - if not result: - output.put((None, None, None, None)) - return - result = json.loads(result) - output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - except BaseException: - output.put((None, None, None, None)) - raise - workers = [Thread(target=work) for i in range(ns.use_mp)] - for worker in workers: - worker.start() - finished = 0 - test_index = 1 - try: - while finished < ns.use_mp: - test, stdout, stderr, result = output.get() - if test is None: - finished += 1 - continue - accumulate_result(test, result) - if not ns.quiet: - if bad and not ns.pgo: - fmt = "[{1:{0}}{2}/{3}] {4}" - else: - fmt = "[{1:{0}}{2}] {4}" - print(fmt.format( - test_count_width, test_index, test_count, - len(bad), test)) - if stdout: - print(stdout) - if stderr and not ns.pgo: - print(stderr, file=sys.stderr) - sys.stdout.flush() - sys.stderr.flush() - if result[0] == INTERRUPTED: - raise KeyboardInterrupt - if result[0] == CHILD_ERROR: - raise Exception("Child error on {}: {}".format(test, result[1])) - test_index += 1 - except KeyboardInterrupt: - interrupted = True - pending.interrupted = True - for worker in workers: - worker.join() - else: - for test_index, test in enumerate(tests, 1): - if not ns.quiet: - if bad and not ns.pgo: - fmt = "[{1:{0}}{2}/{3}] {4}" - else: - fmt = "[{1:{0}}{2}] {4}" - print(fmt.format( - test_count_width, test_index, test_count, len(bad), test)) - sys.stdout.flush() - if ns.trace: - # If we're tracing code coverage, then we don't exit with status - # if on a false return value from main. - tracer.runctx('runtest(test, ns.verbose, ns.quiet, timeout=ns.timeout)', - globals=globals(), locals=vars()) - else: - try: - result = runtest(test, ns.verbose, ns.quiet, - ns.huntrleaks, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests, pgo=ns.pgo) - accumulate_result(test, result) - except KeyboardInterrupt: - interrupted = True - break - if ns.findleaks: - gc.collect() - if gc.garbage: - print("Warning: test created", len(gc.garbage), end=' ') - print("uncollectable object(s).") - # move the uncollectable objects somewhere so we don't see - # them again - found_garbage.extend(gc.garbage) - del gc.garbage[:] - # Unload the newly imported modules (best effort finalization) - for module in sys.modules.keys(): - if module not in save_modules and module.startswith("test."): - support.unload(module) - - if interrupted and not ns.pgo: - # print a newline after ^C - print() - print("Test suite interrupted by signal SIGINT.") - omitted = set(selected) - set(good) - set(bad) - set(skipped) - print(count(len(omitted), "test"), "omitted:") - printlist(omitted) - if good and not ns.quiet and not ns.pgo: - if not bad and not skipped and not interrupted and len(good) > 1: - print("All", end=' ') - print(count(len(good), "test"), "OK.") - if ns.print_slow: - test_times.sort(reverse=True) - print("10 slowest tests:") - for time, test in test_times[:10]: - print("%s: %.1fs" % (test, time)) - if bad and not ns.pgo: - print(count(len(bad), "test"), "failed:") - printlist(bad) - if environment_changed and not ns.pgo: - print("{} altered the execution environment:".format( - count(len(environment_changed), "test"))) - printlist(environment_changed) - if skipped and not ns.quiet and not ns.pgo: - print(count(len(skipped), "test"), "skipped:") - printlist(skipped) - - if ns.verbose2 and bad: - print("Re-running failed tests in verbose mode") - for test in bad[:]: - if not ns.pgo: - print("Re-running test %r in verbose mode" % test) - sys.stdout.flush() - try: - ns.verbose = True - ok = runtest(test, True, ns.quiet, ns.huntrleaks, - timeout=ns.timeout, pgo=ns.pgo) - except KeyboardInterrupt: - # print a newline separate from the ^C - print() - break - else: - if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: - bad.remove(test) - else: - if bad: - print(count(len(bad), 'test'), "failed again:") - printlist(bad) - - if ns.single: - if next_single_test: - with open(filename, 'w') as fp: - fp.write(next_single_test + '\n') - else: - os.unlink(filename) - - if ns.trace: - r = tracer.results() - r.write_results(show_missing=True, summary=True, coverdir=ns.coverdir) - - if ns.runleaks: - os.system("leaks %d" % os.getpid()) - - sys.exit(len(bad) > 0 or interrupted) - - -# small set of tests to determine if we have a basically functioning interpreter -# (i.e. if any of these fail, then anything else is likely to follow) -STDTESTS = [ - 'test_grammar', - 'test_opcodes', - 'test_dict', - 'test_builtin', - 'test_exceptions', - 'test_types', - 'test_unittest', - 'test_doctest', - 'test_doctest2', - 'test_support' -] - -# set of tests that we don't want to be executed when using regrtest -NOTTESTS = set() - -def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): - """Return a list of all applicable test modules.""" - testdir = findtestdir(testdir) - names = os.listdir(testdir) - tests = [] - others = set(stdtests) | nottests - for name in names: - mod, ext = os.path.splitext(name) - if mod[:5] == "test_" and ext in (".py", "") and mod not in others: - tests.append(mod) - return stdtests + sorted(tests) +from test.libregrtest import main -# We do not use a generator so multiple threads can call next(). -class MultiprocessTests(object): - """A thread-safe iterator over tests for multiprocess mode.""" +# Alias for backward compatibility (just in case) +main_in_temp_cwd = main - def __init__(self, tests): - self.interrupted = False - self.lock = threading.Lock() - self.tests = tests - def __iter__(self): - return self +def _main(): + global __file__ - def __next__(self): - with self.lock: - if self.interrupted: - raise StopIteration('tests interrupted') - return next(self.tests) - -def replace_stdout(): - """Set stdout encoder error handler to backslashreplace (as stderr error - handler) to avoid UnicodeEncodeError when printing a traceback""" - import atexit - - stdout = sys.stdout - sys.stdout = open(stdout.fileno(), 'w', - encoding=stdout.encoding, - errors="backslashreplace", - closefd=False, - newline='\n') - - def restore_stdout(): - sys.stdout.close() - sys.stdout = stdout - atexit.register(restore_stdout) - -def runtest(test, verbose, quiet, - huntrleaks=False, use_resources=None, - output_on_failure=False, failfast=False, match_tests=None, - timeout=None, *, pgo=False): - """Run a single test. - - test -- the name of the test - verbose -- if true, print more messages - quiet -- if true, don't print 'skipped' messages (probably redundant) - huntrleaks -- run multiple times to test for leaks; requires a debug - build; a triple corresponding to -R's three arguments - use_resources -- list of extra resources to use - output_on_failure -- if true, display test output on failure - timeout -- dump the traceback and exit if a test takes more than - timeout seconds - failfast, match_tests -- See regrtest command-line flags for these. - pgo -- if true, do not print unnecessary info when running the test - for Profile Guided Optimization build - - Returns the tuple result, test_time, where result is one of the constants: - INTERRUPTED KeyboardInterrupt when run under -j - RESOURCE_DENIED test skipped because resource denied - SKIPPED test skipped for some other reason - ENV_CHANGED test failed because it changed the execution environment - FAILED test failed - PASSED test passed - """ - if use_resources is not None: - support.use_resources = use_resources - use_timeout = (timeout is not None) - if use_timeout: - faulthandler.dump_traceback_later(timeout, exit=True) - try: - support.match_tests = match_tests - if failfast: - support.failfast = True - if output_on_failure: - support.verbose = True - - # Reuse the same instance to all calls to runtest(). Some - # tests keep a reference to sys.stdout or sys.stderr - # (eg. test_argparse). - if runtest.stringio is None: - stream = io.StringIO() - runtest.stringio = stream - else: - stream = runtest.stringio - stream.seek(0) - stream.truncate() - - orig_stdout = sys.stdout - orig_stderr = sys.stderr - try: - sys.stdout = stream - sys.stderr = stream - result = runtest_inner(test, verbose, quiet, huntrleaks, - display_failure=False, pgo=pgo) - if result[0] != PASSED and not pgo: - output = stream.getvalue() - orig_stderr.write(output) - orig_stderr.flush() - finally: - sys.stdout = orig_stdout - sys.stderr = orig_stderr - else: - support.verbose = verbose # Tell tests to be moderately quiet - result = runtest_inner(test, verbose, quiet, huntrleaks, - display_failure=not verbose, pgo=pgo) - return result - finally: - if use_timeout: - faulthandler.cancel_dump_traceback_later() - cleanup_test_droppings(test, verbose) -runtest.stringio = None - -# Unit tests are supposed to leave the execution environment unchanged -# once they complete. But sometimes tests have bugs, especially when -# tests fail, and the changes to environment go on to mess up other -# tests. This can cause issues with buildbot stability, since tests -# are run in random order and so problems may appear to come and go. -# There are a few things we can save and restore to mitigate this, and -# the following context manager handles this task. - -class saved_test_environment: - """Save bits of the test environment and restore them at block exit. - - with saved_test_environment(testname, verbose, quiet): - #stuff - - Unless quiet is True, a warning is printed to stderr if any of - the saved items was changed by the test. The attribute 'changed' - is initially False, but is set to True if a change is detected. - - If verbose is more than 1, the before and after state of changed - items is also printed. - """ - - changed = False - - def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): - self.testname = testname - self.verbose = verbose - self.quiet = quiet - self.pgo = pgo - - # To add things to save and restore, add a name XXX to the resources list - # and add corresponding get_XXX/restore_XXX functions. get_XXX should - # return the value to be saved and compared against a second call to the - # get function when test execution completes. restore_XXX should accept - # the saved value and restore the resource using it. It will be called if - # and only if a change in the value is detected. - # - # Note: XXX will have any '.' replaced with '_' characters when determining - # the corresponding method names. - - resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', - 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', - 'asyncore.socket_map', - 'logging._handlers', 'logging._handlerList', 'sys.gettrace', - 'sys.warnoptions', - # multiprocessing.process._cleanup() may release ref - # to a thread, so check processes first. - 'multiprocessing.process._dangling', 'threading._dangling', - 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', - 'files', 'locale', 'warnings.showwarning', - 'shutil_archive_formats', 'shutil_unpack_formats', - ) - - def get_sys_argv(self): - return id(sys.argv), sys.argv, sys.argv[:] - def restore_sys_argv(self, saved_argv): - sys.argv = saved_argv[1] - sys.argv[:] = saved_argv[2] - - def get_cwd(self): - return os.getcwd() - def restore_cwd(self, saved_cwd): - os.chdir(saved_cwd) - - def get_sys_stdout(self): - return sys.stdout - def restore_sys_stdout(self, saved_stdout): - sys.stdout = saved_stdout - - def get_sys_stderr(self): - return sys.stderr - def restore_sys_stderr(self, saved_stderr): - sys.stderr = saved_stderr - - def get_sys_stdin(self): - return sys.stdin - def restore_sys_stdin(self, saved_stdin): - sys.stdin = saved_stdin - - def get_os_environ(self): - return id(os.environ), os.environ, dict(os.environ) - def restore_os_environ(self, saved_environ): - os.environ = saved_environ[1] - os.environ.clear() - os.environ.update(saved_environ[2]) - - def get_sys_path(self): - return id(sys.path), sys.path, sys.path[:] - def restore_sys_path(self, saved_path): - sys.path = saved_path[1] - sys.path[:] = saved_path[2] - - def get_sys_path_hooks(self): - return id(sys.path_hooks), sys.path_hooks, sys.path_hooks[:] - def restore_sys_path_hooks(self, saved_hooks): - sys.path_hooks = saved_hooks[1] - sys.path_hooks[:] = saved_hooks[2] - - def get_sys_gettrace(self): - return sys.gettrace() - def restore_sys_gettrace(self, trace_fxn): - sys.settrace(trace_fxn) - - def get___import__(self): - return builtins.__import__ - def restore___import__(self, import_): - builtins.__import__ = import_ - - def get_asyncore_socket_map(self): - asyncore = sys.modules.get('asyncore') - # XXX Making a copy keeps objects alive until __exit__ gets called. - return asyncore and asyncore.socket_map.copy() or {} - def restore_asyncore_socket_map(self, saved_map): - asyncore = sys.modules.get('asyncore') - if asyncore is not None: - asyncore.close_all(ignore_all=True) - asyncore.socket_map.update(saved_map) - - def get_shutil_archive_formats(self): - # we could call get_archives_formats() but that only returns the - # registry keys; we want to check the values too (the functions that - # are registered) - return shutil._ARCHIVE_FORMATS, shutil._ARCHIVE_FORMATS.copy() - def restore_shutil_archive_formats(self, saved): - shutil._ARCHIVE_FORMATS = saved[0] - shutil._ARCHIVE_FORMATS.clear() - shutil._ARCHIVE_FORMATS.update(saved[1]) - - def get_shutil_unpack_formats(self): - return shutil._UNPACK_FORMATS, shutil._UNPACK_FORMATS.copy() - def restore_shutil_unpack_formats(self, saved): - shutil._UNPACK_FORMATS = saved[0] - shutil._UNPACK_FORMATS.clear() - shutil._UNPACK_FORMATS.update(saved[1]) - - def get_logging__handlers(self): - # _handlers is a WeakValueDictionary - return id(logging._handlers), logging._handlers, logging._handlers.copy() - def restore_logging__handlers(self, saved_handlers): - # Can't easily revert the logging state - pass - - def get_logging__handlerList(self): - # _handlerList is a list of weakrefs to handlers - return id(logging._handlerList), logging._handlerList, logging._handlerList[:] - def restore_logging__handlerList(self, saved_handlerList): - # Can't easily revert the logging state - pass - - def get_sys_warnoptions(self): - return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] - def restore_sys_warnoptions(self, saved_options): - sys.warnoptions = saved_options[1] - sys.warnoptions[:] = saved_options[2] - - # Controlling dangling references to Thread objects can make it easier - # to track reference leaks. - def get_threading__dangling(self): - if not threading: - return None - # This copies the weakrefs without making any strong reference - return threading._dangling.copy() - def restore_threading__dangling(self, saved): - if not threading: - return - threading._dangling.clear() - threading._dangling.update(saved) - - # Same for Process objects - def get_multiprocessing_process__dangling(self): - if not multiprocessing: - return None - # Unjoined process objects can survive after process exits - multiprocessing.process._cleanup() - # This copies the weakrefs without making any strong reference - return multiprocessing.process._dangling.copy() - def restore_multiprocessing_process__dangling(self, saved): - if not multiprocessing: - return - multiprocessing.process._dangling.clear() - multiprocessing.process._dangling.update(saved) - - def get_sysconfig__CONFIG_VARS(self): - # make sure the dict is initialized - sysconfig.get_config_var('prefix') - return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, - dict(sysconfig._CONFIG_VARS)) - def restore_sysconfig__CONFIG_VARS(self, saved): - sysconfig._CONFIG_VARS = saved[1] - sysconfig._CONFIG_VARS.clear() - sysconfig._CONFIG_VARS.update(saved[2]) - - def get_sysconfig__INSTALL_SCHEMES(self): - return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, - sysconfig._INSTALL_SCHEMES.copy()) - def restore_sysconfig__INSTALL_SCHEMES(self, saved): - sysconfig._INSTALL_SCHEMES = saved[1] - sysconfig._INSTALL_SCHEMES.clear() - sysconfig._INSTALL_SCHEMES.update(saved[2]) - - def get_files(self): - return sorted(fn + ('/' if os.path.isdir(fn) else '') - for fn in os.listdir()) - def restore_files(self, saved_value): - fn = support.TESTFN - if fn not in saved_value and (fn + '/') not in saved_value: - if os.path.isfile(fn): - support.unlink(fn) - elif os.path.isdir(fn): - support.rmtree(fn) - - _lc = [getattr(locale, lc) for lc in dir(locale) - if lc.startswith('LC_')] - def get_locale(self): - pairings = [] - for lc in self._lc: - try: - pairings.append((lc, locale.setlocale(lc, None))) - except (TypeError, ValueError): - continue - return pairings - def restore_locale(self, saved): - for lc, setting in saved: - locale.setlocale(lc, setting) - - def get_warnings_showwarning(self): - return warnings.showwarning - def restore_warnings_showwarning(self, fxn): - warnings.showwarning = fxn - - def resource_info(self): - for name in self.resources: - method_suffix = name.replace('.', '_') - get_name = 'get_' + method_suffix - restore_name = 'restore_' + method_suffix - yield name, getattr(self, get_name), getattr(self, restore_name) - - def __enter__(self): - self.saved_values = dict((name, get()) for name, get, restore - in self.resource_info()) - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - saved_values = self.saved_values - del self.saved_values - support.gc_collect() # Some resources use weak references - for name, get, restore in self.resource_info(): - current = get() - original = saved_values.pop(name) - # Check for changes to the resource's value - if current != original: - self.changed = True - restore(original) - if not self.quiet and not self.pgo: - print("Warning -- {} was modified by {}".format( - name, self.testname), - file=sys.stderr) - if self.verbose > 1 and not self.pgo: - print(" Before: {}\n After: {} ".format( - original, current), - file=sys.stderr) - return False - - -def runtest_inner(test, verbose, quiet, - huntrleaks=False, display_failure=True, pgo=False): - support.unload(test) - - test_time = 0.0 - refleak = False # True if the test leaked references. - try: - if test.startswith('test.'): - abstest = test - else: - # Always import it from the test package - abstest = 'test.' + test - clear_caches() - with saved_test_environment(test, verbose, quiet, pgo=pgo) as environment: - start_time = time.time() - the_module = importlib.import_module(abstest) - # If the test has a test_main, that will run the appropriate - # tests. If not, use normal unittest test loading. - test_runner = getattr(the_module, "test_main", None) - if test_runner is None: - def test_runner(): - loader = unittest.TestLoader() - tests = loader.loadTestsFromModule(the_module) - for error in loader.errors: - print(error, file=sys.stderr) - if loader.errors: - raise Exception("errors while loading tests") - support.run_unittest(tests) - test_runner() - if huntrleaks: - refleak = dash_R(the_module, test, test_runner, huntrleaks) - test_time = time.time() - start_time - except support.ResourceDenied as msg: - if not quiet and not pgo: - print(test, "skipped --", msg) - sys.stdout.flush() - return RESOURCE_DENIED, test_time - except unittest.SkipTest as msg: - if not quiet and not pgo: - print(test, "skipped --", msg) - sys.stdout.flush() - return SKIPPED, test_time - except KeyboardInterrupt: - raise - except support.TestFailed as msg: - if not pgo: - if display_failure: - print("test", test, "failed --", msg, file=sys.stderr) - else: - print("test", test, "failed", file=sys.stderr) - sys.stderr.flush() - return FAILED, test_time - except: - msg = traceback.format_exc() - if not pgo: - print("test", test, "crashed --", msg, file=sys.stderr) - sys.stderr.flush() - return FAILED, test_time - else: - if refleak: - return FAILED, test_time - if environment.changed: - return ENV_CHANGED, test_time - return PASSED, test_time - -def cleanup_test_droppings(testname, verbose): - import shutil - import stat - import gc - - # First kill any dangling references to open files etc. - # This can also issue some ResourceWarnings which would otherwise get - # triggered during the following test run, and possibly produce failures. - gc.collect() - - # Try to clean up junk commonly left behind. While tests shouldn't leave - # any files or directories behind, when a test fails that can be tedious - # for it to arrange. The consequences can be especially nasty on Windows, - # since if a test leaves a file open, it cannot be deleted by name (while - # there's nothing we can do about that here either, we can display the - # name of the offending test, which is a real help). - for name in (support.TESTFN, - "db_home", - ): - if not os.path.exists(name): - continue - - if os.path.isdir(name): - kind, nuker = "directory", shutil.rmtree - elif os.path.isfile(name): - kind, nuker = "file", os.unlink - else: - raise SystemError("os.path says %r exists but is neither " - "directory nor file" % name) - - if verbose: - print("%r left behind %s %r" % (testname, kind, name)) - try: - # if we have chmod, fix possible permissions problems - # that might prevent cleanup - if (hasattr(os, 'chmod')): - os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - nuker(name) - except Exception as msg: - print(("%r left behind %s %r and it couldn't be " - "removed: %s" % (testname, kind, name, msg)), file=sys.stderr) - -def dash_R(the_module, test, indirect_test, huntrleaks): - """Run a test multiple times, looking for reference leaks. - - Returns: - False if the test didn't leak references; True if we detected refleaks. - """ - # This code is hackish and inelegant, but it seems to do the job. - import copyreg - import collections.abc - - if not hasattr(sys, 'gettotalrefcount'): - raise Exception("Tracking reference leaks requires a debug build " - "of Python") - - # Save current values for dash_R_cleanup() to restore. - fs = warnings.filters[:] - ps = copyreg.dispatch_table.copy() - pic = sys.path_importer_cache.copy() - try: - import zipimport - except ImportError: - zdc = None # Run unmodified on platforms without zipimport support - else: - zdc = zipimport._zip_directory_cache.copy() - abcs = {} - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue - for obj in abc.__subclasses__() + [abc]: - abcs[obj] = obj._abc_registry.copy() - - nwarmup, ntracked, fname = huntrleaks - fname = os.path.join(support.SAVEDCWD, fname) - repcount = nwarmup + ntracked - rc_deltas = [0] * repcount - alloc_deltas = [0] * repcount - - print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr) - sys.stderr.flush() - for i in range(repcount): - indirect_test() - alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) - sys.stderr.write('.') - sys.stderr.flush() - if i >= nwarmup: - rc_deltas[i] = rc_after - rc_before - alloc_deltas[i] = alloc_after - alloc_before - alloc_before, rc_before = alloc_after, rc_after - print(file=sys.stderr) - # These checkers return False on success, True on failure - def check_rc_deltas(deltas): - return any(deltas) - def check_alloc_deltas(deltas): - # At least 1/3rd of 0s - if 3 * deltas.count(0) < len(deltas): - return True - # Nothing else than 1s, 0s and -1s - if not set(deltas) <= {1,0,-1}: - return True - return False - failed = False - for deltas, item_name, checker in [ - (rc_deltas, 'references', check_rc_deltas), - (alloc_deltas, 'memory blocks', check_alloc_deltas)]: - if checker(deltas): - msg = '%s leaked %s %s, sum=%s' % ( - test, deltas[nwarmup:], item_name, sum(deltas)) - print(msg, file=sys.stderr) - sys.stderr.flush() - with open(fname, "a") as refrep: - print(msg, file=refrep) - refrep.flush() - failed = True - return failed - -def dash_R_cleanup(fs, ps, pic, zdc, abcs): - import gc, copyreg - import collections.abc - from weakref import WeakSet - - # Restore some original values. - warnings.filters[:] = fs - copyreg.dispatch_table.clear() - copyreg.dispatch_table.update(ps) - sys.path_importer_cache.clear() - sys.path_importer_cache.update(pic) - try: - import zipimport - except ImportError: - pass # Run unmodified on platforms without zipimport support - else: - zipimport._zip_directory_cache.clear() - zipimport._zip_directory_cache.update(zdc) - - # clear type cache - sys._clear_type_cache() - - # Clear ABC registries, restoring previously saved ABC registries. - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue - for obj in abc.__subclasses__() + [abc]: - obj._abc_registry = abcs.get(obj, WeakSet()).copy() - obj._abc_cache.clear() - obj._abc_negative_cache.clear() - - clear_caches() - - # Collect cyclic trash and read memory statistics immediately after. - func1 = sys.getallocatedblocks - func2 = sys.gettotalrefcount - gc.collect() - return func1(), func2() - -def clear_caches(): - import gc - - # Clear the warnings registry, so they can be displayed again - for mod in sys.modules.values(): - if hasattr(mod, '__warningregistry__'): - del mod.__warningregistry__ - - # Flush standard output, so that buffered data is sent to the OS and - # associated Python objects are reclaimed. - for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): - if stream is not None: - stream.flush() - - # Clear assorted module caches. - # Don't worry about resetting the cache if the module is not loaded - try: - distutils_dir_util = sys.modules['distutils.dir_util'] - except KeyError: - pass - else: - distutils_dir_util._path_created.clear() - - re.purge() - - try: - _strptime = sys.modules['_strptime'] - except KeyError: - pass - else: - _strptime._regex_cache.clear() - - try: - urllib_parse = sys.modules['urllib.parse'] - except KeyError: - pass - else: - urllib_parse.clear_cache() - - try: - urllib_request = sys.modules['urllib.request'] - except KeyError: - pass - else: - urllib_request.urlcleanup() - - try: - linecache = sys.modules['linecache'] - except KeyError: - pass - else: - linecache.clearcache() - - try: - mimetypes = sys.modules['mimetypes'] - except KeyError: - pass - else: - mimetypes._default_mime_types() - - try: - filecmp = sys.modules['filecmp'] - except KeyError: - pass - else: - filecmp._cache.clear() - - try: - struct = sys.modules['struct'] - except KeyError: - pass - else: - struct._clearcache() - - try: - doctest = sys.modules['doctest'] - except KeyError: - pass - else: - doctest.master = None - - try: - ctypes = sys.modules['ctypes'] - except KeyError: - pass - else: - ctypes._reset_cache() - - try: - typing = sys.modules['typing'] - except KeyError: - pass - else: - for f in typing._cleanups: - f() - - gc.collect() - -def warm_caches(): - # char cache - s = bytes(range(256)) - for i in range(256): - s[i:i+1] - # unicode cache - x = [chr(i) for i in range(256)] - # int cache - x = list(range(-5, 257)) - -def findtestdir(path=None): - return path or os.path.dirname(__file__) or os.curdir - -def removepy(names): - if not names: - return - for idx, name in enumerate(names): - basename, ext = os.path.splitext(name) - if ext == '.py': - names[idx] = basename - -def count(n, word): - if n == 1: - return "%d %s" % (n, word) - else: - return "%d %ss" % (n, word) - -def printlist(x, width=70, indent=4): - """Print the elements of iterable x to stdout. - - Optional arg width (default 70) is the maximum line length. - Optional arg indent (default 4) is the number of blanks with which to - begin each line. - """ - - from textwrap import fill - blanks = ' ' * indent - # Print the sorted list: 'x' may be a '--random' list or a set() - print(fill(' '.join(str(elt) for elt in sorted(x)), width, - initial_indent=blanks, subsequent_indent=blanks)) - - -def main_in_temp_cwd(): - """Run main() in a temporary working directory.""" - if sysconfig.is_python_build(): - try: - os.mkdir(TEMPDIR) - except FileExistsError: - pass - - # Define a writable temp dir that will be used as cwd while running - # the tests. The name of the dir includes the pid to allow parallel - # testing (see the -j option). - test_cwd = 'test_python_{}'.format(os.getpid()) - test_cwd = os.path.join(TEMPDIR, test_cwd) - - # Run the tests in a context manager that temporarily changes the CWD to a - # temporary and writable directory. If it's not possible to create or - # change the CWD, the original CWD will be used. The original CWD is - # available from support.SAVEDCWD. - with support.temp_cwd(test_cwd, quiet=True): - main() - - -if __name__ == '__main__': # Remove regrtest.py's own directory from the module search path. Despite # the elimination of implicit relative imports, this is still needed to # ensure that submodules of the test package do not inappropriately appear # as top-level modules even when people (or buildbots!) invoke regrtest.py # directly instead of using the -m switch mydir = os.path.abspath(os.path.normpath(os.path.dirname(sys.argv[0]))) - i = len(sys.path) + i = len(sys.path) - 1 while i >= 0: - i -= 1 if os.path.abspath(os.path.normpath(sys.path[i])) == mydir: del sys.path[i] + else: + i -= 1 # findtestdir() gets the dirname out of __file__, so we have to make it # absolute before changing the working directory. @@ -1680,4 +43,8 @@ if __name__ == '__main__': # sanity check assert __file__ == os.path.abspath(sys.argv[0]) - main_in_temp_cwd() + main() + + +if __name__ == '__main__': + _main() diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py index 72f4845a97..1e7a6f6522 100644 --- a/Lib/test/seq_tests.py +++ b/Lib/test/seq_tests.py @@ -318,7 +318,6 @@ class CommonTest(unittest.TestCase): self.assertEqual(id(s), id(s*1)) def test_bigrepeat(self): - import sys if sys.maxsize <= 2147483647: x = self.type2test([0]) x *= 2**16 diff --git a/Lib/test/signalinterproctester.py b/Lib/test/signalinterproctester.py new file mode 100644 index 0000000000..d3ae170983 --- /dev/null +++ b/Lib/test/signalinterproctester.py @@ -0,0 +1,84 @@ +import os +import signal +import subprocess +import sys +import time +import unittest + + +class SIGUSR1Exception(Exception): + pass + + +class InterProcessSignalTests(unittest.TestCase): + def setUp(self): + self.got_signals = {'SIGHUP': 0, 'SIGUSR1': 0, 'SIGALRM': 0} + + def sighup_handler(self, signum, frame): + self.got_signals['SIGHUP'] += 1 + + def sigusr1_handler(self, signum, frame): + self.got_signals['SIGUSR1'] += 1 + raise SIGUSR1Exception + + def wait_signal(self, child, signame, exc_class=None): + try: + if child is not None: + # This wait should be interrupted by exc_class + # (if set) + child.wait() + + timeout = 10.0 + deadline = time.monotonic() + timeout + + while time.monotonic() < deadline: + if self.got_signals[signame]: + return + signal.pause() + except BaseException as exc: + if exc_class is not None and isinstance(exc, exc_class): + # got the expected exception + return + raise + + self.fail('signal %s not received after %s seconds' + % (signame, timeout)) + + def subprocess_send_signal(self, pid, signame): + code = 'import os, signal; os.kill(%s, signal.%s)' % (pid, signame) + args = [sys.executable, '-I', '-c', code] + return subprocess.Popen(args) + + def test_interprocess_signal(self): + # Install handlers. This function runs in a sub-process, so we + # don't worry about re-setting the default handlers. + signal.signal(signal.SIGHUP, self.sighup_handler) + signal.signal(signal.SIGUSR1, self.sigusr1_handler) + signal.signal(signal.SIGUSR2, signal.SIG_IGN) + signal.signal(signal.SIGALRM, signal.default_int_handler) + + # Let the sub-processes know who to send signals to. + pid = str(os.getpid()) + + with self.subprocess_send_signal(pid, "SIGHUP") as child: + self.wait_signal(child, 'SIGHUP') + self.assertEqual(self.got_signals, {'SIGHUP': 1, 'SIGUSR1': 0, + 'SIGALRM': 0}) + + with self.subprocess_send_signal(pid, "SIGUSR1") as child: + self.wait_signal(child, 'SIGUSR1', SIGUSR1Exception) + self.assertEqual(self.got_signals, {'SIGHUP': 1, 'SIGUSR1': 1, + 'SIGALRM': 0}) + + with self.subprocess_send_signal(pid, "SIGUSR2") as child: + # Nothing should happen: SIGUSR2 is ignored + child.wait() + + signal.alarm(1) + self.wait_signal(None, 'SIGALRM', KeyboardInterrupt) + self.assertEqual(self.got_signals, {'SIGHUP': 1, 'SIGUSR1': 1, + 'SIGALRM': 0}) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/sortperf.py b/Lib/test/sortperf.py index 90722f7abc..171e5cef5e 100644 --- a/Lib/test/sortperf.py +++ b/Lib/test/sortperf.py @@ -64,7 +64,7 @@ def doit(L): flush() def tabulate(r): - """Tabulate sort speed for lists of various sizes. + r"""Tabulate sort speed for lists of various sizes. The sizes are 2**i for i in r (the argument, a list). diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index d0d126a3a0..15d8fc849b 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -26,6 +26,7 @@ import sys import sysconfig import tempfile import time +import types import unittest import urllib.error import warnings @@ -89,11 +90,13 @@ __all__ = [ "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", "requires_IEEE_754", "skip_unless_xattr", "requires_zlib", "anticipate_failure", "load_package_tests", "detect_api_mismatch", - "requires_multiprocessing_queue", + "check__all__", "requires_android_level", "requires_multiprocessing_queue", # sys - "is_jython", "check_impl_detail", + "is_jython", "is_android", "check_impl_detail", "unix_shell", + "setswitchinterval", "android_not_root", # network "HOST", "IPV6_ENABLED", "find_unused_port", "bind_port", "open_urlresource", + "bind_unix_socket", # processes 'temp_umask', "reap_children", # logging @@ -104,7 +107,7 @@ __all__ = [ "check_warnings", "check_no_resource_warning", "EnvironmentVarGuard", "run_with_locale", "swap_item", "swap_attr", "Matcher", "set_memlimit", "SuppressCrashReport", "sortdict", - "run_with_tz", + "run_with_tz", "PGO", "missing_compiler_executable", ] class Error(Exception): @@ -706,6 +709,15 @@ def bind_port(sock, host=HOST): port = sock.getsockname()[1] return port +def bind_unix_socket(sock, addr): + """Bind a unix socket, raising SkipTest if PermissionError is raised.""" + assert sock.family == socket.AF_UNIX + try: + sock.bind(addr) + except PermissionError: + sock.close() + raise unittest.SkipTest('cannot bind AF_UNIX sockets') + def _is_ipv6_enabled(): """Check whether IPv6 is enabled on this host.""" if socket.has_ipv6: @@ -765,6 +777,15 @@ requires_lzma = unittest.skipUnless(lzma, 'requires lzma') is_jython = sys.platform.startswith('java') +_ANDROID_API_LEVEL = sysconfig.get_config_var('ANDROID_API_LEVEL') +is_android = (_ANDROID_API_LEVEL is not None and _ANDROID_API_LEVEL > 0) +android_not_root = (is_android and os.geteuid() != 0) + +if sys.platform != 'win32': + unix_shell = '/system/bin/sh' if is_android else '/bin/sh' +else: + unix_shell = None + # Filename used for testing if os.name == 'java': # Jython disallows @ in module names @@ -834,7 +855,7 @@ TESTFN_ENCODING = sys.getfilesystemencoding() # encoded by the filesystem encoding (in strict mode). It can be None if we # cannot generate such filename. TESTFN_UNENCODABLE = None -if os.name in ('nt', 'ce'): +if os.name == 'nt': # skip win32s (0) or Windows 9x/ME (1) if sys.getwindowsversion().platform >= 2: # Different kinds of characters from various languages to minimize the @@ -901,6 +922,10 @@ else: # Save the initial cwd SAVEDCWD = os.getcwd() +# Set by libregrtest/main.py so we can skip tests that are not +# useful for PGO +PGO = False + @contextlib.contextmanager def temp_dir(path=None, quiet=False): """Return a context manager that creates a temporary directory. @@ -1751,6 +1776,13 @@ def requires_resource(resource): else: return unittest.skip("resource {0!r} is not enabled".format(resource)) +def requires_android_level(level, reason): + if is_android and _ANDROID_API_LEVEL < level: + return unittest.skip('%s at Android API level %d' % + (reason, _ANDROID_API_LEVEL)) + else: + return _id + def cpython_only(test): """ Decorator for tests only applicable on CPython. @@ -2133,6 +2165,11 @@ def args_from_interpreter_flags(): settings in sys.flags and sys.warnoptions.""" return subprocess._args_from_interpreter_flags() +def optim_args_from_interpreter_flags(): + """Return a list of command-line arguments reproducing the current + optimization settings in sys.flags.""" + return subprocess._optim_args_from_interpreter_flags() + #============================================================ # Support for assertions about logging. #============================================================ @@ -2239,7 +2276,7 @@ def can_xattr(): os.setxattr(fp.fileno(), b"user.test", b"") # Kernels < 2.6.39 don't respect setxattr flags. kernel_version = platform.release() - m = re.match("2.6.(\d{1,2})", kernel_version) + m = re.match(r"2.6.(\d{1,2})", kernel_version) can = m is None or int(m.group(1)) >= 39 except OSError: can = False @@ -2284,6 +2321,65 @@ def detect_api_mismatch(ref_api, other_api, *, ignore=()): return missing_items +def check__all__(test_case, module, name_of_module=None, extra=(), + blacklist=()): + """Assert that the __all__ variable of 'module' contains all public names. + + The module's public names (its API) are detected automatically based on + whether they match the public name convention and were defined in + 'module'. + + The 'name_of_module' argument can specify (as a string or tuple thereof) + what module(s) an API could be defined in in order to be detected as a + public API. One case for this is when 'module' imports part of its public + API from other modules, possibly a C backend (like 'csv' and its '_csv'). + + The 'extra' argument can be a set of names that wouldn't otherwise be + automatically detected as "public", like objects without a proper + '__module__' attriubute. If provided, it will be added to the + automatically detected ones. + + The 'blacklist' argument can be a set of names that must not be treated + as part of the public API even though their names indicate otherwise. + + Usage: + import bar + import foo + import unittest + from test import support + + class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, foo) + + class OtherTestCase(unittest.TestCase): + def test__all__(self): + extra = {'BAR_CONST', 'FOO_CONST'} + blacklist = {'baz'} # Undocumented name. + # bar imports part of its API from _bar. + support.check__all__(self, bar, ('bar', '_bar'), + extra=extra, blacklist=blacklist) + + """ + + if name_of_module is None: + name_of_module = (module.__name__, ) + elif isinstance(name_of_module, str): + name_of_module = (name_of_module, ) + + expected = set(extra) + + for name in dir(module): + if name.startswith('_') or name in blacklist: + continue + obj = getattr(module, name) + if (getattr(obj, '__module__', None) in name_of_module or + (not hasattr(obj, '__module__') and + not isinstance(obj, types.ModuleType))): + expected.add(name) + test_case.assertCountEqual(module.__all__, expected) + + class SuppressCrashReport: """Try to prevent a crash report from popping up. @@ -2445,3 +2541,43 @@ def check_free_after_iterating(test, iter, cls, args=()): # The sequence should be deallocated just after the end of iterating gc_collect() test.assertTrue(done) + + +def missing_compiler_executable(cmd_names=[]): + """Check if the compiler components used to build the interpreter exist. + + Check for the existence of the compiler executables whose names are listed + in 'cmd_names' or all the compiler executables when 'cmd_names' is empty + and return the first missing executable or None when none is found + missing. + + """ + from distutils import ccompiler, sysconfig, spawn + compiler = ccompiler.new_compiler() + sysconfig.customize_compiler(compiler) + for name in compiler.executables: + if cmd_names and name not in cmd_names: + continue + cmd = getattr(compiler, name) + if cmd_names: + assert cmd is not None, \ + "the '%s' executable is not configured" % name + elif cmd is None: + continue + if spawn.find_executable(cmd[0]) is None: + return cmd[0] + + +_is_android_emulator = None +def setswitchinterval(interval): + # Setting a very low gil interval on the Android emulator causes python + # to hang (issue #26939). + minimum_interval = 1e-5 + if is_android and interval < minimum_interval: + global _is_android_emulator + if _is_android_emulator is None: + _is_android_emulator = (subprocess.check_output( + ['getprop', 'ro.kernel.qemu']).strip() == b'1') + if _is_android_emulator: + interval = minimum_interval + return sys.setswitchinterval(interval) diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index 2fc5e3e374..f6e82eb64a 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -38,6 +38,8 @@ class AllTest(unittest.TestCase): modname, e.__class__.__name__, e)) if "__builtins__" in names: del names["__builtins__"] + if '__annotations__' in names: + del names['__annotations__'] keys = set(names) all_list = sys.modules[modname].__all__ all_set = set(all_list) diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index 58f2f04fcd..ab4e247961 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -4,7 +4,6 @@ try: except ImportError: nl_langinfo = None -import codecs import locale import sys import unittest diff --git a/Lib/test/test__osx_support.py b/Lib/test/test__osx_support.py index ac6325a751..bcba8caa29 100644 --- a/Lib/test/test__osx_support.py +++ b/Lib/test/test__osx_support.py @@ -4,7 +4,6 @@ Test suite for _osx_support: shared OS X support functions. import os import platform -import shutil import stat import sys import unittest diff --git a/Lib/test/test_aifc.py b/Lib/test/test_aifc.py index ab5143787b..1bd1f89c8a 100644 --- a/Lib/test/test_aifc.py +++ b/Lib/test/test_aifc.py @@ -2,7 +2,6 @@ from test.support import findfile, TESTFN, unlink import unittest from test import audiotests from audioop import byteswap -import os import io import sys import struct diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 197d576eb6..a5c4a8ec97 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -4529,6 +4529,21 @@ class TestStrings(TestCase): string = "Namespace(bar='spam', foo=42)" self.assertStringEqual(ns, string) + def test_namespace_starkwargs_notidentifier(self): + ns = argparse.Namespace(**{'"': 'quote'}) + string = """Namespace(**{'"': 'quote'})""" + self.assertStringEqual(ns, string) + + def test_namespace_kwargs_and_starkwargs_notidentifier(self): + ns = argparse.Namespace(a=1, **{'"': 'quote'}) + string = """Namespace(a=1, **{'"': 'quote'})""" + self.assertStringEqual(ns, string) + + def test_namespace_starkwargs_identifier(self): + ns = argparse.Namespace(**{'valid': True}) + string = "Namespace(valid=True)" + self.assertStringEqual(ns, string) + def test_parser(self): parser = argparse.ArgumentParser(prog='PROG') string = ( diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 2a21e745b1..1f8967ccff 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -7,8 +7,6 @@ from test import support import weakref import pickle import operator -import io -import math import struct import sys import warnings @@ -328,8 +326,19 @@ class BaseTest: d = pickle.dumps((itorig, orig), proto) it, a = pickle.loads(d) a.fromlist(data2) - self.assertEqual(type(it), type(itorig)) - self.assertEqual(list(it), data2) + self.assertEqual(list(it), []) + + def test_exhausted_iterator(self): + a = array.array(self.typecode, self.example) + self.assertEqual(list(a), list(self.example)) + exhit = iter(a) + empit = iter(a) + for x in exhit: # exhaust the iterator + next(empit) # not exhausted + a.append(self.outside) + self.assertEqual(list(exhit), []) + self.assertEqual(list(empit), [self.outside]) + self.assertEqual(list(a), list(self.example) + [self.outside]) def test_insert(self): a = array.array(self.typecode, self.example) @@ -1080,6 +1089,12 @@ class BaseTest: a = array.array('B', b"") self.assertRaises(BufferError, getbuffer_with_null_view, a) + def test_free_after_iterating(self): + support.check_free_after_iterating(self, iter, array.array, + (self.typecode,)) + support.check_free_after_iterating(self, reversed, array.array, + (self.typecode,)) + class StringTest(BaseTest): def test_setitem(self): diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index d3e6d35943..8c62408bd8 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1,7 +1,8 @@ +import ast +import dis import os import sys import unittest -import ast import weakref from test import support @@ -115,6 +116,8 @@ exec_tests = [ # PEP 448: Additional Unpacking Generalizations "{**{1:2}, 2:3}", "{*{1, 2}, 3}", + # Asynchronous comprehensions + "async def f():\n [i async for b in c]", ] # These are compiled through "single" @@ -238,7 +241,7 @@ class AST_Tests(unittest.TestCase): ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) self.assertEqual(to_tuple(ast_tree), o) self._assertTrueorder(ast_tree, (0, 0)) - with self.subTest(action="compiling", input=i): + with self.subTest(action="compiling", input=i, kind=kind): compile(ast_tree, "?", kind) def test_slice(self): @@ -547,6 +550,17 @@ class ASTHelpers_Test(unittest.TestCase): compile(mod, 'test', 'exec') self.assertIn("invalid integer value: None", str(cm.exception)) + def test_level_as_none(self): + body = [ast.ImportFrom(module='time', + names=[ast.alias(name='sleep')], + level=None, + lineno=0, col_offset=0)] + mod = ast.Module(body) + code = compile(mod, 'test', 'exec') + ns = {} + exec(code, ns) + self.assertIn('sleep', ns) + class ASTValidatorTests(unittest.TestCase): @@ -742,7 +756,7 @@ class ASTValidatorTests(unittest.TestCase): def test_importfrom(self): imp = ast.ImportFrom(None, [ast.alias("x", None)], -42) - self.stmt(imp, "level less than -1") + self.stmt(imp, "Negative ImportFrom level") self.stmt(ast.ImportFrom(None, [], 0), "empty names on ImportFrom") def test_global(self): @@ -797,21 +811,21 @@ class ASTValidatorTests(unittest.TestCase): def _check_comprehension(self, fac): self.expr(fac([]), "comprehension with no generators") g = ast.comprehension(ast.Name("x", ast.Load()), - ast.Name("x", ast.Load()), []) + ast.Name("x", ast.Load()), [], 0) self.expr(fac([g]), "must have Store context") g = ast.comprehension(ast.Name("x", ast.Store()), - ast.Name("x", ast.Store()), []) + ast.Name("x", ast.Store()), [], 0) self.expr(fac([g]), "must have Load context") x = ast.Name("x", ast.Store()) y = ast.Name("y", ast.Load()) - g = ast.comprehension(x, y, [None]) + g = ast.comprehension(x, y, [None], 0) self.expr(fac([g]), "None disallowed") - g = ast.comprehension(x, y, [ast.Name("x", ast.Store())]) + g = ast.comprehension(x, y, [ast.Name("x", ast.Store())], 0) self.expr(fac([g]), "must have Load context") def _simple_comp(self, fac): g = ast.comprehension(ast.Name("x", ast.Store()), - ast.Name("x", ast.Load()), []) + ast.Name("x", ast.Load()), [], 0) self.expr(fac(ast.Name("x", ast.Store()), [g]), "must have Load context") def wrap(gens): @@ -829,7 +843,7 @@ class ASTValidatorTests(unittest.TestCase): def test_dictcomp(self): g = ast.comprehension(ast.Name("y", ast.Store()), - ast.Name("p", ast.Load()), []) + ast.Name("p", ast.Load()), [], 0) c = ast.DictComp(ast.Name("x", ast.Store()), ast.Name("y", ast.Load()), [g]) self.expr(c, "must have Load context") @@ -933,6 +947,125 @@ class ASTValidatorTests(unittest.TestCase): compile(mod, fn, "exec") +class ConstantTests(unittest.TestCase): + """Tests on the ast.Constant node type.""" + + def compile_constant(self, value): + tree = ast.parse("x = 123") + + node = tree.body[0].value + new_node = ast.Constant(value=value) + ast.copy_location(new_node, node) + tree.body[0].value = new_node + + code = compile(tree, "<string>", "exec") + + ns = {} + exec(code, ns) + return ns['x'] + + def test_validation(self): + with self.assertRaises(TypeError) as cm: + self.compile_constant([1, 2, 3]) + self.assertEqual(str(cm.exception), + "got an invalid type in Constant: list") + + def test_singletons(self): + for const in (None, False, True, Ellipsis, b'', frozenset()): + with self.subTest(const=const): + value = self.compile_constant(const) + self.assertIs(value, const) + + def test_values(self): + nested_tuple = (1,) + nested_frozenset = frozenset({1}) + for level in range(3): + nested_tuple = (nested_tuple, 2) + nested_frozenset = frozenset({nested_frozenset, 2}) + values = (123, 123.0, 123j, + "unicode", b'bytes', + tuple("tuple"), frozenset("frozenset"), + nested_tuple, nested_frozenset) + for value in values: + with self.subTest(value=value): + result = self.compile_constant(value) + self.assertEqual(result, value) + + def test_assign_to_constant(self): + tree = ast.parse("x = 1") + + target = tree.body[0].targets[0] + new_target = ast.Constant(value=1) + ast.copy_location(new_target, target) + tree.body[0].targets[0] = new_target + + with self.assertRaises(ValueError) as cm: + compile(tree, "string", "exec") + self.assertEqual(str(cm.exception), + "expression which can't be assigned " + "to in Store context") + + def test_get_docstring(self): + tree = ast.parse("'docstring'\nx = 1") + self.assertEqual(ast.get_docstring(tree), 'docstring') + + tree.body[0].value = ast.Constant(value='constant docstring') + self.assertEqual(ast.get_docstring(tree), 'constant docstring') + + def get_load_const(self, tree): + # Compile to bytecode, disassemble and get parameter of LOAD_CONST + # instructions + co = compile(tree, '<string>', 'exec') + consts = [] + for instr in dis.get_instructions(co): + if instr.opname == 'LOAD_CONST': + consts.append(instr.argval) + return consts + + @support.cpython_only + def test_load_const(self): + consts = [None, + True, False, + 124, + 2.0, + 3j, + "unicode", + b'bytes', + (1, 2, 3)] + + code = '\n'.join(['x={!r}'.format(const) for const in consts]) + code += '\nx = ...' + consts.extend((Ellipsis, None)) + + tree = ast.parse(code) + self.assertEqual(self.get_load_const(tree), + consts) + + # Replace expression nodes with constants + for assign, const in zip(tree.body, consts): + assert isinstance(assign, ast.Assign), ast.dump(assign) + new_node = ast.Constant(value=const) + ast.copy_location(new_node, assign.value) + assign.value = new_node + + self.assertEqual(self.get_load_const(tree), + consts) + + def test_literal_eval(self): + tree = ast.parse("1 + 2") + binop = tree.body[0].value + + new_left = ast.Constant(value=10) + ast.copy_location(new_left, binop.left) + binop.left = new_left + + new_right = ast.Constant(value=20) + ast.copy_location(new_right, binop.right) + binop.right = new_right + + self.assertEqual(ast.literal_eval(binop), 30) + + def main(): if __name__ != '__main__': return @@ -940,8 +1073,9 @@ def main(): for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), (eval_tests, "eval")): print(kind+"_results = [") - for s in statements: - print(repr(to_tuple(compile(s, "?", kind, 0x400)))+",") + for statement in statements: + tree = ast.parse(statement, "?", kind) + print("%r," % (to_tuple(tree),)) print("]") print("main()") raise SystemExit @@ -979,19 +1113,20 @@ exec_results = [ ('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Break', (1, 11))], [])]), ('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Continue', (1, 11))], [])]), ('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [])]), -('Module', [('Expr', (1, 0), ('ListComp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [])]))]), -('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [])]))]), -('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 12), [('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 20), 'c', ('Load',)), [])]))]), -('Module', [('Expr', (1, 0), ('GeneratorExp', (2, 4), ('Tuple', (3, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [])]))]), -('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), []), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))])]))]), -('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [])]))]), -('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))])]))]), -('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [])]))]), +('Module', [('Expr', (1, 0), ('ListComp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [], 0)]))]), +('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [], 0)]))]), +('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 12), [('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 20), 'c', ('Load',)), [], 0)]))]), +('Module', [('Expr', (1, 0), ('GeneratorExp', (2, 4), ('Tuple', (3, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [], 0)]))]), +('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))], 0)]))]), +('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [], 0)]))]), +('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))], 0)]))]), +('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [], 0)]))]), ('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Await', (2, 1), ('Call', (2, 7), ('Name', (2, 7), 'something', ('Load',)), [], [])))], [], None)]), ('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 7), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Num', (2, 19), 1))], [('Expr', (3, 7), ('Num', (3, 7), 2))])], [], None)]), ('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 7), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Num', (2, 20), 1))])], [], None)]), ('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Num', (1, 10), 2)], [('Dict', (1, 3), [('Num', (1, 4), 1)], [('Num', (1, 6), 2)]), ('Num', (1, 12), 3)]))]), ('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Num', (1, 3), 1), ('Num', (1, 6), 2)]), ('Load',)), ('Num', (1, 10), 3)]))]), +('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('ListComp', (2, 2), ('Name', (2, 2), 'i', ('Load',)), [('comprehension', ('Name', (2, 14), 'b', ('Store',)), ('Name', (2, 19), 'c', ('Load',)), [], 1)]))], [], None)]), ] single_results = [ ('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]), @@ -1006,8 +1141,8 @@ eval_results = [ ('Expression', ('Dict', (1, 0), [], [])), ('Expression', ('Set', (1, 0), [('NameConstant', (1, 1), None)])), ('Expression', ('Dict', (1, 0), [('Num', (2, 6), 1)], [('Num', (4, 10), 2)])), -('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])), -('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])), +('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))], 0)])), +('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))], 0)])), ('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])), ('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2), ('Starred', (1, 10), ('Name', (1, 11), 'd', ('Load',)), ('Load',))], [('keyword', 'c', ('Num', (1, 8), 3)), ('keyword', None, ('Name', (1, 15), 'e', ('Load',)))])), ('Expression', ('Num', (1, 0), 10)), diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py new file mode 100644 index 0000000000..34ab8a04ee --- /dev/null +++ b/Lib/test/test_asyncgen.py @@ -0,0 +1,1044 @@ +import inspect +import sys +import types +import unittest + +from unittest import mock + +from test.support import import_module +asyncio = import_module("asyncio") + + +class AwaitException(Exception): + pass + + +@types.coroutine +def awaitable(*, throw=False): + if throw: + yield ('throw',) + else: + yield ('result',) + + +def run_until_complete(coro): + exc = False + while True: + try: + if exc: + exc = False + fut = coro.throw(AwaitException) + else: + fut = coro.send(None) + except StopIteration as ex: + return ex.args[0] + + if fut == ('throw',): + exc = True + + +def to_list(gen): + async def iterate(): + res = [] + async for i in gen: + res.append(i) + return res + + return run_until_complete(iterate()) + + +class AsyncGenSyntaxTest(unittest.TestCase): + + def test_async_gen_syntax_01(self): + code = '''async def foo(): + await abc + yield from 123 + ''' + + with self.assertRaisesRegex(SyntaxError, 'yield from.*inside async'): + exec(code, {}, {}) + + def test_async_gen_syntax_02(self): + code = '''async def foo(): + yield from 123 + ''' + + with self.assertRaisesRegex(SyntaxError, 'yield from.*inside async'): + exec(code, {}, {}) + + def test_async_gen_syntax_03(self): + code = '''async def foo(): + await abc + yield + return 123 + ''' + + with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'): + exec(code, {}, {}) + + def test_async_gen_syntax_04(self): + code = '''async def foo(): + yield + return 123 + ''' + + with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'): + exec(code, {}, {}) + + def test_async_gen_syntax_05(self): + code = '''async def foo(): + if 0: + yield + return 12 + ''' + + with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'): + exec(code, {}, {}) + + +class AsyncGenTest(unittest.TestCase): + + def compare_generators(self, sync_gen, async_gen): + def sync_iterate(g): + res = [] + while True: + try: + res.append(g.__next__()) + except StopIteration: + res.append('STOP') + break + except Exception as ex: + res.append(str(type(ex))) + return res + + def async_iterate(g): + res = [] + while True: + try: + g.__anext__().__next__() + except StopAsyncIteration: + res.append('STOP') + break + except StopIteration as ex: + if ex.args: + res.append(ex.args[0]) + else: + res.append('EMPTY StopIteration') + break + except Exception as ex: + res.append(str(type(ex))) + return res + + sync_gen_result = sync_iterate(sync_gen) + async_gen_result = async_iterate(async_gen) + self.assertEqual(sync_gen_result, async_gen_result) + return async_gen_result + + def test_async_gen_iteration_01(self): + async def gen(): + await awaitable() + a = yield 123 + self.assertIs(a, None) + await awaitable() + yield 456 + await awaitable() + yield 789 + + self.assertEqual(to_list(gen()), [123, 456, 789]) + + def test_async_gen_iteration_02(self): + async def gen(): + await awaitable() + yield 123 + await awaitable() + + g = gen() + ai = g.__aiter__() + self.assertEqual(ai.__anext__().__next__(), ('result',)) + + try: + ai.__anext__().__next__() + except StopIteration as ex: + self.assertEqual(ex.args[0], 123) + else: + self.fail('StopIteration was not raised') + + self.assertEqual(ai.__anext__().__next__(), ('result',)) + + try: + ai.__anext__().__next__() + except StopAsyncIteration as ex: + self.assertFalse(ex.args) + else: + self.fail('StopAsyncIteration was not raised') + + def test_async_gen_exception_03(self): + async def gen(): + await awaitable() + yield 123 + await awaitable(throw=True) + yield 456 + + with self.assertRaises(AwaitException): + to_list(gen()) + + def test_async_gen_exception_04(self): + async def gen(): + await awaitable() + yield 123 + 1 / 0 + + g = gen() + ai = g.__aiter__() + self.assertEqual(ai.__anext__().__next__(), ('result',)) + + try: + ai.__anext__().__next__() + except StopIteration as ex: + self.assertEqual(ex.args[0], 123) + else: + self.fail('StopIteration was not raised') + + with self.assertRaises(ZeroDivisionError): + ai.__anext__().__next__() + + def test_async_gen_exception_05(self): + async def gen(): + yield 123 + raise StopAsyncIteration + + with self.assertRaisesRegex(RuntimeError, + 'async generator.*StopAsyncIteration'): + to_list(gen()) + + def test_async_gen_exception_06(self): + async def gen(): + yield 123 + raise StopIteration + + with self.assertRaisesRegex(RuntimeError, + 'async generator.*StopIteration'): + to_list(gen()) + + def test_async_gen_exception_07(self): + def sync_gen(): + try: + yield 1 + 1 / 0 + finally: + yield 2 + yield 3 + + yield 100 + + async def async_gen(): + try: + yield 1 + 1 / 0 + finally: + yield 2 + yield 3 + + yield 100 + + self.compare_generators(sync_gen(), async_gen()) + + def test_async_gen_exception_08(self): + def sync_gen(): + try: + yield 1 + finally: + yield 2 + 1 / 0 + yield 3 + + yield 100 + + async def async_gen(): + try: + yield 1 + await awaitable() + finally: + await awaitable() + yield 2 + 1 / 0 + yield 3 + + yield 100 + + self.compare_generators(sync_gen(), async_gen()) + + def test_async_gen_exception_09(self): + def sync_gen(): + try: + yield 1 + 1 / 0 + finally: + yield 2 + yield 3 + + yield 100 + + async def async_gen(): + try: + await awaitable() + yield 1 + 1 / 0 + finally: + yield 2 + await awaitable() + yield 3 + + yield 100 + + self.compare_generators(sync_gen(), async_gen()) + + def test_async_gen_exception_10(self): + async def gen(): + yield 123 + with self.assertRaisesRegex(TypeError, + "non-None value .* async generator"): + gen().__anext__().send(100) + + def test_async_gen_api_01(self): + async def gen(): + yield 123 + + g = gen() + + self.assertEqual(g.__name__, 'gen') + g.__name__ = '123' + self.assertEqual(g.__name__, '123') + + self.assertIn('.gen', g.__qualname__) + g.__qualname__ = '123' + self.assertEqual(g.__qualname__, '123') + + self.assertIsNone(g.ag_await) + self.assertIsInstance(g.ag_frame, types.FrameType) + self.assertFalse(g.ag_running) + self.assertIsInstance(g.ag_code, types.CodeType) + + self.assertTrue(inspect.isawaitable(g.aclose())) + + +class AsyncGenAsyncioTest(unittest.TestCase): + + def setUp(self): + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(None) + + def tearDown(self): + self.loop.close() + self.loop = None + + async def to_list(self, gen): + res = [] + async for i in gen: + res.append(i) + return res + + def test_async_gen_asyncio_01(self): + async def gen(): + yield 1 + await asyncio.sleep(0.01, loop=self.loop) + yield 2 + await asyncio.sleep(0.01, loop=self.loop) + return + yield 3 + + res = self.loop.run_until_complete(self.to_list(gen())) + self.assertEqual(res, [1, 2]) + + def test_async_gen_asyncio_02(self): + async def gen(): + yield 1 + await asyncio.sleep(0.01, loop=self.loop) + yield 2 + 1 / 0 + yield 3 + + with self.assertRaises(ZeroDivisionError): + self.loop.run_until_complete(self.to_list(gen())) + + def test_async_gen_asyncio_03(self): + loop = self.loop + + class Gen: + async def __aiter__(self): + yield 1 + await asyncio.sleep(0.01, loop=loop) + yield 2 + + res = loop.run_until_complete(self.to_list(Gen())) + self.assertEqual(res, [1, 2]) + + def test_async_gen_asyncio_anext_04(self): + async def foo(): + yield 1 + await asyncio.sleep(0.01, loop=self.loop) + try: + yield 2 + yield 3 + except ZeroDivisionError: + yield 1000 + await asyncio.sleep(0.01, loop=self.loop) + yield 4 + + async def run1(): + it = foo().__aiter__() + + self.assertEqual(await it.__anext__(), 1) + self.assertEqual(await it.__anext__(), 2) + self.assertEqual(await it.__anext__(), 3) + self.assertEqual(await it.__anext__(), 4) + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + + async def run2(): + it = foo().__aiter__() + + self.assertEqual(await it.__anext__(), 1) + self.assertEqual(await it.__anext__(), 2) + try: + it.__anext__().throw(ZeroDivisionError) + except StopIteration as ex: + self.assertEqual(ex.args[0], 1000) + else: + self.fail('StopIteration was not raised') + self.assertEqual(await it.__anext__(), 4) + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + + self.loop.run_until_complete(run1()) + self.loop.run_until_complete(run2()) + + def test_async_gen_asyncio_anext_05(self): + async def foo(): + v = yield 1 + v = yield v + yield v * 100 + + async def run(): + it = foo().__aiter__() + + try: + it.__anext__().send(None) + except StopIteration as ex: + self.assertEqual(ex.args[0], 1) + else: + self.fail('StopIteration was not raised') + + try: + it.__anext__().send(10) + except StopIteration as ex: + self.assertEqual(ex.args[0], 10) + else: + self.fail('StopIteration was not raised') + + try: + it.__anext__().send(12) + except StopIteration as ex: + self.assertEqual(ex.args[0], 1200) + else: + self.fail('StopIteration was not raised') + + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + + self.loop.run_until_complete(run()) + + def test_async_gen_asyncio_anext_06(self): + DONE = 0 + + # test synchronous generators + def foo(): + try: + yield + except: + pass + g = foo() + g.send(None) + with self.assertRaises(StopIteration): + g.send(None) + + # now with asynchronous generators + + async def gen(): + nonlocal DONE + try: + yield + except: + pass + DONE = 1 + + async def run(): + nonlocal DONE + g = gen() + await g.asend(None) + with self.assertRaises(StopAsyncIteration): + await g.asend(None) + DONE += 10 + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 11) + + def test_async_gen_asyncio_anext_tuple(self): + async def foo(): + try: + yield (1,) + except ZeroDivisionError: + yield (2,) + + async def run(): + it = foo().__aiter__() + + self.assertEqual(await it.__anext__(), (1,)) + with self.assertRaises(StopIteration) as cm: + it.__anext__().throw(ZeroDivisionError) + self.assertEqual(cm.exception.args[0], (2,)) + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + + self.loop.run_until_complete(run()) + + def test_async_gen_asyncio_anext_stopiteration(self): + async def foo(): + try: + yield StopIteration(1) + except ZeroDivisionError: + yield StopIteration(3) + + async def run(): + it = foo().__aiter__() + + v = await it.__anext__() + self.assertIsInstance(v, StopIteration) + self.assertEqual(v.value, 1) + with self.assertRaises(StopIteration) as cm: + it.__anext__().throw(ZeroDivisionError) + v = cm.exception.args[0] + self.assertIsInstance(v, StopIteration) + self.assertEqual(v.value, 3) + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + + self.loop.run_until_complete(run()) + + def test_async_gen_asyncio_aclose_06(self): + async def foo(): + try: + yield 1 + 1 / 0 + finally: + await asyncio.sleep(0.01, loop=self.loop) + yield 12 + + async def run(): + gen = foo() + it = gen.__aiter__() + await it.__anext__() + await gen.aclose() + + with self.assertRaisesRegex( + RuntimeError, + "async generator ignored GeneratorExit"): + self.loop.run_until_complete(run()) + + def test_async_gen_asyncio_aclose_07(self): + DONE = 0 + + async def foo(): + nonlocal DONE + try: + yield 1 + 1 / 0 + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE += 1 + DONE += 1000 + + async def run(): + gen = foo() + it = gen.__aiter__() + await it.__anext__() + await gen.aclose() + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + def test_async_gen_asyncio_aclose_08(self): + DONE = 0 + + fut = asyncio.Future(loop=self.loop) + + async def foo(): + nonlocal DONE + try: + yield 1 + await fut + DONE += 1000 + yield 2 + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE += 1 + DONE += 1000 + + async def run(): + gen = foo() + it = gen.__aiter__() + self.assertEqual(await it.__anext__(), 1) + t = self.loop.create_task(it.__anext__()) + await asyncio.sleep(0.01, loop=self.loop) + await gen.aclose() + return t + + t = self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + # Silence ResourceWarnings + fut.cancel() + t.cancel() + self.loop.run_until_complete(asyncio.sleep(0.01, loop=self.loop)) + + def test_async_gen_asyncio_gc_aclose_09(self): + DONE = 0 + + async def gen(): + nonlocal DONE + try: + while True: + yield 1 + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE = 1 + + async def run(): + g = gen() + await g.__anext__() + await g.__anext__() + del g + + await asyncio.sleep(0.1, loop=self.loop) + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + def test_async_gen_asyncio_aclose_10(self): + DONE = 0 + + # test synchronous generators + def foo(): + try: + yield + except: + pass + g = foo() + g.send(None) + g.close() + + # now with asynchronous generators + + async def gen(): + nonlocal DONE + try: + yield + except: + pass + DONE = 1 + + async def run(): + nonlocal DONE + g = gen() + await g.asend(None) + await g.aclose() + DONE += 10 + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 11) + + def test_async_gen_asyncio_aclose_11(self): + DONE = 0 + + # test synchronous generators + def foo(): + try: + yield + except: + pass + yield + g = foo() + g.send(None) + with self.assertRaisesRegex(RuntimeError, 'ignored GeneratorExit'): + g.close() + + # now with asynchronous generators + + async def gen(): + nonlocal DONE + try: + yield + except: + pass + yield + DONE += 1 + + async def run(): + nonlocal DONE + g = gen() + await g.asend(None) + with self.assertRaisesRegex(RuntimeError, 'ignored GeneratorExit'): + await g.aclose() + DONE += 10 + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 10) + + def test_async_gen_asyncio_asend_01(self): + DONE = 0 + + # Sanity check: + def sgen(): + v = yield 1 + yield v * 2 + sg = sgen() + v = sg.send(None) + self.assertEqual(v, 1) + v = sg.send(100) + self.assertEqual(v, 200) + + async def gen(): + nonlocal DONE + try: + await asyncio.sleep(0.01, loop=self.loop) + v = yield 1 + await asyncio.sleep(0.01, loop=self.loop) + yield v * 2 + await asyncio.sleep(0.01, loop=self.loop) + return + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE = 1 + + async def run(): + g = gen() + + v = await g.asend(None) + self.assertEqual(v, 1) + + v = await g.asend(100) + self.assertEqual(v, 200) + + with self.assertRaises(StopAsyncIteration): + await g.asend(None) + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + def test_async_gen_asyncio_asend_02(self): + DONE = 0 + + async def sleep_n_crash(delay): + await asyncio.sleep(delay, loop=self.loop) + 1 / 0 + + async def gen(): + nonlocal DONE + try: + await asyncio.sleep(0.01, loop=self.loop) + v = yield 1 + await sleep_n_crash(0.01) + DONE += 1000 + yield v * 2 + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE = 1 + + async def run(): + g = gen() + + v = await g.asend(None) + self.assertEqual(v, 1) + + await g.asend(100) + + with self.assertRaises(ZeroDivisionError): + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + def test_async_gen_asyncio_asend_03(self): + DONE = 0 + + async def sleep_n_crash(delay): + fut = asyncio.ensure_future(asyncio.sleep(delay, loop=self.loop), + loop=self.loop) + self.loop.call_later(delay / 2, lambda: fut.cancel()) + return await fut + + async def gen(): + nonlocal DONE + try: + await asyncio.sleep(0.01, loop=self.loop) + v = yield 1 + await sleep_n_crash(0.01) + DONE += 1000 + yield v * 2 + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE = 1 + + async def run(): + g = gen() + + v = await g.asend(None) + self.assertEqual(v, 1) + + await g.asend(100) + + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + def test_async_gen_asyncio_athrow_01(self): + DONE = 0 + + class FooEr(Exception): + pass + + # Sanity check: + def sgen(): + try: + v = yield 1 + except FooEr: + v = 1000 + yield v * 2 + sg = sgen() + v = sg.send(None) + self.assertEqual(v, 1) + v = sg.throw(FooEr) + self.assertEqual(v, 2000) + with self.assertRaises(StopIteration): + sg.send(None) + + async def gen(): + nonlocal DONE + try: + await asyncio.sleep(0.01, loop=self.loop) + try: + v = yield 1 + except FooEr: + v = 1000 + await asyncio.sleep(0.01, loop=self.loop) + yield v * 2 + await asyncio.sleep(0.01, loop=self.loop) + # return + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE = 1 + + async def run(): + g = gen() + + v = await g.asend(None) + self.assertEqual(v, 1) + + v = await g.athrow(FooEr) + self.assertEqual(v, 2000) + + with self.assertRaises(StopAsyncIteration): + await g.asend(None) + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + def test_async_gen_asyncio_athrow_02(self): + DONE = 0 + + class FooEr(Exception): + pass + + async def sleep_n_crash(delay): + fut = asyncio.ensure_future(asyncio.sleep(delay, loop=self.loop), + loop=self.loop) + self.loop.call_later(delay / 2, lambda: fut.cancel()) + return await fut + + async def gen(): + nonlocal DONE + try: + await asyncio.sleep(0.01, loop=self.loop) + try: + v = yield 1 + except FooEr: + await sleep_n_crash(0.01) + yield v * 2 + await asyncio.sleep(0.01, loop=self.loop) + # return + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE = 1 + + async def run(): + g = gen() + + v = await g.asend(None) + self.assertEqual(v, 1) + + try: + await g.athrow(FooEr) + except asyncio.CancelledError: + self.assertEqual(DONE, 1) + raise + else: + self.fail('CancelledError was not raised') + + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + def test_async_gen_asyncio_athrow_03(self): + DONE = 0 + + # test synchronous generators + def foo(): + try: + yield + except: + pass + g = foo() + g.send(None) + with self.assertRaises(StopIteration): + g.throw(ValueError) + + # now with asynchronous generators + + async def gen(): + nonlocal DONE + try: + yield + except: + pass + DONE = 1 + + async def run(): + nonlocal DONE + g = gen() + await g.asend(None) + with self.assertRaises(StopAsyncIteration): + await g.athrow(ValueError) + DONE += 10 + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 11) + + def test_async_gen_asyncio_athrow_tuple(self): + async def gen(): + try: + yield 1 + except ZeroDivisionError: + yield (2,) + + async def run(): + g = gen() + v = await g.asend(None) + self.assertEqual(v, 1) + v = await g.athrow(ZeroDivisionError) + self.assertEqual(v, (2,)) + with self.assertRaises(StopAsyncIteration): + await g.asend(None) + + self.loop.run_until_complete(run()) + + def test_async_gen_asyncio_athrow_stopiteration(self): + async def gen(): + try: + yield 1 + except ZeroDivisionError: + yield StopIteration(2) + + async def run(): + g = gen() + v = await g.asend(None) + self.assertEqual(v, 1) + v = await g.athrow(ZeroDivisionError) + self.assertIsInstance(v, StopIteration) + self.assertEqual(v.value, 2) + with self.assertRaises(StopAsyncIteration): + await g.asend(None) + + self.loop.run_until_complete(run()) + + def test_async_gen_asyncio_shutdown_01(self): + finalized = 0 + + async def waiter(timeout): + nonlocal finalized + try: + await asyncio.sleep(timeout, loop=self.loop) + yield 1 + finally: + await asyncio.sleep(0, loop=self.loop) + finalized += 1 + + async def wait(): + async for _ in waiter(1): + pass + + t1 = self.loop.create_task(wait()) + t2 = self.loop.create_task(wait()) + + self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop)) + + self.loop.run_until_complete(self.loop.shutdown_asyncgens()) + self.assertEqual(finalized, 2) + + # Silence warnings + t1.cancel() + t2.cancel() + self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop)) + + def test_async_gen_asyncio_shutdown_02(self): + logged = 0 + + def logger(loop, context): + nonlocal logged + self.assertIn('asyncgen', context) + expected = 'an error occurred during closing of asynchronous' + if expected in context['message']: + logged += 1 + + async def waiter(timeout): + try: + await asyncio.sleep(timeout, loop=self.loop) + yield 1 + finally: + 1 / 0 + + async def wait(): + async for _ in waiter(1): + pass + + t = self.loop.create_task(wait()) + self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop)) + + self.loop.set_exception_handler(logger) + self.loop.run_until_complete(self.loop.shutdown_asyncgens()) + + self.assertEqual(logged, 1) + + # Silence warnings + t.cancel() + self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop)) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_asynchat.py b/Lib/test/test_asynchat.py index 3a33fc8b2d..0eba76db47 100644 --- a/Lib/test/test_asynchat.py +++ b/Lib/test/test_asynchat.py @@ -12,7 +12,6 @@ import socket import sys import time import unittest -import warnings import unittest.mock try: import threading @@ -297,37 +296,6 @@ class TestHelperFunctions(unittest.TestCase): self.assertEqual(asynchat.find_prefix_at_end("qwertydkjf", "\r\n"), 0) -class TestFifo(unittest.TestCase): - def test_basic(self): - with self.assertWarns(DeprecationWarning) as cm: - f = asynchat.fifo() - self.assertEqual(str(cm.warning), - "fifo class will be removed in Python 3.6") - f.push(7) - f.push(b'a') - self.assertEqual(len(f), 2) - self.assertEqual(f.first(), 7) - self.assertEqual(f.pop(), (1, 7)) - self.assertEqual(len(f), 1) - self.assertEqual(f.first(), b'a') - self.assertEqual(f.is_empty(), False) - self.assertEqual(f.pop(), (1, b'a')) - self.assertEqual(len(f), 0) - self.assertEqual(f.is_empty(), True) - self.assertEqual(f.pop(), (0, None)) - - def test_given_list(self): - with self.assertWarns(DeprecationWarning) as cm: - f = asynchat.fifo([b'x', 17, 3]) - self.assertEqual(str(cm.warning), - "fifo class will be removed in Python 3.6") - self.assertEqual(len(f), 3) - self.assertEqual(f.pop(), (1, b'x')) - self.assertEqual(f.pop(), (1, 17)) - self.assertEqual(f.pop(), (1, 3)) - self.assertEqual(f.pop(), (0, None)) - - class TestNotConnected(unittest.TestCase): def test_disallow_negative_terminator(self): # Issue #11259 diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index c306b77e65..89afdcaff2 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -9,6 +9,7 @@ from unittest import mock import asyncio from asyncio import test_utils +from asyncio import futures try: from test import support except ImportError: @@ -94,7 +95,10 @@ class DuckTests(test_utils.TestCase): assert g is f -class FutureTests(test_utils.TestCase): +class BaseFutureTests: + + def _new_future(self, loop=None): + raise NotImplementedError def setUp(self): super().setUp() @@ -110,22 +114,22 @@ class FutureTests(test_utils.TestCase): self.assertFalse(asyncio.isfuture(MyFuture)) self.assertTrue(asyncio.isfuture(MyFuture())) - self.assertFalse(asyncio.isfuture(1)) - self.assertFalse(asyncio.isfuture(asyncio.Future)) # As `isinstance(Mock(), Future)` returns `False` self.assertFalse(asyncio.isfuture(mock.Mock())) + f = self._new_future(loop=self.loop) + self.assertTrue(asyncio.isfuture(f)) + self.assertFalse(asyncio.isfuture(type(f))) + # As `isinstance(Mock(Future), Future)` returns `True` - self.assertTrue(asyncio.isfuture(mock.Mock(asyncio.Future))) + self.assertTrue(asyncio.isfuture(mock.Mock(type(f)))) - f = asyncio.Future(loop=self.loop) - self.assertTrue(asyncio.isfuture(f)) f.cancel() def test_initial_state(self): - f = asyncio.Future(loop=self.loop) + f = self._new_future(loop=self.loop) self.assertFalse(f.cancelled()) self.assertFalse(f.done()) f.cancel() @@ -133,15 +137,15 @@ class FutureTests(test_utils.TestCase): def test_init_constructor_default_loop(self): asyncio.set_event_loop(self.loop) - f = asyncio.Future() + f = self._new_future() self.assertIs(f._loop, self.loop) def test_constructor_positional(self): # Make sure Future doesn't accept a positional argument - self.assertRaises(TypeError, asyncio.Future, 42) + self.assertRaises(TypeError, self._new_future, 42) def test_cancel(self): - f = asyncio.Future(loop=self.loop) + f = self._new_future(loop=self.loop) self.assertTrue(f.cancel()) self.assertTrue(f.cancelled()) self.assertTrue(f.done()) @@ -152,7 +156,7 @@ class FutureTests(test_utils.TestCase): self.assertFalse(f.cancel()) def test_result(self): - f = asyncio.Future(loop=self.loop) + f = self._new_future(loop=self.loop) self.assertRaises(asyncio.InvalidStateError, f.result) f.set_result(42) @@ -166,7 +170,7 @@ class FutureTests(test_utils.TestCase): def test_exception(self): exc = RuntimeError() - f = asyncio.Future(loop=self.loop) + f = self._new_future(loop=self.loop) self.assertRaises(asyncio.InvalidStateError, f.exception) # StopIteration cannot be raised into a Future - CPython issue26221 @@ -183,12 +187,12 @@ class FutureTests(test_utils.TestCase): self.assertFalse(f.cancel()) def test_exception_class(self): - f = asyncio.Future(loop=self.loop) + f = self._new_future(loop=self.loop) f.set_exception(RuntimeError) self.assertIsInstance(f.exception(), RuntimeError) def test_yield_from_twice(self): - f = asyncio.Future(loop=self.loop) + f = self._new_future(loop=self.loop) def fixture(): yield 'A' @@ -207,7 +211,7 @@ class FutureTests(test_utils.TestCase): def test_future_repr(self): self.loop.set_debug(True) - f_pending_debug = asyncio.Future(loop=self.loop) + f_pending_debug = self._new_future(loop=self.loop) frame = f_pending_debug._source_traceback[-1] self.assertEqual(repr(f_pending_debug), '<Future pending created at %s:%s>' @@ -215,21 +219,21 @@ class FutureTests(test_utils.TestCase): f_pending_debug.cancel() self.loop.set_debug(False) - f_pending = asyncio.Future(loop=self.loop) + f_pending = self._new_future(loop=self.loop) self.assertEqual(repr(f_pending), '<Future pending>') f_pending.cancel() - f_cancelled = asyncio.Future(loop=self.loop) + f_cancelled = self._new_future(loop=self.loop) f_cancelled.cancel() self.assertEqual(repr(f_cancelled), '<Future cancelled>') - f_result = asyncio.Future(loop=self.loop) + f_result = self._new_future(loop=self.loop) f_result.set_result(4) self.assertEqual(repr(f_result), '<Future finished result=4>') self.assertEqual(f_result.result(), 4) exc = RuntimeError() - f_exception = asyncio.Future(loop=self.loop) + f_exception = self._new_future(loop=self.loop) f_exception.set_exception(exc) self.assertEqual(repr(f_exception), '<Future finished exception=RuntimeError()>') @@ -240,7 +244,7 @@ class FutureTests(test_utils.TestCase): text = '%s() at %s:%s' % (func.__qualname__, filename, lineno) return re.escape(text) - f_one_callbacks = asyncio.Future(loop=self.loop) + f_one_callbacks = self._new_future(loop=self.loop) f_one_callbacks.add_done_callback(_fakefunc) fake_repr = func_repr(_fakefunc) self.assertRegex(repr(f_one_callbacks), @@ -249,7 +253,7 @@ class FutureTests(test_utils.TestCase): self.assertEqual(repr(f_one_callbacks), '<Future cancelled>') - f_two_callbacks = asyncio.Future(loop=self.loop) + f_two_callbacks = self._new_future(loop=self.loop) f_two_callbacks.add_done_callback(first_cb) f_two_callbacks.add_done_callback(last_cb) first_repr = func_repr(first_cb) @@ -258,7 +262,7 @@ class FutureTests(test_utils.TestCase): r'<Future pending cb=\[%s, %s\]>' % (first_repr, last_repr)) - f_many_callbacks = asyncio.Future(loop=self.loop) + f_many_callbacks = self._new_future(loop=self.loop) f_many_callbacks.add_done_callback(first_cb) for i in range(8): f_many_callbacks.add_done_callback(_fakefunc) @@ -273,31 +277,31 @@ class FutureTests(test_utils.TestCase): def test_copy_state(self): from asyncio.futures import _copy_future_state - f = asyncio.Future(loop=self.loop) + f = self._new_future(loop=self.loop) f.set_result(10) - newf = asyncio.Future(loop=self.loop) + newf = self._new_future(loop=self.loop) _copy_future_state(f, newf) self.assertTrue(newf.done()) self.assertEqual(newf.result(), 10) - f_exception = asyncio.Future(loop=self.loop) + f_exception = self._new_future(loop=self.loop) f_exception.set_exception(RuntimeError()) - newf_exception = asyncio.Future(loop=self.loop) + newf_exception = self._new_future(loop=self.loop) _copy_future_state(f_exception, newf_exception) self.assertTrue(newf_exception.done()) self.assertRaises(RuntimeError, newf_exception.result) - f_cancelled = asyncio.Future(loop=self.loop) + f_cancelled = self._new_future(loop=self.loop) f_cancelled.cancel() - newf_cancelled = asyncio.Future(loop=self.loop) + newf_cancelled = self._new_future(loop=self.loop) _copy_future_state(f_cancelled, newf_cancelled) self.assertTrue(newf_cancelled.cancelled()) def test_iter(self): - fut = asyncio.Future(loop=self.loop) + fut = self._new_future(loop=self.loop) def coro(): yield from fut @@ -310,20 +314,20 @@ class FutureTests(test_utils.TestCase): @mock.patch('asyncio.base_events.logger') def test_tb_logger_abandoned(self, m_log): - fut = asyncio.Future(loop=self.loop) + fut = self._new_future(loop=self.loop) del fut self.assertFalse(m_log.error.called) @mock.patch('asyncio.base_events.logger') def test_tb_logger_result_unretrieved(self, m_log): - fut = asyncio.Future(loop=self.loop) + fut = self._new_future(loop=self.loop) fut.set_result(42) del fut self.assertFalse(m_log.error.called) @mock.patch('asyncio.base_events.logger') def test_tb_logger_result_retrieved(self, m_log): - fut = asyncio.Future(loop=self.loop) + fut = self._new_future(loop=self.loop) fut.set_result(42) fut.result() del fut @@ -331,7 +335,7 @@ class FutureTests(test_utils.TestCase): @mock.patch('asyncio.base_events.logger') def test_tb_logger_exception_unretrieved(self, m_log): - fut = asyncio.Future(loop=self.loop) + fut = self._new_future(loop=self.loop) fut.set_exception(RuntimeError('boom')) del fut test_utils.run_briefly(self.loop) @@ -340,7 +344,7 @@ class FutureTests(test_utils.TestCase): @mock.patch('asyncio.base_events.logger') def test_tb_logger_exception_retrieved(self, m_log): - fut = asyncio.Future(loop=self.loop) + fut = self._new_future(loop=self.loop) fut.set_exception(RuntimeError('boom')) fut.exception() del fut @@ -348,7 +352,7 @@ class FutureTests(test_utils.TestCase): @mock.patch('asyncio.base_events.logger') def test_tb_logger_exception_result_retrieved(self, m_log): - fut = asyncio.Future(loop=self.loop) + fut = self._new_future(loop=self.loop) fut.set_exception(RuntimeError('boom')) self.assertRaises(RuntimeError, fut.result) del fut @@ -362,12 +366,12 @@ class FutureTests(test_utils.TestCase): f1 = ex.submit(run, 'oi') f2 = asyncio.wrap_future(f1, loop=self.loop) res, ident = self.loop.run_until_complete(f2) - self.assertIsInstance(f2, asyncio.Future) + self.assertTrue(asyncio.isfuture(f2)) self.assertEqual(res, 'oi') self.assertNotEqual(ident, threading.get_ident()) def test_wrap_future_future(self): - f1 = asyncio.Future(loop=self.loop) + f1 = self._new_future(loop=self.loop) f2 = asyncio.wrap_future(f1) self.assertIs(f1, f2) @@ -402,10 +406,10 @@ class FutureTests(test_utils.TestCase): def test_future_source_traceback(self): self.loop.set_debug(True) - future = asyncio.Future(loop=self.loop) + future = self._new_future(loop=self.loop) lineno = sys._getframe().f_lineno - 1 self.assertIsInstance(future._source_traceback, list) - self.assertEqual(future._source_traceback[-1][:3], + self.assertEqual(future._source_traceback[-2][:3], (__file__, lineno, 'test_future_source_traceback')) @@ -421,57 +425,18 @@ class FutureTests(test_utils.TestCase): return exc exc = memory_error() - future = asyncio.Future(loop=self.loop) - if debug: - source_traceback = future._source_traceback + future = self._new_future(loop=self.loop) future.set_exception(exc) future = None test_utils.run_briefly(self.loop) support.gc_collect() if sys.version_info >= (3, 4): - if debug: - frame = source_traceback[-1] - regex = (r'^Future exception was never retrieved\n' - r'future: <Future finished exception=MemoryError\(\) ' - r'created at {filename}:{lineno}>\n' - r'source_traceback: Object ' - r'created at \(most recent call last\):\n' - r' File' - r'.*\n' - r' File "{filename}", line {lineno}, ' - r'in check_future_exception_never_retrieved\n' - r' future = asyncio\.Future\(loop=self\.loop\)$' - ).format(filename=re.escape(frame[0]), - lineno=frame[1]) - else: - regex = (r'^Future exception was never retrieved\n' - r'future: ' - r'<Future finished exception=MemoryError\(\)>$' - ) + regex = r'^Future exception was never retrieved\n' exc_info = (type(exc), exc, exc.__traceback__) m_log.error.assert_called_once_with(mock.ANY, exc_info=exc_info) else: - if debug: - frame = source_traceback[-1] - regex = (r'^Future/Task exception was never retrieved\n' - r'Future/Task created at \(most recent call last\):\n' - r' File' - r'.*\n' - r' File "{filename}", line {lineno}, ' - r'in check_future_exception_never_retrieved\n' - r' future = asyncio\.Future\(loop=self\.loop\)\n' - r'Traceback \(most recent call last\):\n' - r'.*\n' - r'MemoryError$' - ).format(filename=re.escape(frame[0]), - lineno=frame[1]) - else: - regex = (r'^Future/Task exception was never retrieved\n' - r'Traceback \(most recent call last\):\n' - r'.*\n' - r'MemoryError$' - ) + regex = r'^Future/Task exception was never retrieved\n' m_log.error.assert_called_once_with(mock.ANY, exc_info=False) message = m_log.error.call_args[0][0] self.assertRegex(message, re.compile(regex, re.DOTALL)) @@ -483,14 +448,49 @@ class FutureTests(test_utils.TestCase): self.check_future_exception_never_retrieved(True) def test_set_result_unless_cancelled(self): - from asyncio import futures - fut = asyncio.Future(loop=self.loop) + fut = self._new_future(loop=self.loop) fut.cancel() futures._set_result_unless_cancelled(fut, 2) self.assertTrue(fut.cancelled()) + def test_future_stop_iteration_args(self): + fut = self._new_future(loop=self.loop) + fut.set_result((1, 2)) + fi = fut.__iter__() + result = None + try: + fi.send(None) + except StopIteration as ex: + result = ex.args[0] + else: + self.fail('StopIteration was expected') + self.assertEqual(result, (1, 2)) + + def test_future_iter_throw(self): + fut = self._new_future(loop=self.loop) + fi = iter(fut) + self.assertRaises(TypeError, fi.throw, + Exception, Exception("elephant"), 32) + self.assertRaises(TypeError, fi.throw, + Exception("elephant"), Exception("elephant")) + self.assertRaises(TypeError, fi.throw, list) + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CFutureTests(BaseFutureTests, test_utils.TestCase): + + def _new_future(self, *args, **kwargs): + return futures._CFuture(*args, **kwargs) + -class FutureDoneCallbackTests(test_utils.TestCase): +class PyFutureTests(BaseFutureTests, test_utils.TestCase): + + def _new_future(self, *args, **kwargs): + return futures._PyFuture(*args, **kwargs) + + +class BaseFutureDoneCallbackTests(): def setUp(self): super().setUp() @@ -506,7 +506,7 @@ class FutureDoneCallbackTests(test_utils.TestCase): return bag_appender def _new_future(self): - return asyncio.Future(loop=self.loop) + raise NotImplementedError def test_callbacks_invoked_on_set_result(self): bag = [] @@ -570,5 +570,21 @@ class FutureDoneCallbackTests(test_utils.TestCase): self.assertEqual(f.result(), 'foo') +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CFutureDoneCallbackTests(BaseFutureDoneCallbackTests, + test_utils.TestCase): + + def _new_future(self): + return futures._CFuture(loop=self.loop) + + +class PyFutureDoneCallbackTests(BaseFutureDoneCallbackTests, + test_utils.TestCase): + + def _new_future(self): + return futures._PyFuture(loop=self.loop) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 9872926739..a18d49ae37 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1,5 +1,6 @@ """Tests for tasks.py.""" +import collections import contextlib import functools import io @@ -14,6 +15,8 @@ from unittest import mock import asyncio from asyncio import coroutines +from asyncio import futures +from asyncio import tasks from asyncio import test_utils try: from test import support @@ -72,15 +75,26 @@ class Dummy: pass -class TaskTests(test_utils.TestCase): +class BaseTaskTests: + + Task = None + Future = None + + def new_task(self, loop, coro): + return self.__class__.Task(coro, loop=loop) + + def new_future(self, loop): + return self.__class__.Future(loop=loop) def setUp(self): super().setUp() self.loop = self.new_test_loop() + self.loop.set_task_factory(self.new_task) + self.loop.create_future = lambda: self.new_future(self.loop) def test_other_loop_future(self): other_loop = asyncio.new_event_loop() - fut = asyncio.Future(loop=other_loop) + fut = self.new_future(other_loop) @asyncio.coroutine def run(fut): @@ -108,7 +122,7 @@ class TaskTests(test_utils.TestCase): @asyncio.coroutine def notmuch(): return 'ok' - t = asyncio.Task(notmuch(), loop=self.loop) + t = self.new_task(self.loop, notmuch()) self.loop.run_until_complete(t) self.assertTrue(t.done()) self.assertEqual(t.result(), 'ok') @@ -116,7 +130,7 @@ class TaskTests(test_utils.TestCase): loop = asyncio.new_event_loop() self.set_event_loop(loop) - t = asyncio.Task(notmuch(), loop=loop) + t = self.new_task(loop, notmuch()) self.assertIs(t._loop, loop) loop.run_until_complete(t) loop.close() @@ -139,7 +153,7 @@ class TaskTests(test_utils.TestCase): loop.close() def test_ensure_future_future(self): - f_orig = asyncio.Future(loop=self.loop) + f_orig = self.new_future(self.loop) f_orig.set_result('ko') f = asyncio.ensure_future(f_orig) @@ -163,7 +177,7 @@ class TaskTests(test_utils.TestCase): @asyncio.coroutine def notmuch(): return 'ok' - t_orig = asyncio.Task(notmuch(), loop=self.loop) + t_orig = self.new_task(self.loop, notmuch()) t = asyncio.ensure_future(t_orig) self.loop.run_until_complete(t) self.assertTrue(t.done()) @@ -204,7 +218,7 @@ class TaskTests(test_utils.TestCase): asyncio.ensure_future('ok') def test_async_warning(self): - f = asyncio.Future(loop=self.loop) + f = self.new_future(self.loop) with self.assertWarnsRegex(DeprecationWarning, 'function is deprecated, use ensure_'): self.assertIs(f, asyncio.async(f)) @@ -251,8 +265,8 @@ class TaskTests(test_utils.TestCase): # test coroutine function self.assertEqual(notmuch.__name__, 'notmuch') if PY35: - self.assertEqual(notmuch.__qualname__, - 'TaskTests.test_task_repr.<locals>.notmuch') + self.assertRegex(notmuch.__qualname__, + r'\w+.test_task_repr.<locals>.notmuch') self.assertEqual(notmuch.__module__, __name__) filename, lineno = test_utils.get_function_source(notmuch) @@ -261,7 +275,7 @@ class TaskTests(test_utils.TestCase): # test coroutine object gen = notmuch() if coroutines._DEBUG or PY35: - coro_qualname = 'TaskTests.test_task_repr.<locals>.notmuch' + coro_qualname = 'BaseTaskTests.test_task_repr.<locals>.notmuch' else: coro_qualname = 'notmuch' self.assertEqual(gen.__name__, 'notmuch') @@ -270,7 +284,7 @@ class TaskTests(test_utils.TestCase): coro_qualname) # test pending Task - t = asyncio.Task(gen, loop=self.loop) + t = self.new_task(self.loop, gen) t.add_done_callback(Dummy()) coro = format_coroutine(coro_qualname, 'running', src, @@ -292,7 +306,7 @@ class TaskTests(test_utils.TestCase): '<Task cancelled %s>' % coro) # test finished Task - t = asyncio.Task(notmuch(), loop=self.loop) + t = self.new_task(self.loop, notmuch()) self.loop.run_until_complete(t) coro = format_coroutine(coro_qualname, 'done', src, t._source_traceback) @@ -311,9 +325,9 @@ class TaskTests(test_utils.TestCase): # test coroutine function self.assertEqual(notmuch.__name__, 'notmuch') if PY35: - self.assertEqual(notmuch.__qualname__, - 'TaskTests.test_task_repr_coro_decorator' - '.<locals>.notmuch') + self.assertRegex(notmuch.__qualname__, + r'\w+.test_task_repr_coro_decorator' + r'\.<locals>\.notmuch') self.assertEqual(notmuch.__module__, __name__) # test coroutine object @@ -323,7 +337,7 @@ class TaskTests(test_utils.TestCase): # function, as expected, and have a qualified name (__qualname__ # attribute). coro_name = 'notmuch' - coro_qualname = ('TaskTests.test_task_repr_coro_decorator' + coro_qualname = ('BaseTaskTests.test_task_repr_coro_decorator' '.<locals>.notmuch') else: # On Python < 3.5, generators inherit the name of the code, not of @@ -351,7 +365,7 @@ class TaskTests(test_utils.TestCase): self.assertEqual(repr(gen), '<CoroWrapper %s>' % coro) # test pending Task - t = asyncio.Task(gen, loop=self.loop) + t = self.new_task(self.loop, gen) t.add_done_callback(Dummy()) # format the coroutine object @@ -374,8 +388,8 @@ class TaskTests(test_utils.TestCase): def wait_for(fut): return (yield from fut) - fut = asyncio.Future(loop=self.loop) - task = asyncio.Task(wait_for(fut), loop=self.loop) + fut = self.new_future(self.loop) + task = self.new_task(self.loop, wait_for(fut)) test_utils.run_briefly(self.loop) self.assertRegex(repr(task), '<Task .* wait_for=%s>' % re.escape(repr(fut))) @@ -401,10 +415,11 @@ class TaskTests(test_utils.TestCase): self.addCleanup(task._coro.close) coro_repr = repr(task._coro) - expected = ('<CoroWrapper TaskTests.test_task_repr_partial_corowrapper' - '.<locals>.func(1)() running, ') - self.assertTrue(coro_repr.startswith(expected), - coro_repr) + expected = ( + r'<CoroWrapper \w+.test_task_repr_partial_corowrapper' + r'\.<locals>\.func\(1\)\(\) running, ' + ) + self.assertRegex(coro_repr, expected) def test_task_basics(self): @asyncio.coroutine @@ -438,7 +453,7 @@ class TaskTests(test_utils.TestCase): yield from asyncio.sleep(10.0, loop=loop) return 12 - t = asyncio.Task(task(), loop=loop) + t = self.new_task(loop, task()) loop.call_soon(t.cancel) with self.assertRaises(asyncio.CancelledError): loop.run_until_complete(t) @@ -453,7 +468,7 @@ class TaskTests(test_utils.TestCase): yield return 12 - t = asyncio.Task(task(), loop=self.loop) + t = self.new_task(self.loop, task()) test_utils.run_briefly(self.loop) # start coro t.cancel() self.assertRaises( @@ -463,14 +478,14 @@ class TaskTests(test_utils.TestCase): self.assertFalse(t.cancel()) def test_cancel_inner_future(self): - f = asyncio.Future(loop=self.loop) + f = self.new_future(self.loop) @asyncio.coroutine def task(): yield from f return 12 - t = asyncio.Task(task(), loop=self.loop) + t = self.new_task(self.loop, task()) test_utils.run_briefly(self.loop) # start task f.cancel() with self.assertRaises(asyncio.CancelledError): @@ -479,14 +494,14 @@ class TaskTests(test_utils.TestCase): self.assertTrue(t.cancelled()) def test_cancel_both_task_and_inner_future(self): - f = asyncio.Future(loop=self.loop) + f = self.new_future(self.loop) @asyncio.coroutine def task(): yield from f return 12 - t = asyncio.Task(task(), loop=self.loop) + t = self.new_task(self.loop, task()) test_utils.run_briefly(self.loop) f.cancel() @@ -500,8 +515,8 @@ class TaskTests(test_utils.TestCase): self.assertTrue(t.cancelled()) def test_cancel_task_catching(self): - fut1 = asyncio.Future(loop=self.loop) - fut2 = asyncio.Future(loop=self.loop) + fut1 = self.new_future(self.loop) + fut2 = self.new_future(self.loop) @asyncio.coroutine def task(): @@ -511,7 +526,7 @@ class TaskTests(test_utils.TestCase): except asyncio.CancelledError: return 42 - t = asyncio.Task(task(), loop=self.loop) + t = self.new_task(self.loop, task()) test_utils.run_briefly(self.loop) self.assertIs(t._fut_waiter, fut1) # White-box test. fut1.set_result(None) @@ -524,9 +539,9 @@ class TaskTests(test_utils.TestCase): self.assertFalse(t.cancelled()) def test_cancel_task_ignoring(self): - fut1 = asyncio.Future(loop=self.loop) - fut2 = asyncio.Future(loop=self.loop) - fut3 = asyncio.Future(loop=self.loop) + fut1 = self.new_future(self.loop) + fut2 = self.new_future(self.loop) + fut3 = self.new_future(self.loop) @asyncio.coroutine def task(): @@ -538,7 +553,7 @@ class TaskTests(test_utils.TestCase): res = yield from fut3 return res - t = asyncio.Task(task(), loop=self.loop) + t = self.new_task(self.loop, task()) test_utils.run_briefly(self.loop) self.assertIs(t._fut_waiter, fut1) # White-box test. fut1.set_result(None) @@ -566,7 +581,7 @@ class TaskTests(test_utils.TestCase): yield from asyncio.sleep(100, loop=loop) return 12 - t = asyncio.Task(task(), loop=loop) + t = self.new_task(loop, task()) self.assertRaises( asyncio.CancelledError, loop.run_until_complete, t) self.assertTrue(t.done()) @@ -599,7 +614,7 @@ class TaskTests(test_utils.TestCase): if x == 2: loop.stop() - t = asyncio.Task(task(), loop=loop) + t = self.new_task(loop, task()) with self.assertRaises(RuntimeError) as cm: loop.run_until_complete(t) self.assertEqual(str(cm.exception), @@ -637,7 +652,7 @@ class TaskTests(test_utils.TestCase): foo_running = False return 'done' - fut = asyncio.Task(foo(), loop=loop) + fut = self.new_task(loop, foo()) with self.assertRaises(asyncio.TimeoutError): loop.run_until_complete(asyncio.wait_for(fut, 0.1, loop=loop)) @@ -677,7 +692,7 @@ class TaskTests(test_utils.TestCase): asyncio.set_event_loop(loop) try: - fut = asyncio.Task(foo(), loop=loop) + fut = self.new_task(loop, foo()) with self.assertRaises(asyncio.TimeoutError): loop.run_until_complete(asyncio.wait_for(fut, 0.01)) finally: @@ -696,7 +711,7 @@ class TaskTests(test_utils.TestCase): loop = self.new_test_loop(gen) - fut = asyncio.Future(loop=loop) + fut = self.new_future(loop) task = asyncio.wait_for(fut, timeout=0.2, loop=loop) loop.call_later(0.1, fut.set_result, "ok") res = loop.run_until_complete(task) @@ -713,8 +728,8 @@ class TaskTests(test_utils.TestCase): loop = self.new_test_loop(gen) - a = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop) - b = asyncio.Task(asyncio.sleep(0.15, loop=loop), loop=loop) + a = self.new_task(loop, asyncio.sleep(0.1, loop=loop)) + b = self.new_task(loop, asyncio.sleep(0.15, loop=loop)) @asyncio.coroutine def foo(): @@ -723,12 +738,12 @@ class TaskTests(test_utils.TestCase): self.assertEqual(pending, set()) return 42 - res = loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + res = loop.run_until_complete(self.new_task(loop, foo())) self.assertEqual(res, 42) self.assertAlmostEqual(0.15, loop.time()) # Doing it again should take no time and exercise a different path. - res = loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + res = loop.run_until_complete(self.new_task(loop, foo())) self.assertAlmostEqual(0.15, loop.time()) self.assertEqual(res, 42) @@ -743,8 +758,8 @@ class TaskTests(test_utils.TestCase): loop = self.new_test_loop(gen) - a = asyncio.Task(asyncio.sleep(0.01, loop=loop), loop=loop) - b = asyncio.Task(asyncio.sleep(0.015, loop=loop), loop=loop) + a = self.new_task(loop, asyncio.sleep(0.01, loop=loop)) + b = self.new_task(loop, asyncio.sleep(0.015, loop=loop)) @asyncio.coroutine def foo(): @@ -755,7 +770,7 @@ class TaskTests(test_utils.TestCase): asyncio.set_event_loop(loop) res = loop.run_until_complete( - asyncio.Task(foo(), loop=loop)) + self.new_task(loop, foo())) self.assertEqual(res, 42) @@ -765,9 +780,9 @@ class TaskTests(test_utils.TestCase): return s c = coro('test') - task = asyncio.Task( - asyncio.wait([c, c, coro('spam')], loop=self.loop), - loop=self.loop) + task =self.new_task( + self.loop, + asyncio.wait([c, c, coro('spam')], loop=self.loop)) done, pending = self.loop.run_until_complete(task) @@ -798,12 +813,12 @@ class TaskTests(test_utils.TestCase): loop = self.new_test_loop(gen) - a = asyncio.Task(asyncio.sleep(10.0, loop=loop), loop=loop) - b = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop) - task = asyncio.Task( + a = self.new_task(loop, asyncio.sleep(10.0, loop=loop)) + b = self.new_task(loop, asyncio.sleep(0.1, loop=loop)) + task = self.new_task( + loop, asyncio.wait([b, a], return_when=asyncio.FIRST_COMPLETED, - loop=loop), - loop=loop) + loop=loop)) done, pending = loop.run_until_complete(task) self.assertEqual({b}, done) @@ -830,12 +845,12 @@ class TaskTests(test_utils.TestCase): yield yield - a = asyncio.Task(coro1(), loop=self.loop) - b = asyncio.Task(coro2(), loop=self.loop) - task = asyncio.Task( + a = self.new_task(self.loop, coro1()) + b = self.new_task(self.loop, coro2()) + task = self.new_task( + self.loop, asyncio.wait([b, a], return_when=asyncio.FIRST_COMPLETED, - loop=self.loop), - loop=self.loop) + loop=self.loop)) done, pending = self.loop.run_until_complete(task) self.assertEqual({a, b}, done) @@ -854,17 +869,17 @@ class TaskTests(test_utils.TestCase): loop = self.new_test_loop(gen) # first_exception, task already has exception - a = asyncio.Task(asyncio.sleep(10.0, loop=loop), loop=loop) + a = self.new_task(loop, asyncio.sleep(10.0, loop=loop)) @asyncio.coroutine def exc(): raise ZeroDivisionError('err') - b = asyncio.Task(exc(), loop=loop) - task = asyncio.Task( + b = self.new_task(loop, exc()) + task = self.new_task( + loop, asyncio.wait([b, a], return_when=asyncio.FIRST_EXCEPTION, - loop=loop), - loop=loop) + loop=loop)) done, pending = loop.run_until_complete(task) self.assertEqual({b}, done) @@ -887,14 +902,14 @@ class TaskTests(test_utils.TestCase): loop = self.new_test_loop(gen) # first_exception, exception during waiting - a = asyncio.Task(asyncio.sleep(10.0, loop=loop), loop=loop) + a = self.new_task(loop, asyncio.sleep(10.0, loop=loop)) @asyncio.coroutine def exc(): yield from asyncio.sleep(0.01, loop=loop) raise ZeroDivisionError('err') - b = asyncio.Task(exc(), loop=loop) + b = self.new_task(loop, exc()) task = asyncio.wait([b, a], return_when=asyncio.FIRST_EXCEPTION, loop=loop) @@ -918,14 +933,14 @@ class TaskTests(test_utils.TestCase): loop = self.new_test_loop(gen) - a = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop) + a = self.new_task(loop, asyncio.sleep(0.1, loop=loop)) @asyncio.coroutine def sleeper(): yield from asyncio.sleep(0.15, loop=loop) raise ZeroDivisionError('really') - b = asyncio.Task(sleeper(), loop=loop) + b = self.new_task(loop, sleeper()) @asyncio.coroutine def foo(): @@ -935,10 +950,10 @@ class TaskTests(test_utils.TestCase): errors = set(f for f in done if f.exception() is not None) self.assertEqual(len(errors), 1) - loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + loop.run_until_complete(self.new_task(loop, foo())) self.assertAlmostEqual(0.15, loop.time()) - loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + loop.run_until_complete(self.new_task(loop, foo())) self.assertAlmostEqual(0.15, loop.time()) def test_wait_with_timeout(self): @@ -954,8 +969,8 @@ class TaskTests(test_utils.TestCase): loop = self.new_test_loop(gen) - a = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop) - b = asyncio.Task(asyncio.sleep(0.15, loop=loop), loop=loop) + a = self.new_task(loop, asyncio.sleep(0.1, loop=loop)) + b = self.new_task(loop, asyncio.sleep(0.15, loop=loop)) @asyncio.coroutine def foo(): @@ -964,7 +979,7 @@ class TaskTests(test_utils.TestCase): self.assertEqual(done, set([a])) self.assertEqual(pending, set([b])) - loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + loop.run_until_complete(self.new_task(loop, foo())) self.assertAlmostEqual(0.11, loop.time()) # move forward to close generator @@ -984,8 +999,8 @@ class TaskTests(test_utils.TestCase): loop = self.new_test_loop(gen) - a = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop) - b = asyncio.Task(asyncio.sleep(0.15, loop=loop), loop=loop) + a = self.new_task(loop, asyncio.sleep(0.1, loop=loop)) + b = self.new_task(loop, asyncio.sleep(0.15, loop=loop)) done, pending = loop.run_until_complete( asyncio.wait([b, a], timeout=0.1, loop=loop)) @@ -1033,14 +1048,14 @@ class TaskTests(test_utils.TestCase): values.append((yield from f)) return values - res = loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + res = loop.run_until_complete(self.new_task(loop, foo())) self.assertAlmostEqual(0.15, loop.time()) self.assertTrue('a' in res[:2]) self.assertTrue('b' in res[:2]) self.assertEqual(res[2], 'c') # Doing it again should take no time and exercise a different path. - res = loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + res = loop.run_until_complete(self.new_task(loop, foo())) self.assertAlmostEqual(0.15, loop.time()) def test_as_completed_with_timeout(self): @@ -1069,7 +1084,7 @@ class TaskTests(test_utils.TestCase): values.append((2, exc)) return values - res = loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + res = loop.run_until_complete(self.new_task(loop, foo())) self.assertEqual(len(res), 2, res) self.assertEqual(res[0], (1, 'a')) self.assertEqual(res[1][0], 2) @@ -1097,7 +1112,7 @@ class TaskTests(test_utils.TestCase): v = yield from f self.assertEqual(v, 'a') - loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + loop.run_until_complete(self.new_task(loop, foo())) def test_as_completed_reverse_wait(self): @@ -1157,7 +1172,7 @@ class TaskTests(test_utils.TestCase): result.append((yield from f)) return result - fut = asyncio.Task(runner(), loop=self.loop) + fut = self.new_task(self.loop, runner()) self.loop.run_until_complete(fut) result = fut.result() self.assertEqual(set(result), {'ham', 'spam'}) @@ -1180,7 +1195,7 @@ class TaskTests(test_utils.TestCase): res = yield from asyncio.sleep(dt/2, arg, loop=loop) return res - t = asyncio.Task(sleeper(0.1, 'yeah'), loop=loop) + t = self.new_task(loop, sleeper(0.1, 'yeah')) loop.run_until_complete(t) self.assertTrue(t.done()) self.assertEqual(t.result(), 'yeah') @@ -1195,8 +1210,7 @@ class TaskTests(test_utils.TestCase): loop = self.new_test_loop(gen) - t = asyncio.Task(asyncio.sleep(10.0, 'yeah', loop=loop), - loop=loop) + t = self.new_task(loop, asyncio.sleep(10.0, 'yeah', loop=loop)) handle = None orig_call_later = loop.call_later @@ -1232,7 +1246,7 @@ class TaskTests(test_utils.TestCase): @asyncio.coroutine def doit(): - sleeper = asyncio.Task(sleep(5000), loop=loop) + sleeper = self.new_task(loop, sleep(5000)) loop.call_later(0.1, sleeper.cancel) try: yield from sleeper @@ -1246,13 +1260,13 @@ class TaskTests(test_utils.TestCase): self.assertAlmostEqual(0.1, loop.time()) def test_task_cancel_waiter_future(self): - fut = asyncio.Future(loop=self.loop) + fut = self.new_future(self.loop) @asyncio.coroutine def coro(): yield from fut - task = asyncio.Task(coro(), loop=self.loop) + task = self.new_task(self.loop, coro()) test_utils.run_briefly(self.loop) self.assertIs(task._fut_waiter, fut) @@ -1269,7 +1283,7 @@ class TaskTests(test_utils.TestCase): return 'ko' gen = notmuch() - task = asyncio.Task(gen, loop=self.loop) + task = self.new_task(self.loop, gen) task.set_result('ok') self.assertRaises(AssertionError, task._step) @@ -1305,7 +1319,7 @@ class TaskTests(test_utils.TestCase): nonlocal result result = yield from fut - t = asyncio.Task(wait_for_future(), loop=self.loop) + t = self.new_task(self.loop, wait_for_future()) test_utils.run_briefly(self.loop) self.assertTrue(fut.cb_added) @@ -1321,7 +1335,7 @@ class TaskTests(test_utils.TestCase): def notmutch(): raise BaseException() - task = asyncio.Task(notmutch(), loop=self.loop) + task = self.new_task(self.loop, notmutch()) self.assertRaises(BaseException, task._step) self.assertTrue(task.done()) @@ -1349,7 +1363,7 @@ class TaskTests(test_utils.TestCase): except asyncio.CancelledError: raise base_exc - task = asyncio.Task(notmutch(), loop=loop) + task = self.new_task(loop, notmutch()) test_utils.run_briefly(loop) task.cancel() @@ -1379,7 +1393,7 @@ class TaskTests(test_utils.TestCase): self.assertFalse(asyncio.iscoroutinefunction(mock.Mock())) def test_yield_vs_yield_from(self): - fut = asyncio.Future(loop=self.loop) + fut = self.new_future(self.loop) @asyncio.coroutine def wait_for_future(): @@ -1423,7 +1437,7 @@ class TaskTests(test_utils.TestCase): self.assertEqual(res, 'test') def test_coroutine_non_gen_function_return_future(self): - fut = asyncio.Future(loop=self.loop) + fut = self.new_future(self.loop) @asyncio.coroutine def func(): @@ -1433,49 +1447,53 @@ class TaskTests(test_utils.TestCase): def coro(): fut.set_result('test') - t1 = asyncio.Task(func(), loop=self.loop) - t2 = asyncio.Task(coro(), loop=self.loop) + t1 = self.new_task(self.loop, func()) + t2 = self.new_task(self.loop, coro()) res = self.loop.run_until_complete(t1) self.assertEqual(res, 'test') self.assertIsNone(t2.result()) def test_current_task(self): - self.assertIsNone(asyncio.Task.current_task(loop=self.loop)) + Task = self.__class__.Task + + self.assertIsNone(Task.current_task(loop=self.loop)) @asyncio.coroutine def coro(loop): - self.assertTrue(asyncio.Task.current_task(loop=loop) is task) + self.assertTrue(Task.current_task(loop=loop) is task) - task = asyncio.Task(coro(self.loop), loop=self.loop) + task = self.new_task(self.loop, coro(self.loop)) self.loop.run_until_complete(task) - self.assertIsNone(asyncio.Task.current_task(loop=self.loop)) + self.assertIsNone(Task.current_task(loop=self.loop)) def test_current_task_with_interleaving_tasks(self): - self.assertIsNone(asyncio.Task.current_task(loop=self.loop)) + Task = self.__class__.Task - fut1 = asyncio.Future(loop=self.loop) - fut2 = asyncio.Future(loop=self.loop) + self.assertIsNone(Task.current_task(loop=self.loop)) + + fut1 = self.new_future(self.loop) + fut2 = self.new_future(self.loop) @asyncio.coroutine def coro1(loop): - self.assertTrue(asyncio.Task.current_task(loop=loop) is task1) + self.assertTrue(Task.current_task(loop=loop) is task1) yield from fut1 - self.assertTrue(asyncio.Task.current_task(loop=loop) is task1) + self.assertTrue(Task.current_task(loop=loop) is task1) fut2.set_result(True) @asyncio.coroutine def coro2(loop): - self.assertTrue(asyncio.Task.current_task(loop=loop) is task2) + self.assertTrue(Task.current_task(loop=loop) is task2) fut1.set_result(True) yield from fut2 - self.assertTrue(asyncio.Task.current_task(loop=loop) is task2) + self.assertTrue(Task.current_task(loop=loop) is task2) - task1 = asyncio.Task(coro1(self.loop), loop=self.loop) - task2 = asyncio.Task(coro2(self.loop), loop=self.loop) + task1 = self.new_task(self.loop, coro1(self.loop)) + task2 = self.new_task(self.loop, coro2(self.loop)) self.loop.run_until_complete(asyncio.wait((task1, task2), loop=self.loop)) - self.assertIsNone(asyncio.Task.current_task(loop=self.loop)) + self.assertIsNone(Task.current_task(loop=self.loop)) # Some thorough tests for cancellation propagation through # coroutines, tasks and wait(). @@ -1483,7 +1501,7 @@ class TaskTests(test_utils.TestCase): def test_yield_future_passes_cancel(self): # Cancelling outer() cancels inner() cancels waiter. proof = 0 - waiter = asyncio.Future(loop=self.loop) + waiter = self.new_future(self.loop) @asyncio.coroutine def inner(): @@ -1517,7 +1535,7 @@ class TaskTests(test_utils.TestCase): # Cancelling outer() makes wait() return early, leaves inner() # running. proof = 0 - waiter = asyncio.Future(loop=self.loop) + waiter = self.new_future(self.loop) @asyncio.coroutine def inner(): @@ -1541,14 +1559,14 @@ class TaskTests(test_utils.TestCase): self.assertEqual(proof, 1) def test_shield_result(self): - inner = asyncio.Future(loop=self.loop) + inner = self.new_future(self.loop) outer = asyncio.shield(inner) inner.set_result(42) res = self.loop.run_until_complete(outer) self.assertEqual(res, 42) def test_shield_exception(self): - inner = asyncio.Future(loop=self.loop) + inner = self.new_future(self.loop) outer = asyncio.shield(inner) test_utils.run_briefly(self.loop) exc = RuntimeError('expected') @@ -1557,7 +1575,7 @@ class TaskTests(test_utils.TestCase): self.assertIs(outer.exception(), exc) def test_shield_cancel(self): - inner = asyncio.Future(loop=self.loop) + inner = self.new_future(self.loop) outer = asyncio.shield(inner) test_utils.run_briefly(self.loop) inner.cancel() @@ -1565,7 +1583,7 @@ class TaskTests(test_utils.TestCase): self.assertTrue(outer.cancelled()) def test_shield_shortcut(self): - fut = asyncio.Future(loop=self.loop) + fut = self.new_future(self.loop) fut.set_result(42) res = self.loop.run_until_complete(asyncio.shield(fut)) self.assertEqual(res, 42) @@ -1573,7 +1591,7 @@ class TaskTests(test_utils.TestCase): def test_shield_effect(self): # Cancelling outer() does not affect inner(). proof = 0 - waiter = asyncio.Future(loop=self.loop) + waiter = self.new_future(self.loop) @asyncio.coroutine def inner(): @@ -1597,8 +1615,8 @@ class TaskTests(test_utils.TestCase): self.assertEqual(proof, 1) def test_shield_gather(self): - child1 = asyncio.Future(loop=self.loop) - child2 = asyncio.Future(loop=self.loop) + child1 = self.new_future(self.loop) + child2 = self.new_future(self.loop) parent = asyncio.gather(child1, child2, loop=self.loop) outer = asyncio.shield(parent, loop=self.loop) test_utils.run_briefly(self.loop) @@ -1611,8 +1629,8 @@ class TaskTests(test_utils.TestCase): self.assertEqual(parent.result(), [1, 2]) def test_gather_shield(self): - child1 = asyncio.Future(loop=self.loop) - child2 = asyncio.Future(loop=self.loop) + child1 = self.new_future(self.loop) + child2 = self.new_future(self.loop) inner1 = asyncio.shield(child1, loop=self.loop) inner2 = asyncio.shield(child2, loop=self.loop) parent = asyncio.gather(inner1, inner2, loop=self.loop) @@ -1628,7 +1646,7 @@ class TaskTests(test_utils.TestCase): test_utils.run_briefly(self.loop) def test_as_completed_invalid_args(self): - fut = asyncio.Future(loop=self.loop) + fut = self.new_future(self.loop) # as_completed() expects a list of futures, not a future instance self.assertRaises(TypeError, self.loop.run_until_complete, @@ -1639,7 +1657,7 @@ class TaskTests(test_utils.TestCase): coro.close() def test_wait_invalid_args(self): - fut = asyncio.Future(loop=self.loop) + fut = self.new_future(self.loop) # wait() expects a list of futures, not a future instance self.assertRaises(TypeError, self.loop.run_until_complete, @@ -1666,7 +1684,7 @@ class TaskTests(test_utils.TestCase): yield from fut # A completed Future used to run the coroutine. - fut = asyncio.Future(loop=self.loop) + fut = self.new_future(self.loop) fut.set_result(None) # Call the coroutine. @@ -1700,15 +1718,15 @@ class TaskTests(test_utils.TestCase): @asyncio.coroutine def t2(): - f = asyncio.Future(loop=self.loop) - asyncio.Task(t3(f), loop=self.loop) + f = self.new_future(self.loop) + self.new_task(self.loop, t3(f)) return (yield from f) @asyncio.coroutine def t3(f): f.set_result((1, 2, 3)) - task = asyncio.Task(t1(), loop=self.loop) + task = self.new_task(self.loop, t1()) val = self.loop.run_until_complete(task) self.assertEqual(val, (1, 2, 3)) @@ -1771,9 +1789,11 @@ class TaskTests(test_utils.TestCase): @unittest.skipUnless(PY34, 'need python 3.4 or later') def test_log_destroyed_pending_task(self): + Task = self.__class__.Task + @asyncio.coroutine def kill_me(loop): - future = asyncio.Future(loop=loop) + future = self.new_future(loop) yield from future # at this point, the only reference to kill_me() task is # the Task._wakeup() method in future._callbacks @@ -1786,7 +1806,7 @@ class TaskTests(test_utils.TestCase): # schedule the task coro = kill_me(self.loop) task = asyncio.ensure_future(coro, loop=self.loop) - self.assertEqual(asyncio.Task.all_tasks(loop=self.loop), {task}) + self.assertEqual(Task.all_tasks(loop=self.loop), {task}) # execute the task so it waits for future self.loop._run_once() @@ -1801,7 +1821,7 @@ class TaskTests(test_utils.TestCase): # no more reference to kill_me() task: the task is destroyed by the GC support.gc_collect() - self.assertEqual(asyncio.Task.all_tasks(loop=self.loop), set()) + self.assertEqual(Task.all_tasks(loop=self.loop), set()) mock_handler.assert_called_with(self.loop, { 'message': 'Task was destroyed but it is pending!', @@ -1866,10 +1886,10 @@ class TaskTests(test_utils.TestCase): def test_task_source_traceback(self): self.loop.set_debug(True) - task = asyncio.Task(coroutine_function(), loop=self.loop) + task = self.new_task(self.loop, coroutine_function()) lineno = sys._getframe().f_lineno - 1 self.assertIsInstance(task._source_traceback, list) - self.assertEqual(task._source_traceback[-1][:3], + self.assertEqual(task._source_traceback[-2][:3], (__file__, lineno, 'test_task_source_traceback')) @@ -1881,7 +1901,7 @@ class TaskTests(test_utils.TestCase): @asyncio.coroutine def blocking_coroutine(): - fut = asyncio.Future(loop=loop) + fut = self.new_future(loop) # Block: fut result is never set yield from fut @@ -1908,7 +1928,7 @@ class TaskTests(test_utils.TestCase): loop = asyncio.new_event_loop() self.addCleanup(loop.close) - fut = asyncio.Future(loop=loop) + fut = self.new_future(loop) # The indirection fut->child_coro is needed since otherwise the # gathering task is done at the same time as the child future def child_coro(): @@ -1932,6 +1952,172 @@ class TaskTests(test_utils.TestCase): self.assertFalse(gather_task.cancelled()) self.assertEqual(gather_task.result(), [42]) + def test_exception_traceback(self): + # See http://bugs.python.org/issue28843 + + @asyncio.coroutine + def foo(): + 1 / 0 + + @asyncio.coroutine + def main(): + task = self.new_task(self.loop, foo()) + yield # skip one loop iteration + self.assertIsNotNone(task.exception().__traceback__) + + self.loop.run_until_complete(main()) + + @mock.patch('asyncio.base_events.logger') + def test_error_in_call_soon(self, m_log): + def call_soon(callback, *args): + raise ValueError + self.loop.call_soon = call_soon + + @asyncio.coroutine + def coro(): + pass + + self.assertFalse(m_log.error.called) + + with self.assertRaises(ValueError): + self.new_task(self.loop, coro()) + + self.assertTrue(m_log.error.called) + message = m_log.error.call_args[0][0] + self.assertIn('Task was destroyed but it is pending', message) + + self.assertEqual(self.Task.all_tasks(self.loop), set()) + + +def add_subclass_tests(cls): + BaseTask = cls.Task + BaseFuture = cls.Future + + if BaseTask is None or BaseFuture is None: + return cls + + class CommonFuture: + def __init__(self, *args, **kwargs): + self.calls = collections.defaultdict(lambda: 0) + super().__init__(*args, **kwargs) + + def _schedule_callbacks(self): + self.calls['_schedule_callbacks'] += 1 + return super()._schedule_callbacks() + + def add_done_callback(self, *args): + self.calls['add_done_callback'] += 1 + return super().add_done_callback(*args) + + class Task(CommonFuture, BaseTask): + def _step(self, *args): + self.calls['_step'] += 1 + return super()._step(*args) + + def _wakeup(self, *args): + self.calls['_wakeup'] += 1 + return super()._wakeup(*args) + + class Future(CommonFuture, BaseFuture): + pass + + def test_subclasses_ctask_cfuture(self): + fut = self.Future(loop=self.loop) + + async def func(): + self.loop.call_soon(lambda: fut.set_result('spam')) + return await fut + + task = self.Task(func(), loop=self.loop) + + result = self.loop.run_until_complete(task) + + self.assertEqual(result, 'spam') + + self.assertEqual( + dict(task.calls), + {'_step': 2, '_wakeup': 1, 'add_done_callback': 1, + '_schedule_callbacks': 1}) + + self.assertEqual( + dict(fut.calls), + {'add_done_callback': 1, '_schedule_callbacks': 1}) + + # Add patched Task & Future back to the test case + cls.Task = Task + cls.Future = Future + + # Add an extra unit-test + cls.test_subclasses_ctask_cfuture = test_subclasses_ctask_cfuture + + # Disable the "test_task_source_traceback" test + # (the test is hardcoded for a particular call stack, which + # is slightly different for Task subclasses) + cls.test_task_source_traceback = None + + return cls + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CTask_CFuture_Tests(BaseTaskTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + Future = getattr(futures, '_CFuture', None) + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +@add_subclass_tests +class CTask_CFuture_SubclassTests(BaseTaskTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + Future = getattr(futures, '_CFuture', None) + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CTask_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + Future = futures._PyFuture + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class PyTask_CFuture_Tests(BaseTaskTests, test_utils.TestCase): + Task = tasks._PyTask + Future = getattr(futures, '_CFuture', None) + + +class PyTask_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): + Task = tasks._PyTask + Future = futures._PyFuture + + +@add_subclass_tests +class PyTask_PyFuture_SubclassTests(BaseTaskTests, test_utils.TestCase): + Task = tasks._PyTask + Future = futures._PyFuture + + +class GenericTaskTests(test_utils.TestCase): + + def test_future_subclass(self): + self.assertTrue(issubclass(asyncio.Task, asyncio.Future)) + + def test_asyncio_module_compiled(self): + # Because of circular imports it's easy to make _asyncio + # module non-importable. This is a simple test that will + # fail on systems where C modules were successfully compiled + # (hence the test for _functools), but _asyncio somehow didn't. + try: + import _functools + except ImportError: + pass + else: + try: + import _asyncio + except ImportError: + self.fail('_asyncio module is missing') + class GatherTestsBase: diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index 38579168cf..51c65737a4 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -11,6 +11,9 @@ import struct from test import support from io import BytesIO +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + try: import threading except ImportError: @@ -92,7 +95,9 @@ def bind_af_aware(sock, addr): if HAS_UNIX_SOCKETS and sock.family == socket.AF_UNIX: # Make sure the path doesn't exist. support.unlink(addr) - sock.bind(addr) + support.bind_unix_socket(sock, addr) + else: + sock.bind(addr) class HelperFunctionTests(unittest.TestCase): diff --git a/Lib/test/test_augassign.py b/Lib/test/test_augassign.py index 5093e9d0f3..5930d9e7a2 100644 --- a/Lib/test/test_augassign.py +++ b/Lib/test/test_augassign.py @@ -83,6 +83,10 @@ class AugAssignTest(unittest.TestCase): def __iadd__(self, val): return aug_test3(self.val + val) + class aug_test4(aug_test3): + """Blocks inheritance, and fallback to __add__""" + __iadd__ = None + x = aug_test(1) y = x x += 10 @@ -106,6 +110,10 @@ class AugAssignTest(unittest.TestCase): self.assertTrue(y is not x) self.assertEqual(x.val, 13) + x = aug_test4(4) + with self.assertRaises(TypeError): + x += 10 + def testCustomMethods2(test_self): output = [] diff --git a/Lib/test/test_pep352.py b/Lib/test/test_baseexception.py index 7c98c460b9..27d514fe2e 100644 --- a/Lib/test/test_pep352.py +++ b/Lib/test/test_baseexception.py @@ -1,6 +1,5 @@ import unittest import builtins -import warnings import os from platform import system as platform_system diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index 40ac9bff2f..6133bbcac5 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -14,7 +14,6 @@ from test.support import bigmemtest, _1G, _2G, _4G import unittest import operator import sys -import functools # These tests all use one of the bigmemtest decorators to indicate how much # memory they use and how much memory they need to be even meaningful. The diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 034245d74e..6b3e437f43 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -159,11 +159,25 @@ class BinASCIITest(unittest.TestCase): # Then calculate the hexbin4 binary-to-ASCII translation rle = binascii.rlecode_hqx(self.data) a = binascii.b2a_hqx(self.type2test(rle)) + b, _ = binascii.a2b_hqx(self.type2test(a)) res = binascii.rledecode_hqx(b) - self.assertEqual(res, self.rawdata) + def test_rle(self): + # test repetition with a repetition longer than the limit of 255 + data = (b'a' * 100 + b'b' + b'c' * 300) + + encoded = binascii.rlecode_hqx(data) + self.assertEqual(encoded, + (b'a\x90d' # 'a' * 100 + b'b' # 'b' + b'c\x90\xff' # 'c' * 255 + b'c\x90-')) # 'c' * 45 + + decoded = binascii.rledecode_hqx(encoded) + self.assertEqual(decoded, data) + def test_hex(self): # test hexlification s = b'{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000' @@ -331,6 +345,16 @@ class BinASCIITest(unittest.TestCase): # non-ASCII string self.assertRaises(ValueError, a2b, "\x80") + def test_b2a_base64_newline(self): + # Issue #25357: test newline parameter + b = self.type2test(b'hello') + self.assertEqual(binascii.b2a_base64(b), + b'aGVsbG8=\n') + self.assertEqual(binascii.b2a_base64(b, newline=True), + b'aGVsbG8=\n') + self.assertEqual(binascii.b2a_base64(b, newline=False), + b'aGVsbG8=') + class ArrayBinASCIITest(BinASCIITest): def type2test(self, s): diff --git a/Lib/test/test_binhex.py b/Lib/test/test_binhex.py index 9d4c85afaa..21f446325a 100644 --- a/Lib/test/test_binhex.py +++ b/Lib/test/test_binhex.py @@ -4,7 +4,6 @@ Based on an original test by Roger E. Masse. """ import binhex -import os import unittest from test import support diff --git a/Lib/test/test_binop.py b/Lib/test/test_binop.py index e9dbddcd93..3ed018e089 100644 --- a/Lib/test/test_binop.py +++ b/Lib/test/test_binop.py @@ -2,7 +2,7 @@ import unittest from test import support -from operator import eq, ne, lt, gt, le, ge +from operator import eq, le, ne from abc import ABCMeta def gcd(a, b): @@ -388,6 +388,54 @@ class OperationOrderTests(unittest.TestCase): self.assertEqual(op_sequence(eq, B, V), ['B.__eq__', 'V.__eq__']) self.assertEqual(op_sequence(le, B, V), ['B.__le__', 'V.__ge__']) +class SupEq(object): + """Class that can test equality""" + def __eq__(self, other): + return True + +class S(SupEq): + """Subclass of SupEq that should fail""" + __eq__ = None + +class F(object): + """Independent class that should fall back""" + +class X(object): + """Independent class that should fail""" + __eq__ = None + +class SN(SupEq): + """Subclass of SupEq that can test equality, but not non-equality""" + __ne__ = None + +class XN: + """Independent class that can test equality, but not non-equality""" + def __eq__(self, other): + return True + __ne__ = None + +class FallbackBlockingTests(unittest.TestCase): + """Unit tests for None method blocking""" + + def test_fallback_rmethod_blocking(self): + e, f, s, x = SupEq(), F(), S(), X() + self.assertEqual(e, e) + self.assertEqual(e, f) + self.assertEqual(f, e) + # left operand is checked first + self.assertEqual(e, x) + self.assertRaises(TypeError, eq, x, e) + # S is a subclass, so it's always checked first + self.assertRaises(TypeError, eq, e, s) + self.assertRaises(TypeError, eq, s, e) + + def test_fallback_ne_blocking(self): + e, sn, xn = SupEq(), SN(), XN() + self.assertFalse(e != e) + self.assertRaises(TypeError, ne, e, sn) + self.assertRaises(TypeError, ne, sn, e) + self.assertFalse(e != xn) + self.assertRaises(TypeError, ne, xn, e) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index d30a3b9c0f..9f8f0e122c 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -96,6 +96,13 @@ class BoolTest(unittest.TestCase): self.assertEqual(False/1, 0) self.assertIsNot(False/1, False) + self.assertEqual(True%1, 0) + self.assertIsNot(True%1, False) + self.assertEqual(True%2, 1) + self.assertIsNot(True%2, True) + self.assertEqual(False%1, 0) + self.assertIsNot(False%1, False) + for b in False, True: for i in 0, 1, 2: self.assertEqual(b**i, int(b)**i) @@ -333,6 +340,17 @@ class BoolTest(unittest.TestCase): except (Exception) as e_len: self.assertEqual(str(e_bool), str(e_len)) + def test_blocked(self): + class A: + __bool__ = None + self.assertRaises(TypeError, bool, A()) + + class B: + def __len__(self): + return 10 + __bool__ = None + self.assertRaises(TypeError, bool, B()) + def test_real_and_imag(self): self.assertEqual(True.real, 1) self.assertEqual(True.imag, 0) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 2eef9fc685..b83f2f107c 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -16,7 +16,6 @@ import unittest from test import support from itertools import permutations, product from random import randrange, sample, choice -from sysconfig import get_config_var import warnings import sys, array, io from decimal import Decimal diff --git a/Lib/test/test_bufio.py b/Lib/test/test_bufio.py index 9931c84680..fea6da491e 100644 --- a/Lib/test/test_bufio.py +++ b/Lib/test/test_bufio.py @@ -59,7 +59,7 @@ class BufferSizeTest: self.drive_one(b"1234567890\00\01\02\03\04\05\06") def test_nullpat(self): - self.drive_one(bytes(1000)) + self.drive_one(b'\0' * 1000) class CBufferSizeTest(BufferSizeTest, unittest.TestCase): diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 0b033408a8..416316c028 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -82,7 +82,7 @@ test_conv_no_sign = [ ('', ValueError), (' ', ValueError), (' \t\t ', ValueError), - (str(b'\u0663\u0661\u0664 ','raw-unicode-escape'), 314), + (str(br'\u0663\u0661\u0664 ','raw-unicode-escape'), 314), (chr(0x200), ValueError), ] @@ -104,7 +104,7 @@ test_conv_sign = [ ('', ValueError), (' ', ValueError), (' \t\t ', ValueError), - (str(b'\u0663\u0661\u0664 ','raw-unicode-escape'), 314), + (str(br'\u0663\u0661\u0664 ','raw-unicode-escape'), 314), (chr(0x200), ValueError), ] @@ -1627,6 +1627,16 @@ class TestSorted(unittest.TestCase): self.assertEqual(data, sorted(copy, reverse=1)) self.assertNotEqual(data, copy) + def test_bad_arguments(self): + # Issue #29327: The first argument is positional-only. + sorted([]) + with self.assertRaises(TypeError): + sorted(iterable=[]) + # Other arguments are keyword-only + sorted([], key=None) + with self.assertRaises(TypeError): + sorted([], None) + def test_inputtypes(self): s = 'abracadabra' types = [list, tuple, str] @@ -1708,21 +1718,11 @@ class TestType(unittest.TestCase): self.assertEqual(x.spam(), 'spam42') self.assertEqual(x.to_bytes(2, 'little'), b'\x2a\x00') - def test_type_new_keywords(self): - class B: - def ham(self): - return 'ham%d' % self - C = type.__new__(type, - name='C', - bases=(B, int), - dict={'spam': lambda self: 'spam%s' % self}) - self.assertEqual(C.__name__, 'C') - self.assertEqual(C.__qualname__, 'C') - self.assertEqual(C.__module__, __name__) - self.assertEqual(C.__bases__, (B, int)) - self.assertIs(C.__base__, int) - self.assertIn('spam', C.__dict__) - self.assertNotIn('ham', C.__dict__) + def test_type_nokwargs(self): + with self.assertRaises(TypeError): + type('a', (), {}, x=5) + with self.assertRaises(TypeError): + type('a', (), dict={}) def test_type_name(self): for name in 'A', '\xc4', '\U0001f40d', 'B.A', '42', '': diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 3288a5be87..a103a7d39c 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -4,6 +4,7 @@ XXX This is a mess. Common tests should be unified with string_tests.py (and the latter should be modernized). """ +import array import os import re import sys @@ -81,6 +82,18 @@ class BaseBytesTest: self.assertRaises(ValueError, self.type2test, [Indexable(-1)]) self.assertRaises(ValueError, self.type2test, [Indexable(256)]) + def test_from_buffer(self): + a = self.type2test(array.array('B', [1, 2, 3])) + self.assertEqual(a, b"\x01\x02\x03") + + # http://bugs.python.org/issue29159 + # Fallback when __index__ raises exception other than OverflowError + class B(bytes): + def __index__(self): + raise TypeError + + self.assertEqual(self.type2test(B(b"foobar")), b"foobar") + def test_from_ssize(self): self.assertEqual(self.type2test(0), b'') self.assertEqual(self.type2test(1), b'\x00') @@ -216,7 +229,7 @@ class BaseBytesTest: self.assertEqual(b, self.type2test(sample[:-3], "utf-8")) def test_decode(self): - sample = "Hello world\n\u1234\u5678\u9abc\def0\def0" + sample = "Hello world\n\u1234\u5678\u9abc" for enc in ("utf-8", "utf-16"): b = self.type2test(sample, enc) self.assertEqual(b.decode(enc), sample) @@ -269,6 +282,7 @@ class BaseBytesTest: self.assertNotIn(200, b) self.assertRaises(ValueError, lambda: 300 in b) self.assertRaises(ValueError, lambda: -1 in b) + self.assertRaises(ValueError, lambda: sys.maxsize+1 in b) self.assertRaises(TypeError, lambda: None in b) self.assertRaises(TypeError, lambda: float(ord('a')) in b) self.assertRaises(TypeError, lambda: "a" in b) @@ -300,6 +314,20 @@ class BaseBytesTest: self.assertRaises(ValueError, self.type2test.fromhex, '\x00') self.assertRaises(ValueError, self.type2test.fromhex, '12 \x00 34') + for data, pos in ( + # invalid first hexadecimal character + ('12 x4 56', 3), + # invalid second hexadecimal character + ('12 3x 56', 4), + # two invalid hexadecimal characters + ('12 xy 56', 3), + # test non-ASCII string + ('12 3\xff 56', 4), + ): + with self.assertRaises(ValueError) as cm: + self.type2test.fromhex(data) + self.assertIn('at position %s' % pos, str(cm.exception)) + def test_hex(self): self.assertRaises(TypeError, self.type2test.hex) self.assertRaises(TypeError, self.type2test.hex, 1) @@ -674,6 +702,37 @@ class BaseBytesTest: test.support.check_free_after_iterating(self, iter, self.type2test) test.support.check_free_after_iterating(self, reversed, self.type2test) + def test_translate(self): + b = self.type2test(b'hello') + rosetta = bytearray(range(256)) + rosetta[ord('o')] = ord('e') + + self.assertRaises(TypeError, b.translate) + self.assertRaises(TypeError, b.translate, None, None) + self.assertRaises(ValueError, b.translate, bytes(range(255))) + + c = b.translate(rosetta, b'hello') + self.assertEqual(b, b'hello') + self.assertIsInstance(c, self.type2test) + + c = b.translate(rosetta) + d = b.translate(rosetta, b'') + self.assertEqual(c, d) + self.assertEqual(c, b'helle') + + c = b.translate(rosetta, b'l') + self.assertEqual(c, b'hee') + c = b.translate(None, b'e') + self.assertEqual(c, b'hllo') + + # test delete as a keyword argument + c = b.translate(rosetta, delete=b'') + self.assertEqual(c, b'helle') + c = b.translate(rosetta, delete=b'l') + self.assertEqual(c, b'hee') + c = b.translate(None, delete=b'e') + self.assertEqual(c, b'hllo') + class BytesTest(BaseBytesTest, unittest.TestCase): type2test = bytes @@ -722,31 +781,142 @@ class BytesTest(BaseBytesTest, unittest.TestCase): # Test PyBytes_FromFormat() def test_from_format(self): - test.support.import_module('ctypes') - from ctypes import pythonapi, py_object, c_int, c_char_p + ctypes = test.support.import_module('ctypes') + _testcapi = test.support.import_module('_testcapi') + from ctypes import pythonapi, py_object + from ctypes import ( + c_int, c_uint, + c_long, c_ulong, + c_size_t, c_ssize_t, + c_char_p) + PyBytes_FromFormat = pythonapi.PyBytes_FromFormat PyBytes_FromFormat.restype = py_object + # basic tests self.assertEqual(PyBytes_FromFormat(b'format'), b'format') - + self.assertEqual(PyBytes_FromFormat(b'Hello %s !', b'world'), + b'Hello world !') + + # test formatters + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(0)), + b'c=\0') + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(ord('@'))), + b'c=@') + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(255)), + b'c=\xff') + self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', + c_int(1), c_long(2), + c_size_t(3)), + b'd=1 ld=2 zd=3') + self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', + c_int(-1), c_long(-2), + c_size_t(-3)), + b'd=-1 ld=-2 zd=-3') + self.assertEqual(PyBytes_FromFormat(b'u=%u lu=%lu zu=%zu', + c_uint(123), c_ulong(456), + c_size_t(789)), + b'u=123 lu=456 zu=789') + self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(123)), + b'i=123') + self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(-123)), + b'i=-123') + self.assertEqual(PyBytes_FromFormat(b'x=%x', c_int(0xabc)), + b'x=abc') + + sizeof_ptr = ctypes.sizeof(c_char_p) + + if os.name == 'nt': + # Windows (MSCRT) + ptr_format = '0x%0{}X'.format(2 * sizeof_ptr) + def ptr_formatter(ptr): + return (ptr_format % ptr) + else: + # UNIX (glibc) + def ptr_formatter(ptr): + return '%#x' % ptr + + ptr = 0xabcdef + self.assertEqual(PyBytes_FromFormat(b'ptr=%p', c_char_p(ptr)), + ('ptr=' + ptr_formatter(ptr)).encode('ascii')) + self.assertEqual(PyBytes_FromFormat(b's=%s', c_char_p(b'cstr')), + b's=cstr') + + # test minimum and maximum integer values + size_max = c_size_t(-1).value + for formatstr, ctypes_type, value, py_formatter in ( + (b'%d', c_int, _testcapi.INT_MIN, str), + (b'%d', c_int, _testcapi.INT_MAX, str), + (b'%ld', c_long, _testcapi.LONG_MIN, str), + (b'%ld', c_long, _testcapi.LONG_MAX, str), + (b'%lu', c_ulong, _testcapi.ULONG_MAX, str), + (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MIN, str), + (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MAX, str), + (b'%zu', c_size_t, size_max, str), + (b'%p', c_char_p, size_max, ptr_formatter), + ): + self.assertEqual(PyBytes_FromFormat(formatstr, ctypes_type(value)), + py_formatter(value).encode('ascii')), + + # width and precision (width is currently ignored) + self.assertEqual(PyBytes_FromFormat(b'%5s', b'a'), + b'a') + self.assertEqual(PyBytes_FromFormat(b'%.3s', b'abcdef'), + b'abc') + + # '%%' formatter + self.assertEqual(PyBytes_FromFormat(b'%%'), + b'%') + self.assertEqual(PyBytes_FromFormat(b'[%%]'), + b'[%]') + self.assertEqual(PyBytes_FromFormat(b'%%%c', c_int(ord('_'))), + b'%_') + self.assertEqual(PyBytes_FromFormat(b'%%s'), + b'%s') + + # Invalid formats and partial formatting self.assertEqual(PyBytes_FromFormat(b'%'), b'%') - self.assertEqual(PyBytes_FromFormat(b'%%'), b'%') - self.assertEqual(PyBytes_FromFormat(b'%%s'), b'%s') - self.assertEqual(PyBytes_FromFormat(b'[%%]'), b'[%]') - self.assertEqual(PyBytes_FromFormat(b'%%%c', c_int(ord('_'))), b'%_') - - self.assertEqual(PyBytes_FromFormat(b'c:%c', c_int(255)), - b'c:\xff') - self.assertEqual(PyBytes_FromFormat(b's:%s', c_char_p(b'cstr')), - b's:cstr') + self.assertEqual(PyBytes_FromFormat(b'x=%i y=%', c_int(2), c_int(3)), + b'x=2 y=%') - # Issue #19969 + # Issue #19969: %c must raise OverflowError for values + # not in the range [0; 255] self.assertRaises(OverflowError, PyBytes_FromFormat, b'%c', c_int(-1)) self.assertRaises(OverflowError, PyBytes_FromFormat, b'%c', c_int(256)) + def test_bytes_blocking(self): + class IterationBlocked(list): + __bytes__ = None + i = [0, 1, 2, 3] + self.assertEqual(bytes(i), b'\x00\x01\x02\x03') + self.assertRaises(TypeError, bytes, IterationBlocked(i)) + + # At least in CPython, because bytes.__new__ and the C API + # PyBytes_FromObject have different fallback rules, integer + # fallback is handled specially, so test separately. + class IntBlocked(int): + __bytes__ = None + self.assertEqual(bytes(3), b'\0\0\0') + self.assertRaises(TypeError, bytes, IntBlocked(3)) + + # While there is no separately-defined rule for handling bytes + # subclasses differently from other buffer-interface classes, + # an implementation may well special-case them (as CPython 2.x + # str did), so test them separately. + class BytesSubclassBlocked(bytes): + __bytes__ = None + self.assertEqual(bytes(b'ab'), b'ab') + self.assertRaises(TypeError, bytes, BytesSubclassBlocked(b'ab')) + + class BufferBlocked(bytearray): + __bytes__ = None + ba, bb = bytearray(b'ab'), BufferBlocked(b'ab') + self.assertEqual(bytes(ba), b'ab') + self.assertRaises(TypeError, bytes, bb) + class ByteArrayTest(BaseBytesTest, unittest.TestCase): type2test = bytearray @@ -1332,24 +1502,6 @@ class AssortedBytesTest(unittest.TestCase): self.assertRaises(SyntaxError, eval, 'b"%s"' % chr(c)) - def test_translate(self): - b = b'hello' - ba = bytearray(b) - rosetta = bytearray(range(0, 256)) - rosetta[ord('o')] = ord('e') - c = b.translate(rosetta, b'l') - self.assertEqual(b, b'hello') - self.assertEqual(c, b'hee') - c = ba.translate(rosetta, b'l') - self.assertEqual(ba, b'hello') - self.assertEqual(c, b'hee') - c = b.translate(None, b'e') - self.assertEqual(c, b'hllo') - c = ba.translate(None, b'e') - self.assertEqual(c, b'hllo') - self.assertRaises(TypeError, b.translate, None, None) - self.assertRaises(TypeError, ba.translate, None, None) - def test_split_bytearray(self): self.assertEqual(b'a b'.split(memoryview(b' ')), [b'a', b'b']) @@ -1513,7 +1665,32 @@ class SubclassTest: self.assertEqual(type(a), type(b)) self.assertEqual(type(a.y), type(b.y)) - test_fromhex = BaseBytesTest.test_fromhex + def test_fromhex(self): + b = self.type2test.fromhex('1a2B30') + self.assertEqual(b, b'\x1a\x2b\x30') + self.assertIs(type(b), self.type2test) + + class B1(self.basetype): + def __new__(cls, value): + me = self.basetype.__new__(cls, value) + me.foo = 'bar' + return me + + b = B1.fromhex('1a2B30') + self.assertEqual(b, b'\x1a\x2b\x30') + self.assertIs(type(b), B1) + self.assertEqual(b.foo, 'bar') + + class B2(self.basetype): + def __init__(me, *args, **kwargs): + if self.basetype is not bytes: + self.basetype.__init__(me, *args, **kwargs) + me.foo = 'bar' + + b = B2.fromhex('1a2B30') + self.assertEqual(b, b'\x1a\x2b\x30') + self.assertIs(type(b), B2) + self.assertEqual(b.foo, 'bar') class ByteArraySubclass(bytearray): diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 34f6478356..eaa472a6cc 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -6,6 +6,7 @@ from io import BytesIO, DEFAULT_BUFFER_SIZE import os import pickle import glob +import pathlib import random import shutil import subprocess @@ -553,13 +554,20 @@ class BZ2FileTest(BaseTest): with BZ2File(str_filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + def testOpenPathLikeFilename(self): + filename = pathlib.Path(self.filename) + with BZ2File(filename, "wb") as f: + f.write(self.DATA) + with BZ2File(filename, "rb") as f: + self.assertEqual(f.read(), self.DATA) + def testDecompressLimited(self): """Decompressed data buffering should be limited""" - bomb = bz2.compress(bytes(int(2e6)), compresslevel=9) + bomb = bz2.compress(b'\0' * int(2e6), compresslevel=9) self.assertLess(len(bomb), _compression.BUFFER_SIZE) decomp = BZ2File(BytesIO(bomb)) - self.assertEqual(bytes(1), decomp.read(1)) + self.assertEqual(decomp.read(1), b'\0') max_decomp = 1 + DEFAULT_BUFFER_SIZE self.assertLessEqual(decomp._buffer.raw.tell(), max_decomp, "Excessive amount of data was decompressed") diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index accf2518ab..2bc4feebbd 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -722,19 +722,19 @@ class CommandLineTestCase(unittest.TestCase): def assertFailure(self, *args): rc, stdout, stderr = assert_python_failure('-m', 'calendar', *args) - self.assertIn(b'Usage:', stderr) + self.assertIn(b'usage:', stderr) self.assertEqual(rc, 2) def test_help(self): stdout = self.run_ok('-h') - self.assertIn(b'Usage:', stdout) + self.assertIn(b'usage:', stdout) self.assertIn(b'calendar.py', stdout) self.assertIn(b'--help', stdout) def test_illegal_arguments(self): self.assertFailure('-z') - #self.assertFailure('spam') - #self.assertFailure('2004', 'spam') + self.assertFailure('spam') + self.assertFailure('2004', 'spam') self.assertFailure('-t', 'html', '2004', '1') def test_output_current_year(self): @@ -835,5 +835,14 @@ class CommandLineTestCase(unittest.TestCase): b'href="custom.css" />', stdout) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'mdays', 'January', 'February', 'EPOCH', + 'MONDAY', 'TUESDAY', 'WEDNESDAY', 'THURSDAY', 'FRIDAY', + 'SATURDAY', 'SUNDAY', 'different_locale', 'c', + 'prweek', 'week', 'format', 'formatstring', 'main'} + support.check__all__(self, calendar, blacklist=blacklist) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 1eadd2249e..6c3625d487 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -4,8 +4,10 @@ import os import pickle import random +import re import subprocess import sys +import sysconfig import textwrap import time import unittest @@ -220,8 +222,8 @@ class CAPITest(unittest.TestCase): br'result with an error set\n' br'ValueError\n' br'\n' - br'During handling of the above exception, ' - br'another exception occurred:\n' + br'The above exception was the direct cause ' + br'of the following exception:\n' br'\n' br'SystemError: <built-in ' br'function return_result_with_error> ' @@ -442,6 +444,7 @@ class EmbeddingTests(unittest.TestCase): self.maxDiff = None self.assertEqual(out.strip(), expected_output) + class SkipitemTest(unittest.TestCase): def test_skipitem(self): @@ -491,10 +494,10 @@ class SkipitemTest(unittest.TestCase): _testcapi.parse_tuple_and_keywords(tuple_1, dict_b, format.encode("ascii"), keywords) when_not_skipped = False - except TypeError as e: + except SystemError as e: s = "argument 1 (impossible<bad format char>)" when_not_skipped = (str(e) == s) - except RuntimeError as e: + except TypeError: when_not_skipped = False # test the format unit when skipped @@ -503,7 +506,7 @@ class SkipitemTest(unittest.TestCase): _testcapi.parse_tuple_and_keywords(empty_tuple, dict_b, optional_format.encode("ascii"), keywords) when_skipped = False - except RuntimeError as e: + except SystemError as e: s = "impossible<bad format char>: '{}'".format(format) when_skipped = (str(e) == s) @@ -524,6 +527,32 @@ class SkipitemTest(unittest.TestCase): self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, (), {}, b'', [42]) + def test_positional_only(self): + parse = _testcapi.parse_tuple_and_keywords + + parse((1, 2, 3), {}, b'OOO', ['', '', 'a']) + parse((1, 2), {'a': 3}, b'OOO', ['', '', 'a']) + with self.assertRaisesRegex(TypeError, + r'Function takes at least 2 positional arguments \(1 given\)'): + parse((1,), {'a': 3}, b'OOO', ['', '', 'a']) + parse((1,), {}, b'O|OO', ['', '', 'a']) + with self.assertRaisesRegex(TypeError, + r'Function takes at least 1 positional arguments \(0 given\)'): + parse((), {}, b'O|OO', ['', '', 'a']) + parse((1, 2), {'a': 3}, b'OO$O', ['', '', 'a']) + with self.assertRaisesRegex(TypeError, + r'Function takes exactly 2 positional arguments \(1 given\)'): + parse((1,), {'a': 3}, b'OO$O', ['', '', 'a']) + parse((1,), {}, b'O|O$O', ['', '', 'a']) + with self.assertRaisesRegex(TypeError, + r'Function takes at least 1 positional arguments \(0 given\)'): + parse((), {}, b'O|O$O', ['', '', 'a']) + with self.assertRaisesRegex(SystemError, r'Empty parameter name after \$'): + parse((1,), {}, b'O|$OO', ['', '', 'a']) + with self.assertRaisesRegex(SystemError, 'Empty keyword'): + parse((1,), {}, b'O|OO', ['', 'a', '']) + + @unittest.skipUnless(threading, 'Threading required for this test.') class TestThreadState(unittest.TestCase): @@ -548,6 +577,7 @@ class TestThreadState(unittest.TestCase): t.start() t.join() + class Test_testcapi(unittest.TestCase): def test__testcapi(self): for name in dir(_testcapi): @@ -556,5 +586,85 @@ class Test_testcapi(unittest.TestCase): test = getattr(_testcapi, name) test() + +class PyMemDebugTests(unittest.TestCase): + PYTHONMALLOC = 'debug' + # '0x04c06e0' or '04C06E0' + PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+' + + def check(self, code): + with support.SuppressCrashReport(): + out = assert_python_failure('-c', code, + PYTHONMALLOC=self.PYTHONMALLOC) + stderr = out.err + return stderr.decode('ascii', 'replace') + + def test_buffer_overflow(self): + out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') + regex = (r"Debug memory block at address p={ptr}: API 'm'\n" + r" 16 bytes originally requested\n" + r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n" + r" at tail\+0: 0x78 \*\*\* OUCH\n" + r" at tail\+1: 0xfb\n" + r" at tail\+2: 0xfb\n" + r" .*\n" + r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" + r" Data at p: cb cb cb .*\n" + r"\n" + r"Fatal Python error: bad trailing pad byte") + regex = regex.format(ptr=self.PTR_REGEX) + regex = re.compile(regex, flags=re.DOTALL) + self.assertRegex(out, regex) + + def test_api_misuse(self): + out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') + regex = (r"Debug memory block at address p={ptr}: API 'm'\n" + r" 16 bytes originally requested\n" + r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" + r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" + r" Data at p: cb cb cb .*\n" + r"\n" + r"Fatal Python error: bad ID: Allocated using API 'm', verified using API 'r'\n") + regex = regex.format(ptr=self.PTR_REGEX) + self.assertRegex(out, regex) + + @unittest.skipUnless(threading, 'Test requires a GIL (multithreading)') + def check_malloc_without_gil(self, code): + out = self.check(code) + expected = ('Fatal Python error: Python memory allocator called ' + 'without holding the GIL') + self.assertIn(expected, out) + + def test_pymem_malloc_without_gil(self): + # Debug hooks must raise an error if PyMem_Malloc() is called + # without holding the GIL + code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()' + self.check_malloc_without_gil(code) + + def test_pyobject_malloc_without_gil(self): + # Debug hooks must raise an error if PyObject_Malloc() is called + # without holding the GIL + code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' + self.check_malloc_without_gil(code) + + +class PyMemMallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'malloc_debug' + + +@unittest.skipUnless(sysconfig.get_config_var('WITH_PYMALLOC') == 1, + 'need pymalloc') +class PyMemPymallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'pymalloc_debug' + + +@unittest.skipUnless(Py_DEBUG, 'need Py_DEBUG') +class PyMemDefaultTests(PyMemDebugTests): + # test default allocator of Python compiled in debug mode + PYTHONMALLOC = '' + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py index ab9f6ab6a5..637322137d 100644 --- a/Lib/test/test_cgi.py +++ b/Lib/test/test_cgi.py @@ -7,6 +7,7 @@ import unittest import warnings from collections import namedtuple from io import StringIO, BytesIO +from test import support class HackedSysModule: # The regression test will have real values in sys.argv, which @@ -147,7 +148,7 @@ class CgiTests(unittest.TestCase): def test_escape(self): # cgi.escape() is deprecated. with warnings.catch_warnings(): - warnings.filterwarnings('ignore', 'cgi\.escape', + warnings.filterwarnings('ignore', r'cgi\.escape', DeprecationWarning) self.assertEqual("test & string", cgi.escape("test & string")) self.assertEqual("<test string>", cgi.escape("<test string>")) @@ -473,6 +474,11 @@ this is the content of the fake file cgi.parse_header('form-data; name="files"; filename="fo\\"o;bar"'), ("form-data", {"name": "files", "filename": 'fo"o;bar'})) + def test_all(self): + blacklist = {"logfile", "logfp", "initlog", "dolog", "nolog", + "closelog", "log", "maxlen", "valid_boundary"} + support.check__all__(self, cgi, blacklist=blacklist) + BOUNDARY = "---------------------------721837373350705526688164684" diff --git a/Lib/test/test_charmapcodec.py b/Lib/test/test_charmapcodec.py index 4064aef935..0d4594d8c0 100644 --- a/Lib/test/test_charmapcodec.py +++ b/Lib/test/test_charmapcodec.py @@ -9,7 +9,7 @@ Written by Marc-Andre Lemburg (mal@lemburg.com). """#" -import test.support, unittest +import unittest import codecs diff --git a/Lib/test/test_cmath.py b/Lib/test/test_cmath.py index 1f884e52a2..0451fb0aa2 100644 --- a/Lib/test/test_cmath.py +++ b/Lib/test/test_cmath.py @@ -4,6 +4,8 @@ import test.test_math as test_math import unittest import cmath, math from cmath import phase, polar, rect, pi +import platform +import sys import sysconfig INF = float('inf') @@ -154,6 +156,23 @@ class CMathTests(unittest.TestCase): self.assertAlmostEqual(cmath.e, e_expected, places=9, msg="cmath.e is {}; should be {}".format(cmath.e, e_expected)) + def test_infinity_and_nan_constants(self): + self.assertEqual(cmath.inf.real, math.inf) + self.assertEqual(cmath.inf.imag, 0.0) + self.assertEqual(cmath.infj.real, 0.0) + self.assertEqual(cmath.infj.imag, math.inf) + + self.assertTrue(math.isnan(cmath.nan.real)) + self.assertEqual(cmath.nan.imag, 0.0) + self.assertEqual(cmath.nanj.real, 0.0) + self.assertTrue(math.isnan(cmath.nanj.imag)) + + # Check consistency with reprs. + self.assertEqual(repr(cmath.inf), "inf") + self.assertEqual(repr(cmath.infj), "infj") + self.assertEqual(repr(cmath.nan), "nan") + self.assertEqual(repr(cmath.nanj), "nanj") + def test_user_object(self): # Test automatic calling of __complex__ and __float__ by cmath # functions @@ -315,6 +334,18 @@ class CMathTests(unittest.TestCase): @requires_IEEE_754 def test_specific_values(self): + # Some tests need to be skipped on ancient OS X versions. + # See issue #27953. + SKIP_ON_TIGER = {'tan0064'} + + osx_version = None + if sys.platform == 'darwin': + version_txt = platform.mac_ver()[0] + try: + osx_version = tuple(map(int, version_txt.split('.'))) + except ValueError: + pass + def rect_complex(z): """Wrapped version of rect that accepts a complex number instead of two float arguments.""" @@ -328,6 +359,12 @@ class CMathTests(unittest.TestCase): for id, fn, ar, ai, er, ei, flags in parse_testfile(test_file): arg = complex(ar, ai) expected = complex(er, ei) + + # Skip certain tests on OS X 10.4. + if osx_version is not None and osx_version < (10, 5): + if id in SKIP_ON_TIGER: + continue + if fn == 'rect': function = rect_complex elif fn == 'polar': diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 3d2f769a2c..ae2bcd4375 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -8,7 +8,7 @@ import shutil import sys import subprocess import tempfile -from test.support import script_helper +from test.support import script_helper, is_android from test.support.script_helper import (spawn_python, kill_python, assert_python_ok, assert_python_failure) @@ -43,7 +43,7 @@ class CmdLineTest(unittest.TestCase): def test_version(self): version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii") - for switch in '-V', '--version': + for switch in '-V', '--version', '-VV': rc, out, err = assert_python_ok(switch) self.assertFalse(err.startswith(version)) self.assertTrue(out.startswith(version)) @@ -178,15 +178,16 @@ class CmdLineTest(unittest.TestCase): if not stdout.startswith(pattern): raise AssertionError("%a doesn't start with %a" % (stdout, pattern)) - @unittest.skipUnless(sys.platform == 'darwin', 'test specific to Mac OS X') - def test_osx_utf8(self): + @unittest.skipUnless((sys.platform == 'darwin' or + is_android), 'test specific to Mac OS X and Android') + def test_osx_android_utf8(self): def check_output(text): decoded = text.decode('utf-8', 'surrogateescape') expected = ascii(decoded).encode('ascii') + b'\n' env = os.environ.copy() # C locale gives ASCII locale encoding, but Python uses UTF-8 - # to parse the command line arguments on Mac OS X + # to parse the command line arguments on Mac OS X and Android. env['LC_ALL'] = 'C' p = subprocess.Popen( @@ -348,8 +349,9 @@ class CmdLineTest(unittest.TestCase): test.support.SuppressCrashReport().__enter__() sys.stdout.write('x') os.close(sys.stdout.fileno())""" - rc, out, err = assert_python_ok('-c', code) + rc, out, err = assert_python_failure('-c', code) self.assertEqual(b'', out) + self.assertEqual(120, rc) self.assertRegex(err.decode('ascii', 'ignore'), 'Exception ignored in.*\nOSError: .*') diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 919f3616c5..e058ecd086 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -139,9 +139,8 @@ class CmdLineTest(unittest.TestCase): expected_argv0, expected_path0, expected_package, expected_loader, *cmd_line_switches): - if not __debug__: - cmd_line_switches += ('-' + 'O' * sys.flags.optimize,) - run_args = cmd_line_switches + (script_name,) + tuple(example_args) + run_args = [*support.optim_args_from_interpreter_flags(), + *cmd_line_switches, script_name, *example_args] rc, out, err = assert_python_ok(*run_args, __isolated=False) self._check_output(script_name, rc, out + err, expected_file, expected_argv0, expected_path0, @@ -430,7 +429,7 @@ class CmdLineTest(unittest.TestCase): ('builtins.x', br'Error while finding module specification.*' br'AttributeError'), ('builtins.x.y', br'Error while finding module specification.*' - br'ImportError.*No module named.*not a package'), + br'ModuleNotFoundError.*No module named.*not a package'), ('os.path', br'loader.*cannot handle'), ('importlib', br'No module named.*' br'is a package and cannot be directly executed'), diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 3394b39e01..1a8f6990df 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -69,7 +69,7 @@ class TestInteractiveConsole(unittest.TestCase): # with banner self.infunc.side_effect = EOFError('Finished') self.console.interact(banner='Foo') - self.assertEqual(len(self.stderr.method_calls), 2) + self.assertEqual(len(self.stderr.method_calls), 3) banner_call = self.stderr.method_calls[0] self.assertEqual(banner_call, ['write', ('Foo\n',), {}]) @@ -77,8 +77,36 @@ class TestInteractiveConsole(unittest.TestCase): self.stderr.reset_mock() self.infunc.side_effect = EOFError('Finished') self.console.interact(banner='') + self.assertEqual(len(self.stderr.method_calls), 2) + + def test_exit_msg(self): + # default exit message + self.infunc.side_effect = EOFError('Finished') + self.console.interact(banner='') + self.assertEqual(len(self.stderr.method_calls), 2) + err_msg = self.stderr.method_calls[1] + expected = 'now exiting InteractiveConsole...\n' + self.assertEqual(err_msg, ['write', (expected,), {}]) + + # no exit message + self.stderr.reset_mock() + self.infunc.side_effect = EOFError('Finished') + self.console.interact(banner='', exitmsg='') self.assertEqual(len(self.stderr.method_calls), 1) + # custom exit message + self.stderr.reset_mock() + message = ( + 'bye! \N{GREEK SMALL LETTER ZETA}\N{CYRILLIC SMALL LETTER ZHE}' + ) + self.infunc.side_effect = EOFError('Finished') + self.console.interact(banner='', exitmsg=message) + self.assertEqual(len(self.stderr.method_calls), 2) + err_msg = self.stderr.method_calls[1] + expected = message + '\n' + self.assertEqual(err_msg, ['write', (expected,), {}]) + + def test_cause_tb(self): self.infunc.side_effect = ["raise ValueError('') from AttributeError", EOFError('Finished')] diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index ee1e28a763..6a3e993265 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -4,7 +4,6 @@ import sys import test.support import unicodedata import unittest -import warnings class PosReturn: # this can be used for configurable callbacks @@ -281,12 +280,12 @@ class CodecCallbackTest(unittest.TestCase): ) self.assertEqual( - b"\\u3042\u3xxx".decode("unicode-escape", "test.handler1"), + b"\\u3042\\u3xxx".decode("unicode-escape", "test.handler1"), "\u3042[<92><117><51>]xxx" ) self.assertEqual( - b"\\u3042\u3xx".decode("unicode-escape", "test.handler1"), + b"\\u3042\\u3xx".decode("unicode-escape", "test.handler1"), "\u3042[<92><117><51>]xx" ) diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py index d0e3a15d16..3bdf7d0e14 100644 --- a/Lib/test/test_codecencodings_cn.py +++ b/Lib/test/test_codecencodings_cn.py @@ -3,7 +3,6 @@ # Codec encoding tests for PRC encodings. # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecencodings_hk.py b/Lib/test/test_codecencodings_hk.py index bb9be11184..c5e2f999ef 100644 --- a/Lib/test/test_codecencodings_hk.py +++ b/Lib/test/test_codecencodings_hk.py @@ -3,7 +3,6 @@ # Codec encoding tests for HongKong encodings. # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecencodings_iso2022.py b/Lib/test/test_codecencodings_iso2022.py index 8a3ca70de6..00ea1c39dd 100644 --- a/Lib/test/test_codecencodings_iso2022.py +++ b/Lib/test/test_codecencodings_iso2022.py @@ -1,6 +1,5 @@ # Codec encoding tests for ISO 2022 encodings. -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecencodings_jp.py b/Lib/test/test_codecencodings_jp.py index 44b63a00c8..94378d124f 100644 --- a/Lib/test/test_codecencodings_jp.py +++ b/Lib/test/test_codecencodings_jp.py @@ -3,7 +3,6 @@ # Codec encoding tests for Japanese encodings. # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecencodings_kr.py b/Lib/test/test_codecencodings_kr.py index b6a74fbd5b..863d16b8fd 100644 --- a/Lib/test/test_codecencodings_kr.py +++ b/Lib/test/test_codecencodings_kr.py @@ -3,7 +3,6 @@ # Codec encoding tests for ROK encodings. # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecencodings_tw.py b/Lib/test/test_codecencodings_tw.py index 9174296726..bb1d133cad 100644 --- a/Lib/test/test_codecencodings_tw.py +++ b/Lib/test/test_codecencodings_tw.py @@ -3,7 +3,6 @@ # Codec encoding tests for ROC encodings. # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecmaps_cn.py b/Lib/test/test_codecmaps_cn.py index f1bd3840c9..89e51c6e2a 100644 --- a/Lib/test/test_codecmaps_cn.py +++ b/Lib/test/test_codecmaps_cn.py @@ -3,7 +3,6 @@ # Codec mapping tests for PRC encodings # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecmaps_hk.py b/Lib/test/test_codecmaps_hk.py index 4c0c4156da..7a48d24fdd 100644 --- a/Lib/test/test_codecmaps_hk.py +++ b/Lib/test/test_codecmaps_hk.py @@ -3,7 +3,6 @@ # Codec mapping tests for HongKong encodings # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecmaps_jp.py b/Lib/test/test_codecmaps_jp.py index 577382329a..fdfec8085b 100644 --- a/Lib/test/test_codecmaps_jp.py +++ b/Lib/test/test_codecmaps_jp.py @@ -3,7 +3,6 @@ # Codec mapping tests for Japanese encodings # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecmaps_kr.py b/Lib/test/test_codecmaps_kr.py index 6cb41c8b29..471cd749b8 100644 --- a/Lib/test/test_codecmaps_kr.py +++ b/Lib/test/test_codecmaps_kr.py @@ -3,7 +3,6 @@ # Codec mapping tests for ROK encodings # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecmaps_tw.py b/Lib/test/test_codecmaps_tw.py index 2ea44b56f1..145a97d511 100644 --- a/Lib/test/test_codecmaps_tw.py +++ b/Lib/test/test_codecmaps_tw.py @@ -3,7 +3,6 @@ # Codec mapping tests for ROC encodings # -from test import support from test import multibytecodec_support import unittest diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 013ec64b20..1e63ed8d79 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -4,16 +4,10 @@ import io import locale import sys import unittest -import warnings import encodings from test import support -if sys.platform == 'win32': - VISTA_OR_LATER = (sys.getwindowsversion().major >= 6) -else: - VISTA_OR_LATER = False - try: import ctypes except ImportError: @@ -27,6 +21,7 @@ def coding_checker(self, coder): self.assertEqual(coder(input), (expect, len(input))) return check + class Queue(object): """ queue: write bytes at one end, read bytes from the other end @@ -47,6 +42,7 @@ class Queue(object): self._buffer = self._buffer[size:] return s + class MixInCheckStateHandling: def check_state_handling_decode(self, encoding, u, s): for i in range(len(s)+1): @@ -80,6 +76,7 @@ class MixInCheckStateHandling: part2 = d.encode(u[i:], True) self.assertEqual(s, part1+part2) + class ReadTest(MixInCheckStateHandling): def check_partial(self, input, partialresults): # get a StreamReader for the encoding and feed the bytestring version @@ -358,6 +355,12 @@ class ReadTest(MixInCheckStateHandling): self.assertEqual("[\uDC80]".encode(self.encoding, "replace"), "[?]".encode(self.encoding)) + # sequential surrogate characters + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "ignore"), + "[]".encode(self.encoding)) + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "replace"), + "[??]".encode(self.encoding)) + bom = "".encode(self.encoding) for before, after in [("\U00010fff", "A"), ("[", "]"), ("A", "\U00010fff")]: @@ -383,6 +386,7 @@ class ReadTest(MixInCheckStateHandling): self.assertEqual(test_sequence.decode(self.encoding, "backslashreplace"), before + backslashreplace + after) + class UTF32Test(ReadTest, unittest.TestCase): encoding = "utf-32" if sys.byteorder == 'little': @@ -478,6 +482,7 @@ class UTF32Test(ReadTest, unittest.TestCase): self.assertEqual('\U00010000' * 1024, codecs.utf_32_decode(encoded_be)[0]) + class UTF32LETest(ReadTest, unittest.TestCase): encoding = "utf-32-le" ill_formed_sequence = b"\x80\xdc\x00\x00" @@ -523,6 +528,7 @@ class UTF32LETest(ReadTest, unittest.TestCase): self.assertEqual('\U00010000' * 1024, codecs.utf_32_le_decode(encoded)[0]) + class UTF32BETest(ReadTest, unittest.TestCase): encoding = "utf-32-be" ill_formed_sequence = b"\x00\x00\xdc\x80" @@ -747,6 +753,7 @@ class UTF8Test(ReadTest, unittest.TestCase): encoding = "utf-8" ill_formed_sequence = b"\xed\xb2\x80" ill_formed_sequence_replace = "\ufffd" * 3 + BOM = b'' def test_partial(self): self.check_partial( @@ -775,27 +782,49 @@ class UTF8Test(ReadTest, unittest.TestCase): self.check_state_handling_decode(self.encoding, u, u.encode(self.encoding)) + def test_decode_error(self): + for data, error_handler, expected in ( + (b'[\x80\xff]', 'ignore', '[]'), + (b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'), + (b'[\x80\xff]', 'surrogateescape', '[\udc80\udcff]'), + (b'[\x80\xff]', 'backslashreplace', '[\\x80\\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.decode(self.encoding, error_handler), + expected) + def test_lone_surrogates(self): super().test_lone_surrogates() # not sure if this is making sense for # UTF-16 and UTF-32 - self.assertEqual("[\uDC80]".encode('utf-8', "surrogateescape"), - b'[\x80]') + self.assertEqual("[\uDC80]".encode(self.encoding, "surrogateescape"), + self.BOM + b'[\x80]') + + with self.assertRaises(UnicodeEncodeError) as cm: + "[\uDC80\uD800\uDFFF]".encode(self.encoding, "surrogateescape") + exc = cm.exception + self.assertEqual(exc.object[exc.start:exc.end], '\uD800\uDFFF') def test_surrogatepass_handler(self): - self.assertEqual("abc\ud800def".encode("utf-8", "surrogatepass"), - b"abc\xed\xa0\x80def") - self.assertEqual(b"abc\xed\xa0\x80def".decode("utf-8", "surrogatepass"), + self.assertEqual("abc\ud800def".encode(self.encoding, "surrogatepass"), + self.BOM + b"abc\xed\xa0\x80def") + self.assertEqual("\U00010fff\uD800".encode(self.encoding, "surrogatepass"), + self.BOM + b"\xf0\x90\xbf\xbf\xed\xa0\x80") + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "surrogatepass"), + self.BOM + b'[\xed\xa0\x80\xed\xb2\x80]') + + self.assertEqual(b"abc\xed\xa0\x80def".decode(self.encoding, "surrogatepass"), "abc\ud800def") - self.assertEqual("\U00010fff\uD800".encode("utf-8", "surrogatepass"), - b"\xf0\x90\xbf\xbf\xed\xa0\x80") - self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("utf-8", "surrogatepass"), + self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode(self.encoding, "surrogatepass"), "\U00010fff\uD800") + self.assertTrue(codecs.lookup_error("surrogatepass")) with self.assertRaises(UnicodeDecodeError): - b"abc\xed\xa0".decode("utf-8", "surrogatepass") + b"abc\xed\xa0".decode(self.encoding, "surrogatepass") with self.assertRaises(UnicodeDecodeError): - b"abc\xed\xa0z".decode("utf-8", "surrogatepass") + b"abc\xed\xa0z".decode(self.encoding, "surrogatepass") + @unittest.skipUnless(sys.platform == 'win32', 'cp65001 is a Windows-only codec') @@ -807,18 +836,13 @@ class CP65001Test(ReadTest, unittest.TestCase): ('abc', 'strict', b'abc'), ('\xe9\u20ac', 'strict', b'\xc3\xa9\xe2\x82\xac'), ('\U0010ffff', 'strict', b'\xf4\x8f\xbf\xbf'), + ('\udc80', 'strict', None), + ('\udc80', 'ignore', b''), + ('\udc80', 'replace', b'?'), + ('\udc80', 'backslashreplace', b'\\udc80'), + ('\udc80', 'namereplace', b'\\udc80'), + ('\udc80', 'surrogatepass', b'\xed\xb2\x80'), ] - if VISTA_OR_LATER: - tests.extend(( - ('\udc80', 'strict', None), - ('\udc80', 'ignore', b''), - ('\udc80', 'replace', b'?'), - ('\udc80', 'backslashreplace', b'\\udc80'), - ('\udc80', 'namereplace', b'\\udc80'), - ('\udc80', 'surrogatepass', b'\xed\xb2\x80'), - )) - else: - tests.append(('\udc80', 'strict', b'\xed\xb2\x80')) for text, errors, expected in tests: if expected is not None: try: @@ -845,17 +869,10 @@ class CP65001Test(ReadTest, unittest.TestCase): (b'[\xff]', 'ignore', '[]'), (b'[\xff]', 'replace', '[\ufffd]'), (b'[\xff]', 'surrogateescape', '[\udcff]'), + (b'[\xed\xb2\x80]', 'strict', None), + (b'[\xed\xb2\x80]', 'ignore', '[]'), + (b'[\xed\xb2\x80]', 'replace', '[\ufffd\ufffd\ufffd]'), ] - if VISTA_OR_LATER: - tests.extend(( - (b'[\xed\xb2\x80]', 'strict', None), - (b'[\xed\xb2\x80]', 'ignore', '[]'), - (b'[\xed\xb2\x80]', 'replace', '[\ufffd\ufffd\ufffd]'), - )) - else: - tests.extend(( - (b'[\xed\xb2\x80]', 'strict', '[\udc80]'), - )) for raw, errors, expected in tests: if expected is not None: try: @@ -870,7 +887,6 @@ class CP65001Test(ReadTest, unittest.TestCase): self.assertRaises(UnicodeDecodeError, raw.decode, 'cp65001', errors) - @unittest.skipUnless(VISTA_OR_LATER, 'require Windows Vista or later') def test_lone_surrogates(self): self.assertRaises(UnicodeEncodeError, "\ud800".encode, "cp65001") self.assertRaises(UnicodeDecodeError, b"\xed\xa0\x80".decode, "cp65001") @@ -887,7 +903,6 @@ class CP65001Test(ReadTest, unittest.TestCase): self.assertEqual("[\uDC80]".encode("cp65001", "replace"), b'[?]') - @unittest.skipUnless(VISTA_OR_LATER, 'require Windows Vista or later') def test_surrogatepass_handler(self): self.assertEqual("abc\ud800def".encode("cp65001", "surrogatepass"), b"abc\xed\xa0\x80def") @@ -1059,6 +1074,7 @@ class ReadBufferTest(unittest.TestCase): class UTF8SigTest(UTF8Test, unittest.TestCase): encoding = "utf-8-sig" + BOM = codecs.BOM_UTF8 def test_partial(self): self.check_partial( @@ -1159,7 +1175,7 @@ class EscapeDecodeTest(unittest.TestCase): check(b"[\\\n]", b"[]") check(br'[\"]', b'["]') check(br"[\']", b"[']") - check(br"[\\]", br"[\]") + check(br"[\\]", b"[\\]") check(br"[\a]", b"[\x07]") check(br"[\b]", b"[\x08]") check(br"[\t]", b"[\x09]") @@ -1168,7 +1184,6 @@ class EscapeDecodeTest(unittest.TestCase): check(br"[\f]", b"[\x0c]") check(br"[\r]", b"[\x0d]") check(br"[\7]", b"[\x07]") - check(br"[\8]", br"[\8]") check(br"[\78]", b"[\x078]") check(br"[\41]", b"[!]") check(br"[\418]", b"[!8]") @@ -1176,12 +1191,18 @@ class EscapeDecodeTest(unittest.TestCase): check(br"[\1010]", b"[A0]") check(br"[\501]", b"[A]") check(br"[\x41]", b"[A]") - check(br"[\X41]", br"[\X41]") check(br"[\x410]", b"[A0]") - for b in range(256): - if b not in b'\n"\'\\abtnvfr01234567x': - b = bytes([b]) - check(b'\\' + b, b'\\' + b) + for i in range(97, 123): + b = bytes([i]) + if b not in b'abfnrtvx': + with self.assertWarns(DeprecationWarning): + check(b"\\" + b, b"\\" + b) + with self.assertWarns(DeprecationWarning): + check(b"\\" + b.upper(), b"\\" + b.upper()) + with self.assertWarns(DeprecationWarning): + check(br"\8", b"\\8") + with self.assertWarns(DeprecationWarning): + check(br"\9", b"\\9") def test_errors(self): decode = codecs.escape_decode @@ -1194,6 +1215,7 @@ class EscapeDecodeTest(unittest.TestCase): self.assertEqual(decode(br"[\x0]\x0", "ignore"), (b"[]", 8)) self.assertEqual(decode(br"[\x0]\x0", "replace"), (b"[?]?", 8)) + class RecodingTest(unittest.TestCase): def test_recoding(self): f = io.BytesIO() @@ -1313,6 +1335,7 @@ for i in punycode_testcases: if len(i)!=2: print(repr(i)) + class PunycodeTest(unittest.TestCase): def test_encode(self): for uni, puny in punycode_testcases: @@ -1332,6 +1355,7 @@ class PunycodeTest(unittest.TestCase): puny = puny.decode("ascii").encode("ascii") self.assertEqual(uni, puny.decode("punycode")) + class UnicodeInternalTest(unittest.TestCase): @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t') def test_bug1251300(self): @@ -1586,6 +1610,7 @@ class NameprepTest(unittest.TestCase): except Exception as e: raise support.TestFailed("Test 3.%d: %s" % (pos+1, str(e))) + class IDNACodecTest(unittest.TestCase): def test_builtin_decode(self): self.assertEqual(str(b"python.org", "idna"), "python.org") @@ -1672,6 +1697,7 @@ class IDNACodecTest(unittest.TestCase): self.assertRaises(Exception, b"python.org".decode, "idna", errors) + class CodecsModuleTest(unittest.TestCase): def test_decode(self): @@ -1780,6 +1806,7 @@ class CodecsModuleTest(unittest.TestCase): self.assertRaises(UnicodeError, codecs.decode, b'abc', 'undefined', errors) + class StreamReaderTest(unittest.TestCase): def setUp(self): @@ -1790,6 +1817,7 @@ class StreamReaderTest(unittest.TestCase): f = self.reader(self.stream) self.assertEqual(f.readlines(), ['\ud55c\n', '\uae00']) + class EncodedFileTest(unittest.TestCase): def test_basic(self): @@ -1909,6 +1937,8 @@ all_unicode_encodings = [ if hasattr(codecs, "mbcs_encode"): all_unicode_encodings.append("mbcs") +if hasattr(codecs, "oem_encode"): + all_unicode_encodings.append("oem") # The following encoding is not tested, because it's not supposed # to work: @@ -1920,6 +1950,7 @@ broken_unicode_with_stateful = [ "unicode_internal" ] + class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): def test_basics(self): s = "abc123" # all codecs should be able to encode these @@ -2082,6 +2113,7 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): self.check_state_handling_decode(encoding, u, u.encode(encoding)) self.check_state_handling_encode(encoding, u, u.encode(encoding)) + class CharmapTest(unittest.TestCase): def test_decode_with_string_map(self): self.assertEqual( @@ -2332,6 +2364,7 @@ class WithStmtTest(unittest.TestCase): info.streamwriter, 'strict') as srw: self.assertEqual(srw.read(), "\xfc") + class TypesTest(unittest.TestCase): def test_decode_unicode(self): # Most decoders don't accept unicode input @@ -2420,7 +2453,6 @@ class UnicodeEscapeTest(unittest.TestCase): check(br"[\f]", "[\x0c]") check(br"[\r]", "[\x0d]") check(br"[\7]", "[\x07]") - check(br"[\8]", r"[\8]") check(br"[\78]", "[\x078]") check(br"[\41]", "[!]") check(br"[\418]", "[!8]") @@ -2430,9 +2462,18 @@ class UnicodeEscapeTest(unittest.TestCase): check(br"[\x410]", "[A0]") check(br"\u20ac", "\u20ac") check(br"\U0001d120", "\U0001d120") - for b in range(256): - if b not in b'\n"\'\\abtnvfr01234567xuUN': - check(b'\\' + bytes([b]), '\\' + chr(b)) + for i in range(97, 123): + b = bytes([i]) + if b not in b'abfnrtuvx': + with self.assertWarns(DeprecationWarning): + check(b"\\" + b, "\\" + chr(i)) + if b.upper() not in b'UN': + with self.assertWarns(DeprecationWarning): + check(b"\\" + b.upper(), "\\" + chr(i-32)) + with self.assertWarns(DeprecationWarning): + check(br"\8", "\\8") + with self.assertWarns(DeprecationWarning): + check(br"\9", "\\9") def test_decode_errors(self): decode = codecs.unicode_escape_decode @@ -2642,6 +2683,7 @@ else: bytes_transform_encodings.append("bz2_codec") transform_aliases["bz2_codec"] = ["bz2"] + class TransformCodecTest(unittest.TestCase): def test_basics(self): @@ -2694,8 +2736,8 @@ class TransformCodecTest(unittest.TestCase): bad_input = "bad input type" for encoding in bytes_transform_encodings: with self.subTest(encoding=encoding): - fmt = ( "{!r} is not a text encoding; " - "use codecs.encode\(\) to handle arbitrary codecs") + fmt = (r"{!r} is not a text encoding; " + r"use codecs.encode\(\) to handle arbitrary codecs") msg = fmt.format(encoding) with self.assertRaisesRegex(LookupError, msg) as failure: bad_input.encode(encoding) @@ -2704,7 +2746,7 @@ class TransformCodecTest(unittest.TestCase): def test_text_to_binary_blacklists_text_transforms(self): # Check str.encode gives a good error message for str -> str codecs msg = (r"^'rot_13' is not a text encoding; " - "use codecs.encode\(\) to handle arbitrary codecs") + r"use codecs.encode\(\) to handle arbitrary codecs") with self.assertRaisesRegex(LookupError, msg): "just an example message".encode("rot_13") @@ -2716,7 +2758,7 @@ class TransformCodecTest(unittest.TestCase): with self.subTest(encoding=encoding): encoded_data = codecs.encode(data, encoding) fmt = (r"{!r} is not a text encoding; " - "use codecs.decode\(\) to handle arbitrary codecs") + r"use codecs.decode\(\) to handle arbitrary codecs") msg = fmt.format(encoding) with self.assertRaisesRegex(LookupError, msg): encoded_data.decode(encoding) @@ -2728,7 +2770,7 @@ class TransformCodecTest(unittest.TestCase): for bad_input in (b"immutable", bytearray(b"mutable")): with self.subTest(bad_input=bad_input): msg = (r"^'rot_13' is not a text encoding; " - "use codecs.decode\(\) to handle arbitrary codecs") + r"use codecs.decode\(\) to handle arbitrary codecs") with self.assertRaisesRegex(LookupError, msg) as failure: bad_input.decode("rot_13") self.assertIsNone(failure.exception.__cause__) @@ -2947,12 +2989,12 @@ class ExceptionChainingTest(unittest.TestCase): self.assertEqual(decoded, b"not str!") # Text model methods should complain fmt = (r"^{!r} encoder returned 'str' instead of 'bytes'; " - "use codecs.encode\(\) to encode to arbitrary types$") + r"use codecs.encode\(\) to encode to arbitrary types$") msg = fmt.format(self.codec_name) with self.assertRaisesRegex(TypeError, msg): "str_input".encode(self.codec_name) fmt = (r"^{!r} decoder returned 'bytes' instead of 'str'; " - "use codecs.decode\(\) to decode to arbitrary types$") + r"use codecs.decode\(\) to decode to arbitrary types$") msg = fmt.format(self.codec_name) with self.assertRaisesRegex(TypeError, msg): b"bytes input".decode(self.codec_name) @@ -3093,11 +3135,10 @@ class CodePageTest(unittest.TestCase): (b'\xff\xf4\x8f\xbf\xbf', 'ignore', '\U0010ffff'), (b'\xff\xf4\x8f\xbf\xbf', 'replace', '\ufffd\U0010ffff'), )) - if VISTA_OR_LATER: - self.check_encode(self.CP_UTF8, ( - ('[\U0010ffff\uDC80]', 'ignore', b'[\xf4\x8f\xbf\xbf]'), - ('[\U0010ffff\uDC80]', 'replace', b'[\xf4\x8f\xbf\xbf?]'), - )) + self.check_encode(self.CP_UTF8, ( + ('[\U0010ffff\uDC80]', 'ignore', b'[\xf4\x8f\xbf\xbf]'), + ('[\U0010ffff\uDC80]', 'replace', b'[\xf4\x8f\xbf\xbf?]'), + )) def test_incremental(self): decoded = codecs.code_page_decode(932, b'\x82', 'strict', False) @@ -3118,6 +3159,96 @@ class CodePageTest(unittest.TestCase): False) self.assertEqual(decoded, ('abc', 3)) + def test_mbcs_alias(self): + # Check that looking up our 'default' codepage will return + # mbcs when we don't have a more specific one available + import _bootlocale + def _get_fake_codepage(*a): + return 'cp123' + old_getpreferredencoding = _bootlocale.getpreferredencoding + _bootlocale.getpreferredencoding = _get_fake_codepage + try: + codec = codecs.lookup('cp123') + self.assertEqual(codec.name, 'mbcs') + finally: + _bootlocale.getpreferredencoding = old_getpreferredencoding + + +class ASCIITest(unittest.TestCase): + def test_encode(self): + self.assertEqual('abc123'.encode('ascii'), b'abc123') + + def test_encode_error(self): + for data, error_handler, expected in ( + ('[\x80\xff\u20ac]', 'ignore', b'[]'), + ('[\x80\xff\u20ac]', 'replace', b'[???]'), + ('[\x80\xff\u20ac]', 'xmlcharrefreplace', b'[€ÿ€]'), + ('[\x80\xff\u20ac\U000abcde]', 'backslashreplace', + b'[\\x80\\xff\\u20ac\\U000abcde]'), + ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.encode('ascii', error_handler), + expected) + + def test_encode_surrogateescape_error(self): + with self.assertRaises(UnicodeEncodeError): + # the first character can be decoded, but not the second + '\udc80\xff'.encode('ascii', 'surrogateescape') + + def test_decode(self): + self.assertEqual(b'abc'.decode('ascii'), 'abc') + + def test_decode_error(self): + for data, error_handler, expected in ( + (b'[\x80\xff]', 'ignore', '[]'), + (b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'), + (b'[\x80\xff]', 'surrogateescape', '[\udc80\udcff]'), + (b'[\x80\xff]', 'backslashreplace', '[\\x80\\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.decode('ascii', error_handler), + expected) + + +class Latin1Test(unittest.TestCase): + def test_encode(self): + for data, expected in ( + ('abc', b'abc'), + ('\x80\xe9\xff', b'\x80\xe9\xff'), + ): + with self.subTest(data=data, expected=expected): + self.assertEqual(data.encode('latin1'), expected) + + def test_encode_errors(self): + for data, error_handler, expected in ( + ('[\u20ac\udc80]', 'ignore', b'[]'), + ('[\u20ac\udc80]', 'replace', b'[??]'), + ('[\u20ac\U000abcde]', 'backslashreplace', + b'[\\u20ac\\U000abcde]'), + ('[\u20ac\udc80]', 'xmlcharrefreplace', b'[€�]'), + ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.encode('latin1', error_handler), + expected) + + def test_encode_surrogateescape_error(self): + with self.assertRaises(UnicodeEncodeError): + # the first character can be decoded, but not the second + '\udc80\u20ac'.encode('latin1', 'surrogateescape') + + def test_decode(self): + for data, expected in ( + (b'abc', 'abc'), + (b'[\x80\xff]', '[\x80\xff]'), + ): + with self.subTest(data=data, expected=expected): + self.assertEqual(data.decode('latin1'), expected) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index 509bf5dfd2..98da26fa5d 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -282,7 +282,6 @@ class CodeopTests(unittest.TestCase): ai("if (a == 1 and b = 2): pass") ai("del 1") - ai("del ()") ai("del (1,)") ai("del [1]") ai("del '1'") diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 5238382f99..87454cc670 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -19,11 +19,12 @@ from collections import namedtuple, Counter, OrderedDict, _count_elements from collections import UserDict, UserString, UserList from collections import ChainMap from collections import deque -from collections.abc import Awaitable, Coroutine, AsyncIterator, AsyncIterable -from collections.abc import Hashable, Iterable, Iterator, Generator -from collections.abc import Sized, Container, Callable +from collections.abc import Awaitable, Coroutine +from collections.abc import AsyncIterator, AsyncIterable, AsyncGenerator +from collections.abc import Hashable, Iterable, Iterator, Generator, Reversible +from collections.abc import Sized, Container, Callable, Collection from collections.abc import Set, MutableSet -from collections.abc import Mapping, MutableMapping, KeysView, ItemsView +from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence from collections.abc import ByteString @@ -242,6 +243,10 @@ class TestNamedTuple(unittest.TestCase): ]: self.assertEqual(namedtuple('NT', spec, rename=True)._fields, renamed) + def test_module_parameter(self): + NT = namedtuple('NT', ['x', 'y'], module=collections) + self.assertEqual(NT.__module__, collections) + def test_instance(self): Point = namedtuple('Point', 'x y') p = Point(11, 22) @@ -412,6 +417,18 @@ class TestNamedTuple(unittest.TestCase): self.assertEqual(NTColor._fields, ('red', 'green', 'blue')) globals().pop('NTColor', None) # clean-up after this test + def test_keyword_only_arguments(self): + # See issue 25628 + with support.captured_stdout() as template: + NT = namedtuple('NT', ['x', 'y'], verbose=True) + self.assertIn('class NT', NT._source) + with self.assertRaises(TypeError): + NT = namedtuple('NT', ['x', 'y'], True) + + NT = namedtuple('NT', ['abc', 'def'], rename=True) + self.assertEqual(NT._fields, ('abc', '_1')) + with self.assertRaises(TypeError): + NT = namedtuple('NT', ['abc', 'def'], False, True) def test_namedtuple_subclass_issue_24931(self): class Point(namedtuple('_Point', ['x', 'y'])): @@ -487,6 +504,9 @@ class ABCTestCase(unittest.TestCase): self.assertTrue(other.right_side,'Right side not called for %s.%s' % (type(instance), name)) +def _test_gen(): + yield + class TestOneTrickPonyABCs(ABCTestCase): def test_Awaitable(self): @@ -674,7 +694,7 @@ class TestOneTrickPonyABCs(ABCTestCase): samples = [bytes(), str(), tuple(), list(), set(), frozenset(), dict(), dict().keys(), dict().items(), dict().values(), - (lambda: (yield))(), + _test_gen(), (x for x in []), ] for x in samples: @@ -688,6 +708,161 @@ class TestOneTrickPonyABCs(ABCTestCase): self.assertFalse(issubclass(str, I)) self.validate_abstract_methods(Iterable, '__iter__') self.validate_isinstance(Iterable, '__iter__') + # Check None blocking + class It: + def __iter__(self): return iter([]) + class ItBlocked(It): + __iter__ = None + self.assertTrue(issubclass(It, Iterable)) + self.assertTrue(isinstance(It(), Iterable)) + self.assertFalse(issubclass(ItBlocked, Iterable)) + self.assertFalse(isinstance(ItBlocked(), Iterable)) + + def test_Reversible(self): + # Check some non-reversibles + non_samples = [None, 42, 3.14, 1j, dict(), set(), frozenset()] + for x in non_samples: + self.assertNotIsInstance(x, Reversible) + self.assertFalse(issubclass(type(x), Reversible), repr(type(x))) + # Check some non-reversible iterables + non_reversibles = [dict().keys(), dict().items(), dict().values(), + Counter(), Counter().keys(), Counter().items(), + Counter().values(), _test_gen(), + (x for x in []), iter([]), reversed([])] + for x in non_reversibles: + self.assertNotIsInstance(x, Reversible) + self.assertFalse(issubclass(type(x), Reversible), repr(type(x))) + # Check some reversible iterables + samples = [bytes(), str(), tuple(), list(), OrderedDict(), + OrderedDict().keys(), OrderedDict().items(), + OrderedDict().values()] + for x in samples: + self.assertIsInstance(x, Reversible) + self.assertTrue(issubclass(type(x), Reversible), repr(type(x))) + # Check also Mapping, MutableMapping, and Sequence + self.assertTrue(issubclass(Sequence, Reversible), repr(Sequence)) + self.assertFalse(issubclass(Mapping, Reversible), repr(Mapping)) + self.assertFalse(issubclass(MutableMapping, Reversible), repr(MutableMapping)) + # Check direct subclassing + class R(Reversible): + def __iter__(self): + return iter(list()) + def __reversed__(self): + return iter(list()) + self.assertEqual(list(reversed(R())), []) + self.assertFalse(issubclass(float, R)) + self.validate_abstract_methods(Reversible, '__reversed__', '__iter__') + # Check reversible non-iterable (which is not Reversible) + class RevNoIter: + def __reversed__(self): return reversed([]) + class RevPlusIter(RevNoIter): + def __iter__(self): return iter([]) + self.assertFalse(issubclass(RevNoIter, Reversible)) + self.assertFalse(isinstance(RevNoIter(), Reversible)) + self.assertTrue(issubclass(RevPlusIter, Reversible)) + self.assertTrue(isinstance(RevPlusIter(), Reversible)) + # Check None blocking + class Rev: + def __iter__(self): return iter([]) + def __reversed__(self): return reversed([]) + class RevItBlocked(Rev): + __iter__ = None + class RevRevBlocked(Rev): + __reversed__ = None + self.assertTrue(issubclass(Rev, Reversible)) + self.assertTrue(isinstance(Rev(), Reversible)) + self.assertFalse(issubclass(RevItBlocked, Reversible)) + self.assertFalse(isinstance(RevItBlocked(), Reversible)) + self.assertFalse(issubclass(RevRevBlocked, Reversible)) + self.assertFalse(isinstance(RevRevBlocked(), Reversible)) + + def test_Collection(self): + # Check some non-collections + non_collections = [None, 42, 3.14, 1j, lambda x: 2*x] + for x in non_collections: + self.assertNotIsInstance(x, Collection) + self.assertFalse(issubclass(type(x), Collection), repr(type(x))) + # Check some non-collection iterables + non_col_iterables = [_test_gen(), iter(b''), iter(bytearray()), + (x for x in []), dict().values()] + for x in non_col_iterables: + self.assertNotIsInstance(x, Collection) + self.assertFalse(issubclass(type(x), Collection), repr(type(x))) + # Check some collections + samples = [set(), frozenset(), dict(), bytes(), str(), tuple(), + list(), dict().keys(), dict().items()] + for x in samples: + self.assertIsInstance(x, Collection) + self.assertTrue(issubclass(type(x), Collection), repr(type(x))) + # Check also Mapping, MutableMapping, etc. + self.assertTrue(issubclass(Sequence, Collection), repr(Sequence)) + self.assertTrue(issubclass(Mapping, Collection), repr(Mapping)) + self.assertTrue(issubclass(MutableMapping, Collection), + repr(MutableMapping)) + self.assertTrue(issubclass(Set, Collection), repr(Set)) + self.assertTrue(issubclass(MutableSet, Collection), repr(MutableSet)) + self.assertTrue(issubclass(Sequence, Collection), repr(MutableSet)) + # Check direct subclassing + class Col(Collection): + def __iter__(self): + return iter(list()) + def __len__(self): + return 0 + def __contains__(self, item): + return False + class DerCol(Col): pass + self.assertEqual(list(iter(Col())), []) + self.assertFalse(issubclass(list, Col)) + self.assertFalse(issubclass(set, Col)) + self.assertFalse(issubclass(float, Col)) + self.assertEqual(list(iter(DerCol())), []) + self.assertFalse(issubclass(list, DerCol)) + self.assertFalse(issubclass(set, DerCol)) + self.assertFalse(issubclass(float, DerCol)) + self.validate_abstract_methods(Collection, '__len__', '__iter__', + '__contains__') + # Check sized container non-iterable (which is not Collection) etc. + class ColNoIter: + def __len__(self): return 0 + def __contains__(self, item): return False + class ColNoSize: + def __iter__(self): return iter([]) + def __contains__(self, item): return False + class ColNoCont: + def __iter__(self): return iter([]) + def __len__(self): return 0 + self.assertFalse(issubclass(ColNoIter, Collection)) + self.assertFalse(isinstance(ColNoIter(), Collection)) + self.assertFalse(issubclass(ColNoSize, Collection)) + self.assertFalse(isinstance(ColNoSize(), Collection)) + self.assertFalse(issubclass(ColNoCont, Collection)) + self.assertFalse(isinstance(ColNoCont(), Collection)) + # Check None blocking + class SizeBlock: + def __iter__(self): return iter([]) + def __contains__(self): return False + __len__ = None + class IterBlock: + def __len__(self): return 0 + def __contains__(self): return True + __iter__ = None + self.assertFalse(issubclass(SizeBlock, Collection)) + self.assertFalse(isinstance(SizeBlock(), Collection)) + self.assertFalse(issubclass(IterBlock, Collection)) + self.assertFalse(isinstance(IterBlock(), Collection)) + # Check None blocking in subclass + class ColImpl: + def __iter__(self): + return iter(list()) + def __len__(self): + return 0 + def __contains__(self, item): + return False + class NonCol(ColImpl): + __contains__ = None + self.assertFalse(issubclass(NonCol, Collection)) + self.assertFalse(isinstance(NonCol(), Collection)) + def test_Iterator(self): non_samples = [None, 42, 3.14, 1j, b"", "", (), [], {}, set()] @@ -699,7 +874,7 @@ class TestOneTrickPonyABCs(ABCTestCase): iter(set()), iter(frozenset()), iter(dict().keys()), iter(dict().items()), iter(dict().values()), - (lambda: (yield))(), + _test_gen(), (x for x in []), ] for x in samples: @@ -785,9 +960,90 @@ class TestOneTrickPonyABCs(ABCTestCase): self.assertRaises(RuntimeError, IgnoreGeneratorExit().close) + def test_AsyncGenerator(self): + class NonAGen1: + def __aiter__(self): return self + def __anext__(self): return None + def aclose(self): pass + def athrow(self, typ, val=None, tb=None): pass + + class NonAGen2: + def __aiter__(self): return self + def __anext__(self): return None + def aclose(self): pass + def asend(self, value): return value + + class NonAGen3: + def aclose(self): pass + def asend(self, value): return value + def athrow(self, typ, val=None, tb=None): pass + + non_samples = [ + None, 42, 3.14, 1j, b"", "", (), [], {}, set(), + iter(()), iter([]), NonAGen1(), NonAGen2(), NonAGen3()] + for x in non_samples: + self.assertNotIsInstance(x, AsyncGenerator) + self.assertFalse(issubclass(type(x), AsyncGenerator), repr(type(x))) + + class Gen: + def __aiter__(self): return self + async def __anext__(self): return None + async def aclose(self): pass + async def asend(self, value): return value + async def athrow(self, typ, val=None, tb=None): pass + + class MinimalAGen(AsyncGenerator): + async def asend(self, value): + return value + async def athrow(self, typ, val=None, tb=None): + await super().athrow(typ, val, tb) + + async def gen(): + yield 1 + + samples = [gen(), Gen(), MinimalAGen()] + for x in samples: + self.assertIsInstance(x, AsyncIterator) + self.assertIsInstance(x, AsyncGenerator) + self.assertTrue(issubclass(type(x), AsyncGenerator), repr(type(x))) + self.validate_abstract_methods(AsyncGenerator, 'asend', 'athrow') + + def run_async(coro): + result = None + while True: + try: + coro.send(None) + except StopIteration as ex: + result = ex.args[0] if ex.args else None + break + return result + + # mixin tests + mgen = MinimalAGen() + self.assertIs(mgen, mgen.__aiter__()) + self.assertIs(run_async(mgen.asend(None)), run_async(mgen.__anext__())) + self.assertEqual(2, run_async(mgen.asend(2))) + self.assertIsNone(run_async(mgen.aclose())) + with self.assertRaises(ValueError): + run_async(mgen.athrow(ValueError)) + + class FailOnClose(AsyncGenerator): + async def asend(self, value): return value + async def athrow(self, *args): raise ValueError + + with self.assertRaises(ValueError): + run_async(FailOnClose().aclose()) + + class IgnoreGeneratorExit(AsyncGenerator): + async def asend(self, value): return value + async def athrow(self, *args): pass + + with self.assertRaises(RuntimeError): + run_async(IgnoreGeneratorExit().aclose()) + def test_Sized(self): non_samples = [None, 42, 3.14, 1j, - (lambda: (yield))(), + _test_gen(), (x for x in []), ] for x in non_samples: @@ -805,7 +1061,7 @@ class TestOneTrickPonyABCs(ABCTestCase): def test_Container(self): non_samples = [None, 42, 3.14, 1j, - (lambda: (yield))(), + _test_gen(), (x for x in []), ] for x in non_samples: @@ -824,7 +1080,7 @@ class TestOneTrickPonyABCs(ABCTestCase): def test_Callable(self): non_samples = [None, 42, 3.14, 1j, "", b"", (), [], {}, set(), - (lambda: (yield))(), + _test_gen(), (x for x in []), ] for x in non_samples: @@ -842,14 +1098,14 @@ class TestOneTrickPonyABCs(ABCTestCase): self.validate_isinstance(Callable, '__call__') def test_direct_subclassing(self): - for B in Hashable, Iterable, Iterator, Sized, Container, Callable: + for B in Hashable, Iterable, Iterator, Reversible, Sized, Container, Callable: class C(B): pass self.assertTrue(issubclass(C, B)) self.assertFalse(issubclass(int, C)) def test_registration(self): - for B in Hashable, Iterable, Iterator, Sized, Container, Callable: + for B in Hashable, Iterable, Iterator, Reversible, Sized, Container, Callable: class C: __hash__ = None # Make sure it isn't hashable by default self.assertFalse(issubclass(C, B), B.__name__) @@ -1049,6 +1305,26 @@ class TestCollectionABCs(ABCTestCase): self.assertFalse(ncs > cs) self.assertTrue(ncs >= cs) + def test_issue26915(self): + # Container membership test should check identity first + class CustomEqualObject: + def __eq__(self, other): + return False + class CustomSequence(list): + def __contains__(self, value): + return Sequence.__contains__(self, value) + + nan = float('nan') + obj = CustomEqualObject() + containers = [ + CustomSequence([nan, obj]), + ItemsView({1: nan, 2: obj}), + ValuesView({1: nan, 2: obj}) + ] + for container in containers: + for elem in container: + self.assertIn(elem, container) + def assertSameSet(self, s1, s2): # coerce both to a real set then check equality self.assertSetEqual(set(s1), set(s2)) @@ -1219,6 +1495,7 @@ class TestCollectionABCs(ABCTestCase): def __iter__(self): return iter(()) self.validate_comparison(MyMapping()) + self.assertRaises(TypeError, reversed, MyMapping()) def test_MutableMapping(self): for sample in [dict]: diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 4ea66a488f..da1db1567b 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -473,10 +473,13 @@ if 1: self.assertEqual(d, {1: 2, 3: 4}) def test_compile_filename(self): - for filename in ('file.py', b'file.py', - bytearray(b'file.py'), memoryview(b'file.py')): + for filename in 'file.py', b'file.py': code = compile('pass', filename, 'exec') self.assertEqual(code.co_filename, 'file.py') + for filename in bytearray(b'file.py'), memoryview(b'file.py'): + with self.assertWarns(DeprecationWarning): + code = compile('pass', filename, 'exec') + self.assertEqual(code.co_filename, 'file.py') self.assertRaises(TypeError, compile, 'pass', list(b'file.py'), 'exec') @support.cpython_only @@ -664,6 +667,16 @@ if 1: self.assertTrue(f1(0)) self.assertTrue(f2(0.0)) + def test_path_like_objects(self): + # An implicit test for PyUnicode_FSDecoder(). + class PathLike: + def __init__(self, path): + self._path = path + def __fspath__(self): + return self._path + + compile("42", PathLike("test_compile_pathlike"), "single") + class TestStackSize(unittest.TestCase): # These tests check that the computed stack size for a code object diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 2ce8a61e39..2356efcaec 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -1,6 +1,7 @@ import sys import compileall import importlib.util +import test.test_importlib.util import os import pathlib import py_compile @@ -40,6 +41,11 @@ class CompileallTests(unittest.TestCase): def tearDown(self): shutil.rmtree(self.directory) + def add_bad_source_file(self): + self.bad_source_path = os.path.join(self.directory, '_test_bad.py') + with open(self.bad_source_path, 'w') as file: + file.write('x (\n') + def data(self): with open(self.bc_path, 'rb') as file: data = file.read(8) @@ -78,15 +84,47 @@ class CompileallTests(unittest.TestCase): os.unlink(fn) except: pass - compileall.compile_file(self.source_path, force=False, quiet=True) + self.assertTrue(compileall.compile_file(self.source_path, + force=False, quiet=True)) self.assertTrue(os.path.isfile(self.bc_path) and not os.path.isfile(self.bc_path2)) os.unlink(self.bc_path) - compileall.compile_dir(self.directory, force=False, quiet=True) + self.assertTrue(compileall.compile_dir(self.directory, force=False, + quiet=True)) self.assertTrue(os.path.isfile(self.bc_path) and os.path.isfile(self.bc_path2)) os.unlink(self.bc_path) os.unlink(self.bc_path2) + # Test against bad files + self.add_bad_source_file() + self.assertFalse(compileall.compile_file(self.bad_source_path, + force=False, quiet=2)) + self.assertFalse(compileall.compile_dir(self.directory, + force=False, quiet=2)) + + def test_compile_file_pathlike(self): + self.assertFalse(os.path.isfile(self.bc_path)) + # we should also test the output + with support.captured_stdout() as stdout: + self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path))) + self.assertRegex(stdout.getvalue(), r'Compiling ([^WindowsPath|PosixPath].*)') + self.assertTrue(os.path.isfile(self.bc_path)) + + def test_compile_file_pathlike_ddir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), + ddir=pathlib.Path('ddir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + + def test_compile_path(self): + with test.test_importlib.util.import_state(path=[self.directory]): + self.assertTrue(compileall.compile_path(quiet=2)) + + with test.test_importlib.util.import_state(path=[self.directory]): + self.add_bad_source_file() + self.assertFalse(compileall.compile_path(skip_curdir=False, + force=True, quiet=2)) def test_no_pycache_in_non_package(self): # Bug 8563 reported that __pycache__ directories got created by @@ -115,6 +153,14 @@ class CompileallTests(unittest.TestCase): optimization=opt) self.assertTrue(os.path.isfile(cached3)) + def test_compile_dir_pathlike(self): + self.assertFalse(os.path.isfile(self.bc_path)) + with support.captured_stdout() as stdout: + compileall.compile_dir(pathlib.Path(self.directory)) + line = stdout.getvalue().splitlines()[0] + self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)') + self.assertTrue(os.path.isfile(self.bc_path)) + @mock.patch('compileall.ProcessPoolExecutor') def test_compile_pool_called(self, pool_mock): compileall.compile_dir(self.directory, quiet=True, workers=5) @@ -197,10 +243,9 @@ class CommandLineTests(unittest.TestCase): raise unittest.SkipTest('not all entries on sys.path are writable') def _get_run_args(self, args): - interp_args = ['-S'] - if sys.flags.optimize: - interp_args.append({1 : '-O', 2 : '-OO'}[sys.flags.optimize]) - return interp_args + ['-m', 'compileall'] + list(args) + return [*support.optim_args_from_interpreter_flags(), + '-S', '-m', 'compileall', + *args] def assertRunOK(self, *args, **env_vars): rc, out, err = script_helper.assert_python_ok( diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index 403ee3bc58..c249ca724b 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -1,5 +1,7 @@ import unittest from test import support +from test.test_grammar import (VALID_UNDERSCORE_LITERALS, + INVALID_UNDERSCORE_LITERALS) from random import random from math import atan2, isnan, copysign @@ -385,6 +387,18 @@ class ComplexTest(unittest.TestCase): self.assertAlmostEqual(complex(complex1(1j)), 2j) self.assertRaises(TypeError, complex, complex2(1j)) + def test_underscores(self): + # check underscores + for lit in VALID_UNDERSCORE_LITERALS: + if not any(ch in lit for ch in 'xXoObB'): + self.assertEqual(complex(lit), eval(lit)) + self.assertEqual(complex(lit), complex(lit.replace('_', ''))) + for lit in INVALID_UNDERSCORE_LITERALS: + if lit in ('0_7', '09_99'): # octals are not recognized here + continue + if not any(ch in lit for ch in 'xXoObB'): + self.assertRaises(ValueError, complex, lit) + def test_hash(self): for x in range(-30, 30): self.assertEqual(hash(x), hash(complex(x, 0))) diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 7513815ce5..23e95b2124 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -154,6 +154,30 @@ class ThreadPoolShutdownTest(ThreadPoolMixin, ExecutorShutdownTest, unittest.Tes for t in threads: t.join() + def test_thread_names_assigned(self): + executor = futures.ThreadPoolExecutor( + max_workers=5, thread_name_prefix='SpecialPool') + executor.map(abs, range(-5, 5)) + threads = executor._threads + del executor + + for t in threads: + self.assertRegex(t.name, r'^SpecialPool_[0-4]$') + t.join() + + def test_thread_names_default(self): + executor = futures.ThreadPoolExecutor(max_workers=5) + executor.map(abs, range(-5, 5)) + threads = executor._threads + del executor + + for t in threads: + # We don't particularly care what the default name is, just that + # it has a default name implying that it is a ThreadPoolExecutor + # followed by what looks like a thread number. + self.assertRegex(t.name, r'^.*ThreadPoolExecutor.*_[0-4]$') + t.join() + class ProcessPoolShutdownTest(ProcessPoolMixin, ExecutorShutdownTest, unittest.TestCase): def _prime_executor(self): diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 864a1d0802..0d06080da3 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2076,5 +2076,11 @@ class BlatantOverrideConvertersTestCase(unittest.TestCase): self.assertEqual(cfg['two'].getlen('one'), 5) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {"Error"} + support.check__all__(self, configparser, blacklist=blacklist) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_contains.py b/Lib/test/test_contains.py index 3c6bdeffda..036a1d012d 100644 --- a/Lib/test/test_contains.py +++ b/Lib/test/test_contains.py @@ -84,6 +84,31 @@ class TestContains(unittest.TestCase): self.assertTrue(container == constructor(values)) self.assertTrue(container == container) + def test_block_fallback(self): + # blocking fallback with __contains__ = None + class ByContains(object): + def __contains__(self, other): + return False + c = ByContains() + class BlockContains(ByContains): + """Is not a container + + This class is a perfectly good iterable (as tested by + list(bc)), as well as inheriting from a perfectly good + container, but __contains__ = None prevents the usual + fallback to iteration in the container protocol. That + is, normally, 0 in bc would fall back to the equivalent + of any(x==0 for x in bc), but here it's blocked from + doing so. + """ + def __iter__(self): + while False: + yield None + __contains__ = None + bc = BlockContains() + self.assertFalse(0 in c) + self.assertFalse(0 in list(bc)) + self.assertRaises(TypeError, lambda: 0 in bc) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 516403ef65..c04c804af5 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -12,6 +12,39 @@ except ImportError: threading = None +class TestAbstractContextManager(unittest.TestCase): + + def test_enter(self): + class DefaultEnter(AbstractContextManager): + def __exit__(self, *args): + super().__exit__(*args) + + manager = DefaultEnter() + self.assertIs(manager.__enter__(), manager) + + def test_exit_is_abstract(self): + class MissingExit(AbstractContextManager): + pass + + with self.assertRaises(TypeError): + MissingExit() + + def test_structural_subclassing(self): + class ManagerFromScratch: + def __enter__(self): + return self + def __exit__(self, exc_type, exc_value, traceback): + return None + + self.assertTrue(issubclass(ManagerFromScratch, AbstractContextManager)) + + class DefaultEnter(AbstractContextManager): + def __exit__(self, *args): + super().__exit__(*args) + + self.assertTrue(issubclass(DefaultEnter, AbstractContextManager)) + + class ContextManagerTestCase(unittest.TestCase): def test_contextmanager_plain(self): @@ -89,7 +122,7 @@ class ContextManagerTestCase(unittest.TestCase): def woohoo(): yield try: - with self.assertWarnsRegex(PendingDeprecationWarning, + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): with woohoo(): raise stop_exc diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 0e1f670bf0..45a692022f 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -98,7 +98,7 @@ class TestCopy(unittest.TestCase): tests = [None, ..., NotImplemented, 42, 2**100, 3.14, True, False, 1j, "hello", "hello\u1234", f.__code__, - b"world", bytes(range(256)), range(10), + b"world", bytes(range(256)), range(10), slice(1, 10, 2), NewStyle, Classic, max, WithMetaclass] for x in tests: self.assertIs(copy.copy(x), x) diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 4a327b5ba9..78439a2aca 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -69,55 +69,130 @@ def silence_coro_gc(): class AsyncBadSyntaxTest(unittest.TestCase): def test_badsyntax_1(self): - with self.assertRaisesRegex(SyntaxError, "'await' outside"): - import test.badsyntax_async1 + samples = [ + """def foo(): + await something() + """, - def test_badsyntax_2(self): - with self.assertRaisesRegex(SyntaxError, "'await' outside"): - import test.badsyntax_async2 + """await something()""", - def test_badsyntax_3(self): - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): - import test.badsyntax_async3 + """async def foo(): + yield from [] + """, - def test_badsyntax_4(self): - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): - import test.badsyntax_async4 + """async def foo(): + await await fut + """, - def test_badsyntax_5(self): - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): - import test.badsyntax_async5 + """async def foo(a=await something()): + pass + """, - def test_badsyntax_6(self): - with self.assertRaisesRegex( - SyntaxError, "'yield' inside async function"): + """async def foo(a:await something()): + pass + """, - import test.badsyntax_async6 + """async def foo(): + def bar(): + [i async for i in els] + """, - def test_badsyntax_7(self): - with self.assertRaisesRegex( - SyntaxError, "'yield from' inside async function"): + """async def foo(): + def bar(): + [await i for i in els] + """, - import test.badsyntax_async7 + """async def foo(): + def bar(): + [i for i in els + async for b in els] + """, - def test_badsyntax_8(self): - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): - import test.badsyntax_async8 + """async def foo(): + def bar(): + [i for i in els + for c in b + async for b in els] + """, - def test_badsyntax_9(self): - ns = {} - for comp in {'(await a for a in b)', - '[await a for a in b]', - '{await a for a in b}', - '{await a: c for a in b}'}: + """async def foo(): + def bar(): + [i for i in els + async for b in els + for c in b] + """, - with self.assertRaisesRegex(SyntaxError, 'await.*in comprehen'): - exec('async def f():\n\t{}'.format(comp), ns, ns) + """async def foo(): + def bar(): + [i for i in els + for b in await els] + """, - def test_badsyntax_10(self): - # Tests for issue 24619 + """async def foo(): + def bar(): + [i for i in els + for b in els + if await b] + """, + + """async def foo(): + def bar(): + [i for i in await els] + """, + + """async def foo(): + def bar(): + [i for i in els if await i] + """, + + """def bar(): + [i async for i in els] + """, + + """def bar(): + [await i for i in els] + """, + + """def bar(): + [i for i in els + async for b in els] + """, + + """def bar(): + [i for i in els + for c in b + async for b in els] + """, + + """def bar(): + [i for i in els + async for b in els + for c in b] + """, + + """def bar(): + [i for i in els + for b in await els] + """, + + """def bar(): + [i for i in els + for b in els + if await b] + """, + + """def bar(): + [i for i in await els] + """, + + """def bar(): + [i for i in els if await i] + """, + + """async def foo(): + await + """, - samples = [ """async def foo(): def bar(): pass await = 1 @@ -283,57 +358,110 @@ class AsyncBadSyntaxTest(unittest.TestCase): with self.subTest(code=code), self.assertRaises(SyntaxError): compile(code, "<test>", "exec") - def test_goodsyntax_1(self): - # Tests for issue 24619 + def test_badsyntax_2(self): + samples = [ + """def foo(): + await = 1 + """, + + """class Bar: + def async(): pass + """, - def foo(await): - async def foo(): pass - async def foo(): + """class Bar: + async = 1 + """, + + """class async: pass - return await + 1 - self.assertEqual(foo(10), 11) + """, - def foo(await): - async def foo(): pass - async def foo(): pass - return await + 2 - self.assertEqual(foo(20), 22) + """class await: + pass + """, - def foo(await): + """import math as await""", - async def foo(): pass + """def async(): + pass""", - async def foo(): pass + """def foo(*, await=1): + pass""" - return await + 2 - self.assertEqual(foo(20), 22) + """async = 1""", - def foo(await): - """spam""" - async def foo(): \ - pass - # 123 - async def foo(): pass - # 456 - return await + 2 - self.assertEqual(foo(20), 22) - - def foo(await): - def foo(): pass - def foo(): pass - async def bar(): return await_ - await_ = await - try: - bar().send(None) - except StopIteration as ex: - return ex.args[0] - self.assertEqual(foo(42), 42) + """print(await=1)""" + ] - async def f(): - async def g(): pass - await z - await = 1 - self.assertTrue(inspect.iscoroutinefunction(f)) + for code in samples: + with self.subTest(code=code), self.assertWarnsRegex( + DeprecationWarning, + "'await' will become reserved keywords"): + compile(code, "<test>", "exec") + + def test_badsyntax_3(self): + with self.assertRaises(DeprecationWarning): + with warnings.catch_warnings(): + warnings.simplefilter("error") + compile("async = 1", "<test>", "exec") + + def test_goodsyntax_1(self): + # Tests for issue 24619 + + samples = [ + '''def foo(await): + async def foo(): pass + async def foo(): + pass + return await + 1 + ''', + + '''def foo(await): + async def foo(): pass + async def foo(): pass + return await + 1 + ''', + + '''def foo(await): + + async def foo(): pass + + async def foo(): pass + + return await + 1 + ''', + + '''def foo(await): + """spam""" + async def foo(): \ + pass + # 123 + async def foo(): pass + # 456 + return await + 1 + ''', + + '''def foo(await): + def foo(): pass + def foo(): pass + async def bar(): return await_ + await_ = await + try: + bar().send(None) + except StopIteration as ex: + return ex.args[0] + 1 + ''' + ] + + for code in samples: + with self.subTest(code=code): + loc = {} + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + exec(code, loc, loc) + + self.assertEqual(loc['foo'](10), 11) class TokenizerRegrTest(unittest.TestCase): @@ -906,7 +1034,7 @@ class CoroutineTest(unittest.TestCase): return await Awaitable() with self.assertRaisesRegex( - TypeError, "__await__\(\) returned a coroutine"): + TypeError, r"__await__\(\) returned a coroutine"): run_async(foo()) @@ -1270,7 +1398,7 @@ class CoroutineTest(unittest.TestCase): buffer = [] async def test1(): - with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + with self.assertWarnsRegex(DeprecationWarning, "legacy"): async for i1, i2 in AsyncIter(): buffer.append(i1 + i2) @@ -1284,7 +1412,7 @@ class CoroutineTest(unittest.TestCase): buffer = [] async def test2(): nonlocal buffer - with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + with self.assertWarnsRegex(DeprecationWarning, "legacy"): async for i in AsyncIter(): buffer.append(i[0]) if i[0] == 20: @@ -1303,7 +1431,7 @@ class CoroutineTest(unittest.TestCase): buffer = [] async def test3(): nonlocal buffer - with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + with self.assertWarnsRegex(DeprecationWarning, "legacy"): async for i in AsyncIter(): if i[0] > 20: continue @@ -1348,7 +1476,7 @@ class CoroutineTest(unittest.TestCase): with self.assertRaisesRegex( TypeError, - "async for' received an invalid object.*__aiter.*\: I"): + r"async for' received an invalid object.*__aiter.*\: I"): run_async(foo()) @@ -1386,7 +1514,7 @@ class CoroutineTest(unittest.TestCase): return 123 async def foo(): - with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + with self.assertWarnsRegex(DeprecationWarning, "legacy"): async for i in I(): print('never going to happen') @@ -1495,7 +1623,7 @@ class CoroutineTest(unittest.TestCase): 1/0 async def foo(): nonlocal CNT - with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + with self.assertWarnsRegex(DeprecationWarning, "legacy"): async for i in AI(): CNT += 1 CNT += 10 @@ -1522,7 +1650,7 @@ class CoroutineTest(unittest.TestCase): self.assertEqual(CNT, 0) def test_for_9(self): - # Test that PendingDeprecationWarning can safely be converted into + # Test that DeprecationWarning can safely be converted into # an exception (__aiter__ should not have a chance to raise # a ZeroDivisionError.) class AI: @@ -1532,13 +1660,13 @@ class CoroutineTest(unittest.TestCase): async for i in AI(): pass - with self.assertRaises(PendingDeprecationWarning): + with self.assertRaises(DeprecationWarning): with warnings.catch_warnings(): warnings.simplefilter("error") run_async(foo()) def test_for_10(self): - # Test that PendingDeprecationWarning can safely be converted into + # Test that DeprecationWarning can safely be converted into # an exception. class AI: async def __aiter__(self): @@ -1547,7 +1675,7 @@ class CoroutineTest(unittest.TestCase): async for i in AI(): pass - with self.assertRaises(PendingDeprecationWarning): + with self.assertRaises(DeprecationWarning): with warnings.catch_warnings(): warnings.simplefilter("error") run_async(foo()) @@ -1598,6 +1726,185 @@ class CoroutineTest(unittest.TestCase): foo().send(None) self.assertEqual(result, [42]) + def test_comp_1(self): + async def f(i): + return i + + async def run_list(): + return [await c for c in [f(1), f(41)]] + + async def run_set(): + return {await c for c in [f(1), f(41)]} + + async def run_dict1(): + return {await c: 'a' for c in [f(1), f(41)]} + + async def run_dict2(): + return {i: await c for i, c in enumerate([f(1), f(41)])} + + self.assertEqual(run_async(run_list()), ([], [1, 41])) + self.assertEqual(run_async(run_set()), ([], {1, 41})) + self.assertEqual(run_async(run_dict1()), ([], {1: 'a', 41: 'a'})) + self.assertEqual(run_async(run_dict2()), ([], {0: 1, 1: 41})) + + def test_comp_2(self): + async def f(i): + return i + + async def run_list(): + return [s for c in [f(''), f('abc'), f(''), f(['de', 'fg'])] + for s in await c] + + self.assertEqual( + run_async(run_list()), + ([], ['a', 'b', 'c', 'de', 'fg'])) + + async def run_set(): + return {d + for c in [f([f([10, 30]), + f([20])])] + for s in await c + for d in await s} + + self.assertEqual( + run_async(run_set()), + ([], {10, 20, 30})) + + async def run_set2(): + return {await s + for c in [f([f(10), f(20)])] + for s in await c} + + self.assertEqual( + run_async(run_set2()), + ([], {10, 20})) + + def test_comp_3(self): + async def f(it): + for i in it: + yield i + + async def run_list(): + return [i + 1 async for i in f([10, 20])] + self.assertEqual( + run_async(run_list()), + ([], [11, 21])) + + async def run_set(): + return {i + 1 async for i in f([10, 20])} + self.assertEqual( + run_async(run_set()), + ([], {11, 21})) + + async def run_dict(): + return {i + 1: i + 2 async for i in f([10, 20])} + self.assertEqual( + run_async(run_dict()), + ([], {11: 12, 21: 22})) + + async def run_gen(): + gen = (i + 1 async for i in f([10, 20])) + return [g + 100 async for g in gen] + self.assertEqual( + run_async(run_gen()), + ([], [111, 121])) + + def test_comp_4(self): + async def f(it): + for i in it: + yield i + + async def run_list(): + return [i + 1 async for i in f([10, 20]) if i > 10] + self.assertEqual( + run_async(run_list()), + ([], [21])) + + async def run_set(): + return {i + 1 async for i in f([10, 20]) if i > 10} + self.assertEqual( + run_async(run_set()), + ([], {21})) + + async def run_dict(): + return {i + 1: i + 2 async for i in f([10, 20]) if i > 10} + self.assertEqual( + run_async(run_dict()), + ([], {21: 22})) + + async def run_gen(): + gen = (i + 1 async for i in f([10, 20]) if i > 10) + return [g + 100 async for g in gen] + self.assertEqual( + run_async(run_gen()), + ([], [121])) + + def test_comp_5(self): + async def f(it): + for i in it: + yield i + + async def run_list(): + return [i + 1 for pair in ([10, 20], [30, 40]) if pair[0] > 10 + async for i in f(pair) if i > 30] + self.assertEqual( + run_async(run_list()), + ([], [41])) + + def test_comp_6(self): + async def f(it): + for i in it: + yield i + + async def run_list(): + return [i + 1 async for seq in f([(10, 20), (30,)]) + for i in seq] + + self.assertEqual( + run_async(run_list()), + ([], [11, 21, 31])) + + def test_comp_7(self): + async def f(): + yield 1 + yield 2 + raise Exception('aaa') + + async def run_list(): + return [i async for i in f()] + + with self.assertRaisesRegex(Exception, 'aaa'): + run_async(run_list()) + + def test_comp_8(self): + async def f(): + return [i for i in [1, 2, 3]] + + self.assertEqual( + run_async(f()), + ([], [1, 2, 3])) + + def test_comp_9(self): + async def gen(): + yield 1 + yield 2 + async def f(): + l = [i async for i in gen()] + return [i for i in l] + + self.assertEqual( + run_async(f()), + ([], [1, 2])) + + def test_comp_10(self): + async def f(): + xx = {i for i in [1, 2, 3]} + return {x: x for x in xx} + + self.assertEqual( + run_async(f()), + ([], {1: 1, 2: 2, 3: 3})) + def test_copy(self): async def func(): pass coro = func() @@ -1728,8 +2035,8 @@ class SysSetCoroWrapperTest(unittest.TestCase): try: with silence_coro_gc(), self.assertRaisesRegex( RuntimeError, - "coroutine wrapper.*\.wrapper at 0x.*attempted to " - "recursively wrap .* wrap .*"): + r"coroutine wrapper.*\.wrapper at 0x.*attempted to " + r"recursively wrap .* wrap .*"): foo() finally: diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index f18983fbb2..53f8917330 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -6,7 +6,7 @@ from test.support import run_unittest, TESTFN, unlink # rip off all interesting stuff from test_profile import cProfile from test.test_profile import ProfileTest, regenerate_expected_output -from test.profilee import testfunc + class CProfileTest(ProfileTest): profilerclass = cProfile.Profile diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 77c315edb1..03ab1840dd 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -2,9 +2,7 @@ # csv package unit tests import copy -import io import sys -import os import unittest from io import StringIO from tempfile import TemporaryFile @@ -12,6 +10,9 @@ import csv import gc import pickle from test import support +from itertools import permutations +from textwrap import dedent +from collections import OrderedDict class Test_Csv(unittest.TestCase): """ @@ -426,17 +427,16 @@ class TestDialectRegistry(unittest.TestCase): self.assertRaises(TypeError, csv.reader, [], quoting = -1) self.assertRaises(TypeError, csv.reader, [], quoting = 100) - # See issue #22995 - ## def test_copy(self): - ## for name in csv.list_dialects(): - ## dialect = csv.get_dialect(name) - ## self.assertRaises(TypeError, copy.copy, dialect) + def test_copy(self): + for name in csv.list_dialects(): + dialect = csv.get_dialect(name) + self.assertRaises(TypeError, copy.copy, dialect) - ## def test_pickle(self): - ## for name in csv.list_dialects(): - ## dialect = csv.get_dialect(name) - ## for proto in range(pickle.HIGHEST_PROTOCOL + 1): - ## self.assertRaises(TypeError, pickle.dumps, dialect, proto) + def test_pickle(self): + for name in csv.list_dialects(): + dialect = csv.get_dialect(name) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertRaises(TypeError, pickle.dumps, dialect, proto) class TestCsvBase(unittest.TestCase): def readerAssertEqual(self, input, expected_result): @@ -626,6 +626,24 @@ class TestDictFields(unittest.TestCase): self.assertNotIn("'f2'", exception) self.assertIn("1", exception) + def test_typo_in_extrasaction_raises_error(self): + fileobj = StringIO() + self.assertRaises(ValueError, csv.DictWriter, fileobj, ['f1', 'f2'], + extrasaction="raised") + + def test_write_field_not_in_field_names_raise(self): + fileobj = StringIO() + writer = csv.DictWriter(fileobj, ['f1', 'f2'], extrasaction="raise") + dictrow = {'f0': 0, 'f1': 1, 'f2': 2, 'f3': 3} + self.assertRaises(ValueError, csv.DictWriter.writerow, writer, dictrow) + + def test_write_field_not_in_field_names_ignore(self): + fileobj = StringIO() + writer = csv.DictWriter(fileobj, ['f1', 'f2'], extrasaction="ignore") + dictrow = {'f0': 0, 'f1': 1, 'f2': 2, 'f3': 3} + csv.DictWriter.writerow(writer, dictrow) + self.assertEqual(fileobj.getvalue(), "1,2\r\n") + def test_read_dict_fields(self): with TemporaryFile("w+") as fileobj: fileobj.write("1,2,abc\r\n") @@ -1080,7 +1098,6 @@ class TestUnicode(unittest.TestCase): "François Pinard"] def test_unicode_read(self): - import io with TemporaryFile("w+", newline='', encoding="utf-8") as fileobj: fileobj.write(",".join(self.names) + "\r\n") fileobj.seek(0) @@ -1089,7 +1106,6 @@ class TestUnicode(unittest.TestCase): def test_unicode_write(self): - import io with TemporaryFile("w+", newline='', encoding="utf-8") as fileobj: writer = csv.writer(fileobj) writer.writerow(self.names) @@ -1097,6 +1113,63 @@ class TestUnicode(unittest.TestCase): fileobj.seek(0) self.assertEqual(fileobj.read(), expected) +class KeyOrderingTest(unittest.TestCase): + + def test_ordering_for_the_dict_reader_and_writer(self): + resultset = set() + for keys in permutations("abcde"): + with TemporaryFile('w+', newline='', encoding="utf-8") as fileobject: + dw = csv.DictWriter(fileobject, keys) + dw.writeheader() + fileobject.seek(0) + dr = csv.DictReader(fileobject) + kt = tuple(dr.fieldnames) + self.assertEqual(keys, kt) + resultset.add(kt) + # Final sanity check: were all permutations unique? + self.assertEqual(len(resultset), 120, "Key ordering: some key permutations not collected (expected 120)") + + def test_ordered_dict_reader(self): + data = dedent('''\ + FirstName,LastName + Eric,Idle + Graham,Chapman,Over1,Over2 + + Under1 + John,Cleese + ''').splitlines() + + self.assertEqual(list(csv.DictReader(data)), + [OrderedDict([('FirstName', 'Eric'), ('LastName', 'Idle')]), + OrderedDict([('FirstName', 'Graham'), ('LastName', 'Chapman'), + (None, ['Over1', 'Over2'])]), + OrderedDict([('FirstName', 'Under1'), ('LastName', None)]), + OrderedDict([('FirstName', 'John'), ('LastName', 'Cleese')]), + ]) + + self.assertEqual(list(csv.DictReader(data, restkey='OtherInfo')), + [OrderedDict([('FirstName', 'Eric'), ('LastName', 'Idle')]), + OrderedDict([('FirstName', 'Graham'), ('LastName', 'Chapman'), + ('OtherInfo', ['Over1', 'Over2'])]), + OrderedDict([('FirstName', 'Under1'), ('LastName', None)]), + OrderedDict([('FirstName', 'John'), ('LastName', 'Cleese')]), + ]) + + del data[0] # Remove the header row + self.assertEqual(list(csv.DictReader(data, fieldnames=['fname', 'lname'])), + [OrderedDict([('fname', 'Eric'), ('lname', 'Idle')]), + OrderedDict([('fname', 'Graham'), ('lname', 'Chapman'), + (None, ['Over1', 'Over2'])]), + OrderedDict([('fname', 'Under1'), ('lname', None)]), + OrderedDict([('fname', 'John'), ('lname', 'Cleese')]), + ]) + + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + extra = {'__doc__', '__version__'} + support.check__all__(self, csv, ('csv', '_csv'), extra=extra) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py index 2d4eb52c62..242e1bba03 100644 --- a/Lib/test/test_datetime.py +++ b/Lib/test/test_datetime.py @@ -23,9 +23,16 @@ test_suffixes = ["_Pure", "_Fast"] test_classes = [] for module, suffix in zip(test_modules, test_suffixes): + test_classes = [] for name, cls in module.__dict__.items(): - if not (isinstance(cls, type) and issubclass(cls, unittest.TestCase)): + if not isinstance(cls, type): continue + if issubclass(cls, unittest.TestCase): + test_classes.append(cls) + elif issubclass(cls, unittest.TestSuite): + suit = cls() + test_classes.extend(type(test) for test in suit) + for cls in test_classes: cls.__name__ = name + suffix @classmethod def setUpClass(cls_, module=module): @@ -39,7 +46,6 @@ for module, suffix in zip(test_modules, test_suffixes): sys.modules.update(cls_._save_sys_modules) cls.setUpClass = setUpClass cls.tearDownClass = tearDownClass - test_classes.append(cls) def test_main(): run_unittest(*test_classes) diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index 623d9929d5..f0a428deaf 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -1,6 +1,5 @@ """Test script for the dbm.open function based on testdumbdbm.py""" -import os import unittest import glob import test.support diff --git a/Lib/test/test_dbm_dumb.py b/Lib/test/test_dbm_dumb.py index 235ab11b61..df531d64e4 100644 --- a/Lib/test/test_dbm_dumb.py +++ b/Lib/test/test_dbm_dumb.py @@ -7,6 +7,7 @@ import operator import os import stat import unittest +import warnings import dbm.dumb as dumbdbm from test import support from functools import partial @@ -79,6 +80,12 @@ class DumbDBMTestCase(unittest.TestCase): self.init_db() f = dumbdbm.open(_fname, 'r') self.read_helper(f) + with self.assertWarnsRegex(DeprecationWarning, + 'The database is opened for reading only'): + f[b'g'] = b'x' + with self.assertWarnsRegex(DeprecationWarning, + 'The database is opened for reading only'): + del f[b'a'] f.close() def test_dumbdbm_keys(self): @@ -149,7 +156,7 @@ class DumbDBMTestCase(unittest.TestCase): self.assertEqual(self._dict[key], f[key]) def init_db(self): - f = dumbdbm.open(_fname, 'w') + f = dumbdbm.open(_fname, 'n') for k in self._dict: f[k] = self._dict[k] f.close() @@ -235,6 +242,24 @@ class DumbDBMTestCase(unittest.TestCase): pass self.assertEqual(stdout.getvalue(), '') + def test_warn_on_ignored_flags(self): + for value in ('r', 'w'): + _delete_files() + with self.assertWarnsRegex(DeprecationWarning, + "The database file is missing, the " + "semantics of the 'c' flag will " + "be used."): + f = dumbdbm.open(_fname, value) + f.close() + + def test_invalid_flag(self): + for flag in ('x', 'rf', None): + with self.assertWarnsRegex(DeprecationWarning, + "Flag must be one of " + "'r', 'w', 'c', or 'n'"): + f = dumbdbm.open(_fname, flag) + f.close() + @unittest.skipUnless(hasattr(os, 'chmod'), 'test needs os.chmod()') def test_readonly_files(self): with support.temp_dir() as dir: diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index a7808f51c7..304b332869 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -2,7 +2,7 @@ from test import support gdbm = support.import_module("dbm.gnu") #skip if not supported import unittest import os -from test.support import verbose, TESTFN, unlink +from test.support import TESTFN, unlink filename = TESTFN diff --git a/Lib/test/test_dbm_ndbm.py b/Lib/test/test_dbm_ndbm.py index 2291561def..49f4426e4c 100644 --- a/Lib/test/test_dbm_ndbm.py +++ b/Lib/test/test_dbm_ndbm.py @@ -1,8 +1,6 @@ from test import support support.import_module("dbm.ndbm") #skip if not supported import unittest -import os -import random import dbm.ndbm from dbm.ndbm import error diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 1aa0bf87d0..617a37eec8 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -554,6 +554,10 @@ class ExplicitConstructionTest(unittest.TestCase): self.assertEqual(str(Decimal(' -7.89')), '-7.89') self.assertEqual(str(Decimal(" 3.45679 ")), '3.45679') + # underscores + self.assertEqual(str(Decimal('1_3.3e4_0')), '1.33E+41') + self.assertEqual(str(Decimal('1_0_0_0')), '1000') + # unicode whitespace for lead in ["", ' ', '\u00a0', '\u205f']: for trail in ["", ' ', '\u00a0', '\u205f']: @@ -578,6 +582,9 @@ class ExplicitConstructionTest(unittest.TestCase): # embedded NUL self.assertRaises(InvalidOperation, Decimal, "12\u00003") + # underscores don't prevent errors + self.assertRaises(InvalidOperation, Decimal, "1_2_\u00003") + @cpython_only def test_from_legacy_strings(self): import _testcapi @@ -772,6 +779,9 @@ class ExplicitConstructionTest(unittest.TestCase): self.assertRaises(InvalidOperation, nc.create_decimal, "xyz") self.assertRaises(ValueError, nc.create_decimal, (1, "xyz", -25)) self.assertRaises(TypeError, nc.create_decimal, "1234", "5678") + # no whitespace and underscore stripping is done with this method + self.assertRaises(InvalidOperation, nc.create_decimal, " 1234") + self.assertRaises(InvalidOperation, nc.create_decimal, "12_34") # too many NaN payload digits nc.prec = 3 @@ -2047,6 +2057,39 @@ class UsabilityTest(unittest.TestCase): d = Decimal( (1, (0, 2, 7, 1), 'F') ) self.assertEqual(d.as_tuple(), (1, (0,), 'F')) + def test_as_integer_ratio(self): + Decimal = self.decimal.Decimal + + # exceptional cases + self.assertRaises(OverflowError, + Decimal.as_integer_ratio, Decimal('inf')) + self.assertRaises(OverflowError, + Decimal.as_integer_ratio, Decimal('-inf')) + self.assertRaises(ValueError, + Decimal.as_integer_ratio, Decimal('-nan')) + self.assertRaises(ValueError, + Decimal.as_integer_ratio, Decimal('snan123')) + + for exp in range(-4, 2): + for coeff in range(1000): + for sign in '+', '-': + d = Decimal('%s%dE%d' % (sign, coeff, exp)) + pq = d.as_integer_ratio() + p, q = pq + + # check return type + self.assertIsInstance(pq, tuple) + self.assertIsInstance(p, int) + self.assertIsInstance(q, int) + + # check normalization: q should be positive; + # p should be relatively prime to q. + self.assertGreater(q, 0) + self.assertEqual(math.gcd(p, q), 1) + + # check that p/q actually gives the correct value + self.assertEqual(Decimal(p) / Decimal(q), d) + def test_subclassing(self): # Different behaviours when subclassing Decimal Decimal = self.decimal.Decimal diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 18e1df0825..ce517b51d5 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -289,7 +289,7 @@ class TestBasic(unittest.TestCase): else: self.assertEqual(d.index(element, start, stop), target) - def test_insert_bug_24913(self): + def test_index_bug_24913(self): d = deque('A' * 3) with self.assertRaises(ValueError): i = d.index("Hello world", 0, 4) @@ -622,20 +622,22 @@ class TestBasic(unittest.TestCase): self.assertEqual(list(d), list(e)) def test_pickle(self): - d = deque(range(200)) - for i in range(pickle.HIGHEST_PROTOCOL + 1): - s = pickle.dumps(d, i) - e = pickle.loads(s) - self.assertNotEqual(id(d), id(e)) - self.assertEqual(list(d), list(e)) - -## def test_pickle_recursive(self): -## d = deque('abc') -## d.append(d) -## for i in range(pickle.HIGHEST_PROTOCOL + 1): -## e = pickle.loads(pickle.dumps(d, i)) -## self.assertNotEqual(id(d), id(e)) -## self.assertEqual(id(e), id(e[-1])) + for d in deque(range(200)), deque(range(200), 100): + for i in range(pickle.HIGHEST_PROTOCOL + 1): + s = pickle.dumps(d, i) + e = pickle.loads(s) + self.assertNotEqual(id(e), id(d)) + self.assertEqual(list(e), list(d)) + self.assertEqual(e.maxlen, d.maxlen) + + def test_pickle_recursive(self): + for d in deque('abc'), deque('abc', 3): + d.append(d) + for i in range(pickle.HIGHEST_PROTOCOL + 1): + e = pickle.loads(pickle.dumps(d, i)) + self.assertNotEqual(id(e), id(d)) + self.assertEqual(id(e[-1]), id(e)) + self.assertEqual(e.maxlen, d.maxlen) def test_iterator_pickle(self): orig = deque(range(200)) @@ -696,6 +698,15 @@ class TestBasic(unittest.TestCase): self.assertNotEqual(id(d), id(e)) self.assertEqual(list(d), list(e)) + for i in range(5): + for maxlen in range(-1, 6): + s = [random.random() for j in range(i)] + d = deque(s) if maxlen == -1 else deque(s, maxlen) + e = d.copy() + self.assertEqual(d, e) + self.assertEqual(d.maxlen, e.maxlen) + self.assertTrue(all(x is y for x, y in zip(d, e))) + def test_copy_method(self): mut = [10] d = deque([mut]) @@ -845,24 +856,26 @@ class TestSubclass(unittest.TestCase): self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e)) -## def test_pickle(self): -## d = Deque('abc') -## d.append(d) -## -## e = pickle.loads(pickle.dumps(d)) -## self.assertNotEqual(id(d), id(e)) -## self.assertEqual(type(d), type(e)) -## dd = d.pop() -## ee = e.pop() -## self.assertEqual(id(e), id(ee)) -## self.assertEqual(d, e) -## -## d.x = d -## e = pickle.loads(pickle.dumps(d)) -## self.assertEqual(id(e), id(e.x)) -## -## d = DequeWithBadIter('abc') -## self.assertRaises(TypeError, pickle.dumps, d) + def test_pickle_recursive(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + for d in Deque('abc'), Deque('abc', 3): + d.append(d) + + e = pickle.loads(pickle.dumps(d, proto)) + self.assertNotEqual(id(e), id(d)) + self.assertEqual(type(e), type(d)) + self.assertEqual(e.maxlen, d.maxlen) + dd = d.pop() + ee = e.pop() + self.assertEqual(id(ee), id(e)) + self.assertEqual(e, d) + + d.x = d + e = pickle.loads(pickle.dumps(d, proto)) + self.assertEqual(id(e.x), id(e)) + + for d in DequeWithBadIter('abc'), DequeWithBadIter('abc', 2): + self.assertRaises(TypeError, pickle.dumps, d, proto) def test_weakref(self): d = deque('gallahad') diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index eda6fc1dac..c5bff7718d 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4828,11 +4828,8 @@ class PicklingTests(unittest.TestCase): return (args, kwargs) obj = C3() for proto in protocols: - if proto >= 4: + if proto >= 2: self._check_reduce(proto, obj, args, kwargs) - elif proto >= 2: - with self.assertRaises(ValueError): - obj.__reduce_ex__(proto) class C4: def __getnewargs_ex__(self): @@ -5151,10 +5148,6 @@ class PicklingTests(unittest.TestCase): kwargs = getattr(cls, 'KWARGS', {}) obj = cls(*cls.ARGS, **kwargs) proto = pickle_copier.proto - if 2 <= proto < 4 and hasattr(cls, '__getnewargs_ex__'): - with self.assertRaises(ValueError): - pickle_copier.dumps(obj, proto) - continue objcopy = pickle_copier.copy(obj) self._assert_is_copy(obj, objcopy) # For test classes that supports this, make sure we didn't go @@ -5213,12 +5206,14 @@ class SharedKeyTests(unittest.TestCase): a, b = A(), B() self.assertEqual(sys.getsizeof(vars(a)), sys.getsizeof(vars(b))) self.assertLess(sys.getsizeof(vars(a)), sys.getsizeof({})) - a.x, a.y, a.z, a.w = range(4) + # Initial hash table can contain at most 5 elements. + # Set 6 attributes to cause internal resizing. + a.x, a.y, a.z, a.w, a.v, a.u = range(6) self.assertNotEqual(sys.getsizeof(vars(a)), sys.getsizeof(vars(b))) a2 = A() self.assertEqual(sys.getsizeof(vars(a)), sys.getsizeof(vars(a2))) self.assertLess(sys.getsizeof(vars(a)), sys.getsizeof({})) - b.u, b.v, b.w, b.t = range(4) + b.u, b.v, b.w, b.t, b.s, b.r = range(6) self.assertLess(sys.getsizeof(vars(b)), sys.getsizeof({})) diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index 506d1abeb6..b84d644785 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -182,6 +182,7 @@ You can get the information from the list type: '__iadd__', '__imul__', '__init__', + '__init_subclass__', '__iter__', '__le__', '__len__', diff --git a/Lib/test/test_devpoll.py b/Lib/test/test_devpoll.py index 955618ae15..c133c811b1 100644 --- a/Lib/test/test_devpoll.py +++ b/Lib/test/test_devpoll.py @@ -5,9 +5,8 @@ import os import random import select -import sys import unittest -from test.support import TESTFN, run_unittest, cpython_only +from test.support import run_unittest, cpython_only if not hasattr(select, 'devpoll') : raise unittest.SkipTest('test works only on Solaris OS family') diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index b85d333dcb..832bb9c8e2 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1,10 +1,13 @@ -import unittest -from test import support - -import collections, random, string +import collections import collections.abc -import gc, weakref +import gc import pickle +import random +import string +import sys +import unittest +import weakref +from test import support class DictTest(unittest.TestCase): @@ -836,21 +839,127 @@ class DictTest(unittest.TestCase): pass self._tracked(MyDict()) + def make_shared_key_dict(self, n): + class C: + pass + + dicts = [] + for i in range(n): + a = C() + a.x, a.y, a.z = 1, 2, 3 + dicts.append(a.__dict__) + + return dicts + + @support.cpython_only + def test_splittable_setdefault(self): + """split table must be combined when setdefault() + breaks insertion order""" + a, b = self.make_shared_key_dict(2) + + a['a'] = 1 + size_a = sys.getsizeof(a) + a['b'] = 2 + b.setdefault('b', 2) + size_b = sys.getsizeof(b) + b['a'] = 1 + + self.assertGreater(size_b, size_a) + self.assertEqual(list(a), ['x', 'y', 'z', 'a', 'b']) + self.assertEqual(list(b), ['x', 'y', 'z', 'b', 'a']) + + @support.cpython_only + def test_splittable_del(self): + """split table must be combined when del d[k]""" + a, b = self.make_shared_key_dict(2) + + orig_size = sys.getsizeof(a) + + del a['y'] # split table is combined + with self.assertRaises(KeyError): + del a['y'] + + self.assertGreater(sys.getsizeof(a), orig_size) + self.assertEqual(list(a), ['x', 'z']) + self.assertEqual(list(b), ['x', 'y', 'z']) + + # Two dicts have different insertion order. + a['y'] = 42 + self.assertEqual(list(a), ['x', 'z', 'y']) + self.assertEqual(list(b), ['x', 'y', 'z']) + + @support.cpython_only + def test_splittable_pop(self): + """split table must be combined when d.pop(k)""" + a, b = self.make_shared_key_dict(2) + + orig_size = sys.getsizeof(a) + + a.pop('y') # split table is combined + with self.assertRaises(KeyError): + a.pop('y') + + self.assertGreater(sys.getsizeof(a), orig_size) + self.assertEqual(list(a), ['x', 'z']) + self.assertEqual(list(b), ['x', 'y', 'z']) + + # Two dicts have different insertion order. + a['y'] = 42 + self.assertEqual(list(a), ['x', 'z', 'y']) + self.assertEqual(list(b), ['x', 'y', 'z']) + + @support.cpython_only + def test_splittable_pop_pending(self): + """pop a pending key in a splitted table should not crash""" + a, b = self.make_shared_key_dict(2) + + a['a'] = 4 + with self.assertRaises(KeyError): + b.pop('a') + + @support.cpython_only + def test_splittable_popitem(self): + """split table must be combined when d.popitem()""" + a, b = self.make_shared_key_dict(2) + + orig_size = sys.getsizeof(a) + + item = a.popitem() # split table is combined + self.assertEqual(item, ('z', 3)) + with self.assertRaises(KeyError): + del a['z'] + + self.assertGreater(sys.getsizeof(a), orig_size) + self.assertEqual(list(a), ['x', 'y']) + self.assertEqual(list(b), ['x', 'y', 'z']) + @support.cpython_only def test_splittable_setattr_after_pop(self): - """setattr must not convert combined table into split table""" + """setattr() must not convert combined table into split table.""" # Issue 28147 import _testcapi class C: pass a = C() + + a.a = 1 + self.assertTrue(_testcapi.dict_hassplittable(a.__dict__)) + + # dict.pop() convert it to combined table + a.__dict__.pop('a') + self.assertFalse(_testcapi.dict_hassplittable(a.__dict__)) + + # But C should not convert a.__dict__ to split table again. + a.a = 1 + self.assertFalse(_testcapi.dict_hassplittable(a.__dict__)) + + # Same for popitem() + a = C() a.a = 2 self.assertTrue(_testcapi.dict_hassplittable(a.__dict__)) - # dict.popitem() convert it to combined table a.__dict__.popitem() self.assertFalse(_testcapi.dict_hassplittable(a.__dict__)) - # But C should not convert a.__dict__ to split table again. a.a = 3 self.assertFalse(_testcapi.dict_hassplittable(a.__dict__)) @@ -976,6 +1085,37 @@ class DictTest(unittest.TestCase): support.check_free_after_iterating(self, lambda d: iter(d.values()), dict) support.check_free_after_iterating(self, lambda d: iter(d.items()), dict) + +class CAPITest(unittest.TestCase): + + # Test _PyDict_GetItem_KnownHash() + @support.cpython_only + def test_getitem_knownhash(self): + from _testcapi import dict_getitem_knownhash + + d = {'x': 1, 'y': 2, 'z': 3} + self.assertEqual(dict_getitem_knownhash(d, 'x', hash('x')), 1) + self.assertEqual(dict_getitem_knownhash(d, 'y', hash('y')), 2) + self.assertEqual(dict_getitem_knownhash(d, 'z', hash('z')), 3) + + # not a dict + self.assertRaises(SystemError, dict_getitem_knownhash, [], 1, hash(1)) + # key does not exist + self.assertRaises(KeyError, dict_getitem_knownhash, {}, 1, hash(1)) + + class Exc(Exception): pass + class BadEq: + def __eq__(self, other): + raise Exc + def __hash__(self): + return 7 + + k1, k2 = BadEq(), BadEq() + d = {k1: 1} + self.assertEqual(dict_getitem_knownhash(d, k1, hash(k1)), 1) + self.assertRaises(Exc, dict_getitem_knownhash, d, k2, hash(k2)) + + from test import mapping_tests class GeneralMappingTests(mapping_tests.BasicTestMappingProtocol): @@ -987,5 +1127,6 @@ class Dict(dict): class SubclassMappingTests(mapping_tests.BasicTestMappingProtocol): type2test = Dict + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_dict_version.py b/Lib/test/test_dict_version.py new file mode 100644 index 0000000000..5671f9fc7c --- /dev/null +++ b/Lib/test/test_dict_version.py @@ -0,0 +1,186 @@ +""" +Test implementation of the PEP 509: dictionary versionning. +""" +import unittest +from test import support + +# PEP 509 is implemented in CPython but other Python implementations +# don't require to implement it +_testcapi = support.import_module('_testcapi') + + +class DictVersionTests(unittest.TestCase): + type2test = dict + + def setUp(self): + self.seen_versions = set() + self.dict = None + + def check_version_unique(self, mydict): + version = _testcapi.dict_get_version(mydict) + self.assertNotIn(version, self.seen_versions) + self.seen_versions.add(version) + + def check_version_changed(self, mydict, method, *args, **kw): + result = method(*args, **kw) + self.check_version_unique(mydict) + return result + + def check_version_dont_change(self, mydict, method, *args, **kw): + version1 = _testcapi.dict_get_version(mydict) + self.seen_versions.add(version1) + + result = method(*args, **kw) + + version2 = _testcapi.dict_get_version(mydict) + self.assertEqual(version2, version1, "version changed") + + return result + + def new_dict(self, *args, **kw): + d = self.type2test(*args, **kw) + self.check_version_unique(d) + return d + + def test_constructor(self): + # new empty dictionaries must all have an unique version + empty1 = self.new_dict() + empty2 = self.new_dict() + empty3 = self.new_dict() + + # non-empty dictionaries must also have an unique version + nonempty1 = self.new_dict(x='x') + nonempty2 = self.new_dict(x='x', y='y') + + def test_copy(self): + d = self.new_dict(a=1, b=2) + + d2 = self.check_version_dont_change(d, d.copy) + + # dict.copy() must create a dictionary with a new unique version + self.check_version_unique(d2) + + def test_setitem(self): + d = self.new_dict() + + # creating new keys must change the version + self.check_version_changed(d, d.__setitem__, 'x', 'x') + self.check_version_changed(d, d.__setitem__, 'y', 'y') + + # changing values must change the version + self.check_version_changed(d, d.__setitem__, 'x', 1) + self.check_version_changed(d, d.__setitem__, 'y', 2) + + def test_setitem_same_value(self): + value = object() + d = self.new_dict() + + # setting a key must change the version + self.check_version_changed(d, d.__setitem__, 'key', value) + + # setting a key to the same value with dict.__setitem__ + # must change the version + self.check_version_changed(d, d.__setitem__, 'key', value) + + # setting a key to the same value with dict.update + # must change the version + self.check_version_changed(d, d.update, key=value) + + d2 = self.new_dict(key=value) + self.check_version_changed(d, d.update, d2) + + def test_setitem_equal(self): + class AlwaysEqual: + def __eq__(self, other): + return True + + value1 = AlwaysEqual() + value2 = AlwaysEqual() + self.assertTrue(value1 == value2) + self.assertFalse(value1 != value2) + + d = self.new_dict() + self.check_version_changed(d, d.__setitem__, 'key', value1) + + # setting a key to a value equal to the current value + # with dict.__setitem__() must change the version + self.check_version_changed(d, d.__setitem__, 'key', value2) + + # setting a key to a value equal to the current value + # with dict.update() must change the version + self.check_version_changed(d, d.update, key=value1) + + d2 = self.new_dict(key=value2) + self.check_version_changed(d, d.update, d2) + + def test_setdefault(self): + d = self.new_dict() + + # setting a key with dict.setdefault() must change the version + self.check_version_changed(d, d.setdefault, 'key', 'value1') + + # don't change the version if the key already exists + self.check_version_dont_change(d, d.setdefault, 'key', 'value2') + + def test_delitem(self): + d = self.new_dict(key='value') + + # deleting a key with dict.__delitem__() must change the version + self.check_version_changed(d, d.__delitem__, 'key') + + # don't change the version if the key doesn't exist + self.check_version_dont_change(d, self.assertRaises, KeyError, + d.__delitem__, 'key') + + def test_pop(self): + d = self.new_dict(key='value') + + # pop() must change the version if the key exists + self.check_version_changed(d, d.pop, 'key') + + # pop() must not change the version if the key does not exist + self.check_version_dont_change(d, self.assertRaises, KeyError, + d.pop, 'key') + + def test_popitem(self): + d = self.new_dict(key='value') + + # popitem() must change the version if the dict is not empty + self.check_version_changed(d, d.popitem) + + # popitem() must not change the version if the dict is empty + self.check_version_dont_change(d, self.assertRaises, KeyError, + d.popitem) + + def test_update(self): + d = self.new_dict(key='value') + + # update() calling with no argument must not change the version + self.check_version_dont_change(d, d.update) + + # update() must change the version + self.check_version_changed(d, d.update, key='new value') + + d2 = self.new_dict(key='value 3') + self.check_version_changed(d, d.update, d2) + + def test_clear(self): + d = self.new_dict(key='value') + + # clear() must change the version if the dict is not empty + self.check_version_changed(d, d.clear) + + # clear() must not change the version if the dict is empty + self.check_version_dont_change(d, d.clear) + + +class Dict(dict): + pass + + +class DictSubtypeVersionTests(DictVersionTests): + type2test = Dict + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_dictcomps.py b/Lib/test/test_dictcomps.py index 3c8b95cf32..0873071883 100644 --- a/Lib/test/test_dictcomps.py +++ b/Lib/test/test_dictcomps.py @@ -1,7 +1,5 @@ import unittest -from test import support - # For scope testing. g = "Global variable" diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index 0c996526d8..49a9e9c007 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -1,3 +1,4 @@ +import collections import copy import pickle import unittest @@ -245,6 +246,27 @@ class DictSetTest(unittest.TestCase): self.assertRaises((TypeError, pickle.PicklingError), pickle.dumps, d.items(), proto) + def test_abc_registry(self): + d = dict(a=1) + + self.assertIsInstance(d.keys(), collections.KeysView) + self.assertIsInstance(d.keys(), collections.MappingView) + self.assertIsInstance(d.keys(), collections.Set) + self.assertIsInstance(d.keys(), collections.Sized) + self.assertIsInstance(d.keys(), collections.Iterable) + self.assertIsInstance(d.keys(), collections.Container) + + self.assertIsInstance(d.values(), collections.ValuesView) + self.assertIsInstance(d.values(), collections.MappingView) + self.assertIsInstance(d.values(), collections.Sized) + + self.assertIsInstance(d.items(), collections.ItemsView) + self.assertIsInstance(d.items(), collections.MappingView) + self.assertIsInstance(d.items(), collections.Set) + self.assertIsInstance(d.items(), collections.Sized) + self.assertIsInstance(d.items(), collections.Iterable) + self.assertIsInstance(d.items(), collections.Container) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index ab9debf8e2..156b523c38 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -122,17 +122,17 @@ patch914575_nonascii_to1 = """ """ patch914575_from2 = """ -\t\tLine 1: preceeded by from:[tt] to:[ssss] - \t\tLine 2: preceeded by from:[sstt] to:[sssst] - \t \tLine 3: preceeded by from:[sstst] to:[ssssss] +\t\tLine 1: preceded by from:[tt] to:[ssss] + \t\tLine 2: preceded by from:[sstt] to:[sssst] + \t \tLine 3: preceded by from:[sstst] to:[ssssss] Line 4: \thas from:[sst] to:[sss] after : Line 5: has from:[t] to:[ss] at end\t """ patch914575_to2 = """ - Line 1: preceeded by from:[tt] to:[ssss] - \tLine 2: preceeded by from:[sstt] to:[sssst] - Line 3: preceeded by from:[sstst] to:[ssssss] + Line 1: preceded by from:[tt] to:[ssss] + \tLine 2: preceded by from:[sstt] to:[sssst] + Line 3: preceded by from:[sstst] to:[ssssss] Line 4: has from:[sst] to:[sss] after : Line 5: has from:[t] to:[ss] at end """ diff --git a/Lib/test/test_difflib_expect.html b/Lib/test/test_difflib_expect.html index ea7a24ef4b..3e6a7b7a99 100644 --- a/Lib/test/test_difflib_expect.html +++ b/Lib/test/test_difflib_expect.html @@ -387,9 +387,9 @@ <tbody> <tr><td class="diff_next" id="difflib_chg_to9__0"><a href="#difflib_chg_to9__0">f</a></td><td class="diff_header" id="from9_1">1</td><td nowrap="nowrap"></td><td class="diff_next"><a href="#difflib_chg_to9__0">f</a></td><td class="diff_header" id="to9_1">1</td><td nowrap="nowrap"></td></tr> - <tr><td class="diff_next"><a href="#difflib_chg_to9__top">t</a></td><td class="diff_header" id="from9_2">2</td><td nowrap="nowrap"><span class="diff_chg"> </span>Line 1: preceeded by from:[tt] to:[ssss]</td><td class="diff_next"><a href="#difflib_chg_to9__top">t</a></td><td class="diff_header" id="to9_2">2</td><td nowrap="nowrap"><span class="diff_chg"> </span>Line 1: preceeded by from:[tt] to:[ssss]</td></tr> - <tr><td class="diff_next"></td><td class="diff_header" id="from9_3">3</td><td nowrap="nowrap"> <span class="diff_chg"> </span> Line 2: preceeded by from:[sstt] to:[sssst]</td><td class="diff_next"></td><td class="diff_header" id="to9_3">3</td><td nowrap="nowrap"> <span class="diff_chg"> </span> Line 2: preceeded by from:[sstt] to:[sssst]</td></tr> - <tr><td class="diff_next"></td><td class="diff_header" id="from9_4">4</td><td nowrap="nowrap"> <span class="diff_chg"> </span>Line 3: preceeded by from:[sstst] to:[ssssss]</td><td class="diff_next"></td><td class="diff_header" id="to9_4">4</td><td nowrap="nowrap"> <span class="diff_chg"> </span>Line 3: preceeded by from:[sstst] to:[ssssss]</td></tr> + <tr><td class="diff_next"><a href="#difflib_chg_to9__top">t</a></td><td class="diff_header" id="from9_2">2</td><td nowrap="nowrap"><span class="diff_chg"> </span>Line 1: preceded by from:[tt] to:[ssss]</td><td class="diff_next"><a href="#difflib_chg_to9__top">t</a></td><td class="diff_header" id="to9_2">2</td><td nowrap="nowrap"><span class="diff_chg"> </span>Line 1: preceded by from:[tt] to:[ssss]</td></tr> + <tr><td class="diff_next"></td><td class="diff_header" id="from9_3">3</td><td nowrap="nowrap"> <span class="diff_chg"> </span> Line 2: preceded by from:[sstt] to:[sssst]</td><td class="diff_next"></td><td class="diff_header" id="to9_3">3</td><td nowrap="nowrap"> <span class="diff_chg"> </span> Line 2: preceded by from:[sstt] to:[sssst]</td></tr> + <tr><td class="diff_next"></td><td class="diff_header" id="from9_4">4</td><td nowrap="nowrap"> <span class="diff_chg"> </span>Line 3: preceded by from:[sstst] to:[ssssss]</td><td class="diff_next"></td><td class="diff_header" id="to9_4">4</td><td nowrap="nowrap"> <span class="diff_chg"> </span>Line 3: preceded by from:[sstst] to:[ssssss]</td></tr> <tr><td class="diff_next"></td><td class="diff_header" id="from9_5">5</td><td nowrap="nowrap">Line 4: <span class="diff_chg"> </span>has from:[sst] to:[sss] after :</td><td class="diff_next"></td><td class="diff_header" id="to9_5">5</td><td nowrap="nowrap">Line 4: <span class="diff_chg"> </span>has from:[sst] to:[sss] after :</td></tr> <tr><td class="diff_next"></td><td class="diff_header" id="from9_6">6</td><td nowrap="nowrap">Line 5: has from:[t] to:[ss] at end<span class="diff_sub"> </span></td><td class="diff_next"></td><td class="diff_header" id="to9_6">6</td><td nowrap="nowrap">Line 5: has from:[t] to:[ss] at end</td></tr> </tbody> @@ -403,9 +403,9 @@ <tbody> <tr><td class="diff_next" id="difflib_chg_to10__0"><a href="#difflib_chg_to10__0">f</a></td><td class="diff_header" id="from10_1">1</td><td nowrap="nowrap"></td><td class="diff_next"><a href="#difflib_chg_to10__0">f</a></td><td class="diff_header" id="to10_1">1</td><td nowrap="nowrap"></td></tr> - <tr><td class="diff_next"><a href="#difflib_chg_to10__top">t</a></td><td class="diff_header" id="from10_2">2</td><td nowrap="nowrap"><span class="diff_chg"> </span>Line 1: preceeded by from:[tt] to:[ssss]</td><td class="diff_next"><a href="#difflib_chg_to10__top">t</a></td><td class="diff_header" id="to10_2">2</td><td nowrap="nowrap"><span class="diff_chg"> </span>Line 1: preceeded by from:[tt] to:[ssss]</td></tr> - <tr><td class="diff_next"></td><td class="diff_header" id="from10_3">3</td><td nowrap="nowrap"> <span class="diff_chg"> </span> Line 2: preceeded by from:[sstt] to:[sssst]</td><td class="diff_next"></td><td class="diff_header" id="to10_3">3</td><td nowrap="nowrap"> <span class="diff_chg"> </span> Line 2: preceeded by from:[sstt] to:[sssst]</td></tr> - <tr><td class="diff_next"></td><td class="diff_header" id="from10_4">4</td><td nowrap="nowrap"> <span class="diff_chg"> </span>Line 3: preceeded by from:[sstst] to:[ssssss]</td><td class="diff_next"></td><td class="diff_header" id="to10_4">4</td><td nowrap="nowrap"> <span class="diff_chg"> </span>Line 3: preceeded by from:[sstst] to:[ssssss]</td></tr> + <tr><td class="diff_next"><a href="#difflib_chg_to10__top">t</a></td><td class="diff_header" id="from10_2">2</td><td nowrap="nowrap"><span class="diff_chg"> </span>Line 1: preceded by from:[tt] to:[ssss]</td><td class="diff_next"><a href="#difflib_chg_to10__top">t</a></td><td class="diff_header" id="to10_2">2</td><td nowrap="nowrap"><span class="diff_chg"> </span>Line 1: preceded by from:[tt] to:[ssss]</td></tr> + <tr><td class="diff_next"></td><td class="diff_header" id="from10_3">3</td><td nowrap="nowrap"> <span class="diff_chg"> </span> Line 2: preceded by from:[sstt] to:[sssst]</td><td class="diff_next"></td><td class="diff_header" id="to10_3">3</td><td nowrap="nowrap"> <span class="diff_chg"> </span> Line 2: preceded by from:[sstt] to:[sssst]</td></tr> + <tr><td class="diff_next"></td><td class="diff_header" id="from10_4">4</td><td nowrap="nowrap"> <span class="diff_chg"> </span>Line 3: preceded by from:[sstst] to:[ssssss]</td><td class="diff_next"></td><td class="diff_header" id="to10_4">4</td><td nowrap="nowrap"> <span class="diff_chg"> </span>Line 3: preceded by from:[sstst] to:[ssssss]</td></tr> <tr><td class="diff_next"></td><td class="diff_header" id="from10_5">5</td><td nowrap="nowrap">Line 4: <span class="diff_chg"> </span>has from:[sst] to:[sss] after :</td><td class="diff_next"></td><td class="diff_header" id="to10_5">5</td><td nowrap="nowrap">Line 4: <span class="diff_chg"> </span>has from:[sst] to:[sss] after :</td></tr> <tr><td class="diff_next"></td><td class="diff_header" id="from10_6">6</td><td nowrap="nowrap">Line 5: has from:[t] to:[ss] at end<span class="diff_sub"> </span></td><td class="diff_next"></td><td class="diff_header" id="to10_6">6</td><td nowrap="nowrap">Line 5: has from:[t] to:[ss] at end</td></tr> </tbody> diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 0fd1348027..b9b5ac2667 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -40,41 +40,41 @@ class _C: dis_c_instance_method = """\ %3d 0 LOAD_FAST 1 (x) - 3 LOAD_CONST 1 (1) - 6 COMPARE_OP 2 (==) - 9 LOAD_FAST 0 (self) - 12 STORE_ATTR 0 (x) - 15 LOAD_CONST 0 (None) - 18 RETURN_VALUE + 2 LOAD_CONST 1 (1) + 4 COMPARE_OP 2 (==) + 6 LOAD_FAST 0 (self) + 8 STORE_ATTR 0 (x) + 10 LOAD_CONST 0 (None) + 12 RETURN_VALUE """ % (_C.__init__.__code__.co_firstlineno + 1,) dis_c_instance_method_bytes = """\ 0 LOAD_FAST 1 (1) - 3 LOAD_CONST 1 (1) - 6 COMPARE_OP 2 (==) - 9 LOAD_FAST 0 (0) - 12 STORE_ATTR 0 (0) - 15 LOAD_CONST 0 (0) - 18 RETURN_VALUE + 2 LOAD_CONST 1 (1) + 4 COMPARE_OP 2 (==) + 6 LOAD_FAST 0 (0) + 8 STORE_ATTR 0 (0) + 10 LOAD_CONST 0 (0) + 12 RETURN_VALUE """ dis_c_class_method = """\ %3d 0 LOAD_FAST 1 (x) - 3 LOAD_CONST 1 (1) - 6 COMPARE_OP 2 (==) - 9 LOAD_FAST 0 (cls) - 12 STORE_ATTR 0 (x) - 15 LOAD_CONST 0 (None) - 18 RETURN_VALUE + 2 LOAD_CONST 1 (1) + 4 COMPARE_OP 2 (==) + 6 LOAD_FAST 0 (cls) + 8 STORE_ATTR 0 (x) + 10 LOAD_CONST 0 (None) + 12 RETURN_VALUE """ % (_C.cm.__code__.co_firstlineno + 2,) dis_c_static_method = """\ %3d 0 LOAD_FAST 0 (x) - 3 LOAD_CONST 1 (1) - 6 COMPARE_OP 2 (==) - 9 STORE_FAST 0 (x) - 12 LOAD_CONST 0 (None) - 15 RETURN_VALUE + 2 LOAD_CONST 1 (1) + 4 COMPARE_OP 2 (==) + 6 STORE_FAST 0 (x) + 8 LOAD_CONST 0 (None) + 10 RETURN_VALUE """ % (_C.sm.__code__.co_firstlineno + 2,) # Class disassembling info has an extra newline at end. @@ -95,23 +95,23 @@ def _f(a): dis_f = """\ %3d 0 LOAD_GLOBAL 0 (print) - 3 LOAD_FAST 0 (a) - 6 CALL_FUNCTION 1 (1 positional, 0 keyword pair) - 9 POP_TOP + 2 LOAD_FAST 0 (a) + 4 CALL_FUNCTION 1 + 6 POP_TOP -%3d 10 LOAD_CONST 1 (1) - 13 RETURN_VALUE +%3d 8 LOAD_CONST 1 (1) + 10 RETURN_VALUE """ % (_f.__code__.co_firstlineno + 1, _f.__code__.co_firstlineno + 2) dis_f_co_code = """\ 0 LOAD_GLOBAL 0 (0) - 3 LOAD_FAST 0 (0) - 6 CALL_FUNCTION 1 (1 positional, 0 keyword pair) - 9 POP_TOP - 10 LOAD_CONST 1 (1) - 13 RETURN_VALUE + 2 LOAD_FAST 0 (0) + 4 CALL_FUNCTION 1 + 6 POP_TOP + 8 LOAD_CONST 1 (1) + 10 RETURN_VALUE """ @@ -121,20 +121,20 @@ def bug708901(): pass dis_bug708901 = """\ -%3d 0 SETUP_LOOP 23 (to 26) - 3 LOAD_GLOBAL 0 (range) - 6 LOAD_CONST 1 (1) - -%3d 9 LOAD_CONST 2 (10) - 12 CALL_FUNCTION 2 (2 positional, 0 keyword pair) - 15 GET_ITER - >> 16 FOR_ITER 6 (to 25) - 19 STORE_FAST 0 (res) - -%3d 22 JUMP_ABSOLUTE 16 - >> 25 POP_BLOCK - >> 26 LOAD_CONST 0 (None) - 29 RETURN_VALUE +%3d 0 SETUP_LOOP 18 (to 20) + 2 LOAD_GLOBAL 0 (range) + 4 LOAD_CONST 1 (1) + +%3d 6 LOAD_CONST 2 (10) + 8 CALL_FUNCTION 2 + 10 GET_ITER + >> 12 FOR_ITER 4 (to 18) + 14 STORE_FAST 0 (res) + +%3d 16 JUMP_ABSOLUTE 12 + >> 18 POP_BLOCK + >> 20 LOAD_CONST 0 (None) + 22 RETURN_VALUE """ % (bug708901.__code__.co_firstlineno + 1, bug708901.__code__.co_firstlineno + 2, bug708901.__code__.co_firstlineno + 3) @@ -147,22 +147,22 @@ def bug1333982(x=[]): dis_bug1333982 = """\ %3d 0 LOAD_CONST 1 (0) - 3 POP_JUMP_IF_TRUE 35 - 6 LOAD_GLOBAL 0 (AssertionError) - 9 LOAD_CONST 2 (<code object <listcomp> at 0x..., file "%s", line %d>) - 12 LOAD_CONST 3 ('bug1333982.<locals>.<listcomp>') - 15 MAKE_FUNCTION 0 - 18 LOAD_FAST 0 (x) - 21 GET_ITER - 22 CALL_FUNCTION 1 (1 positional, 0 keyword pair) - -%3d 25 LOAD_CONST 4 (1) - 28 BINARY_ADD - 29 CALL_FUNCTION 1 (1 positional, 0 keyword pair) - 32 RAISE_VARARGS 1 - -%3d >> 35 LOAD_CONST 0 (None) - 38 RETURN_VALUE + 2 POP_JUMP_IF_TRUE 26 + 4 LOAD_GLOBAL 0 (AssertionError) + 6 LOAD_CONST 2 (<code object <listcomp> at 0x..., file "%s", line %d>) + 8 LOAD_CONST 3 ('bug1333982.<locals>.<listcomp>') + 10 MAKE_FUNCTION 0 + 12 LOAD_FAST 0 (x) + 14 GET_ITER + 16 CALL_FUNCTION 1 + +%3d 18 LOAD_CONST 4 (1) + 20 BINARY_ADD + 22 CALL_FUNCTION 1 + 24 RAISE_VARARGS 1 + +%3d >> 26 LOAD_CONST 0 (None) + 28 RETURN_VALUE """ % (bug1333982.__code__.co_firstlineno + 1, __file__, bug1333982.__code__.co_firstlineno + 1, @@ -171,19 +171,19 @@ dis_bug1333982 = """\ _BIG_LINENO_FORMAT = """\ %3d 0 LOAD_GLOBAL 0 (spam) - 3 POP_TOP + 2 POP_TOP 4 LOAD_CONST 0 (None) - 7 RETURN_VALUE + 6 RETURN_VALUE """ dis_module_expected_results = """\ Disassembly of f: 4 0 LOAD_CONST 0 (None) - 3 RETURN_VALUE + 2 RETURN_VALUE Disassembly of g: 5 0 LOAD_CONST 0 (None) - 3 RETURN_VALUE + 2 RETURN_VALUE """ @@ -191,20 +191,52 @@ expr_str = "x + 1" dis_expr_str = """\ 1 0 LOAD_NAME 0 (x) - 3 LOAD_CONST 0 (1) - 6 BINARY_ADD - 7 RETURN_VALUE + 2 LOAD_CONST 0 (1) + 4 BINARY_ADD + 6 RETURN_VALUE """ simple_stmt_str = "x = x + 1" dis_simple_stmt_str = """\ 1 0 LOAD_NAME 0 (x) - 3 LOAD_CONST 0 (1) - 6 BINARY_ADD - 7 STORE_NAME 0 (x) - 10 LOAD_CONST 1 (None) - 13 RETURN_VALUE + 2 LOAD_CONST 0 (1) + 4 BINARY_ADD + 6 STORE_NAME 0 (x) + 8 LOAD_CONST 1 (None) + 10 RETURN_VALUE +""" + +annot_stmt_str = """\ + +x: int = 1 +y: fun(1) +lst[fun(0)]: int = 1 +""" +# leading newline is for a reason (tests lineno) + +dis_annot_stmt_str = """\ + 2 0 SETUP_ANNOTATIONS + 2 LOAD_CONST 0 (1) + 4 STORE_NAME 0 (x) + 6 LOAD_NAME 1 (int) + 8 STORE_ANNOTATION 0 (x) + + 3 10 LOAD_NAME 2 (fun) + 12 LOAD_CONST 0 (1) + 14 CALL_FUNCTION 1 + 16 STORE_ANNOTATION 3 (y) + + 4 18 LOAD_CONST 0 (1) + 20 LOAD_NAME 4 (lst) + 22 LOAD_NAME 2 (fun) + 24 LOAD_CONST 1 (0) + 26 CALL_FUNCTION 1 + 28 STORE_SUBSCR + 30 LOAD_NAME 1 (int) + 32 POP_TOP + 34 LOAD_CONST 2 (None) + 36 RETURN_VALUE """ compound_stmt_str = """\ @@ -215,60 +247,81 @@ while 1: dis_compound_stmt_str = """\ 1 0 LOAD_CONST 0 (0) - 3 STORE_NAME 0 (x) - - 2 6 SETUP_LOOP 14 (to 23) - - 3 >> 9 LOAD_NAME 0 (x) - 12 LOAD_CONST 1 (1) - 15 INPLACE_ADD - 16 STORE_NAME 0 (x) - 19 JUMP_ABSOLUTE 9 - 22 POP_BLOCK - >> 23 LOAD_CONST 2 (None) - 26 RETURN_VALUE + 2 STORE_NAME 0 (x) + + 2 4 SETUP_LOOP 12 (to 18) + + 3 >> 6 LOAD_NAME 0 (x) + 8 LOAD_CONST 1 (1) + 10 INPLACE_ADD + 12 STORE_NAME 0 (x) + 14 JUMP_ABSOLUTE 6 + 16 POP_BLOCK + >> 18 LOAD_CONST 2 (None) + 20 RETURN_VALUE """ dis_traceback = """\ -%3d 0 SETUP_EXCEPT 12 (to 15) +%3d 0 SETUP_EXCEPT 12 (to 14) -%3d 3 LOAD_CONST 1 (1) - 6 LOAD_CONST 2 (0) - --> 9 BINARY_TRUE_DIVIDE - 10 POP_TOP - 11 POP_BLOCK - 12 JUMP_FORWARD 46 (to 61) +%3d 2 LOAD_CONST 1 (1) + 4 LOAD_CONST 2 (0) + --> 6 BINARY_TRUE_DIVIDE + 8 POP_TOP + 10 POP_BLOCK + 12 JUMP_FORWARD 40 (to 54) -%3d >> 15 DUP_TOP +%3d >> 14 DUP_TOP 16 LOAD_GLOBAL 0 (Exception) - 19 COMPARE_OP 10 (exception match) - 22 POP_JUMP_IF_FALSE 60 - 25 POP_TOP - 26 STORE_FAST 0 (e) - 29 POP_TOP - 30 SETUP_FINALLY 14 (to 47) - -%3d 33 LOAD_FAST 0 (e) - 36 LOAD_ATTR 1 (__traceback__) - 39 STORE_FAST 1 (tb) - 42 POP_BLOCK - 43 POP_EXCEPT - 44 LOAD_CONST 0 (None) - >> 47 LOAD_CONST 0 (None) - 50 STORE_FAST 0 (e) - 53 DELETE_FAST 0 (e) - 56 END_FINALLY - 57 JUMP_FORWARD 1 (to 61) - >> 60 END_FINALLY - -%3d >> 61 LOAD_FAST 1 (tb) - 64 RETURN_VALUE + 18 COMPARE_OP 10 (exception match) + 20 POP_JUMP_IF_FALSE 52 + 22 POP_TOP + 24 STORE_FAST 0 (e) + 26 POP_TOP + 28 SETUP_FINALLY 12 (to 42) + +%3d 30 LOAD_FAST 0 (e) + 32 LOAD_ATTR 1 (__traceback__) + 34 STORE_FAST 1 (tb) + 36 POP_BLOCK + 38 POP_EXCEPT + 40 LOAD_CONST 0 (None) + >> 42 LOAD_CONST 0 (None) + 44 STORE_FAST 0 (e) + 46 DELETE_FAST 0 (e) + 48 END_FINALLY + 50 JUMP_FORWARD 2 (to 54) + >> 52 END_FINALLY + +%3d >> 54 LOAD_FAST 1 (tb) + 56 RETURN_VALUE """ % (TRACEBACK_CODE.co_firstlineno + 1, TRACEBACK_CODE.co_firstlineno + 2, TRACEBACK_CODE.co_firstlineno + 3, TRACEBACK_CODE.co_firstlineno + 4, TRACEBACK_CODE.co_firstlineno + 5) +def _fstring(a, b, c, d): + return f'{a} {b:4} {c!r} {d!r:4}' + +dis_fstring = """\ +%3d 0 LOAD_FAST 0 (a) + 2 FORMAT_VALUE 0 + 4 LOAD_CONST 1 (' ') + 6 LOAD_FAST 1 (b) + 8 LOAD_CONST 2 ('4') + 10 FORMAT_VALUE 4 (with format) + 12 LOAD_CONST 1 (' ') + 14 LOAD_FAST 2 (c) + 16 FORMAT_VALUE 2 (repr) + 18 LOAD_CONST 1 (' ') + 20 LOAD_FAST 3 (d) + 22 LOAD_CONST 2 ('4') + 24 FORMAT_VALUE 6 (repr, with format) + 26 BUILD_STRING 7 + 28 RETURN_VALUE +""" % (_fstring.__code__.co_firstlineno + 1,) + def _g(x): yield x @@ -345,6 +398,7 @@ class DisTests(unittest.TestCase): def test_disassemble_str(self): self.do_disassembly_test(expr_str, dis_expr_str) self.do_disassembly_test(simple_stmt_str, dis_simple_stmt_str) + self.do_disassembly_test(annot_stmt_str, dis_annot_stmt_str) self.do_disassembly_test(compound_stmt_str, dis_compound_stmt_str) def test_disassemble_bytes(self): @@ -371,6 +425,9 @@ class DisTests(unittest.TestCase): gen_disas = self.get_disassembly(_g(1)) # Disassemble generator itself self.assertEqual(gen_disas, gen_func_disas) + def test_disassemble_fstring(self): + self.do_disassembly_test(_fstring, dis_fstring) + def test_dis_none(self): try: del sys.last_traceback @@ -541,7 +598,7 @@ Argument count: 0 Kw-only arguments: 0 Number of locals: 2 Stack size: 17 -Flags: OPTIMIZED, NEWLOCALS, GENERATOR, NOFREE, COROUTINE +Flags: OPTIMIZED, NEWLOCALS, NOFREE, COROUTINE Constants: 0: None 1: 1""" @@ -649,171 +706,169 @@ expected_jumpy_line = 1 Instruction = dis.Instruction expected_opinfo_outer = [ - Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=3, argrepr='3', offset=0, starts_line=2, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=3, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='a', argrepr='a', offset=6, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='b', argrepr='b', offset=9, starts_line=None, is_jump_target=False), - Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=12, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=15, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=18, starts_line=None, is_jump_target=False), - Instruction(opname='MAKE_CLOSURE', opcode=134, arg=2, argval=2, argrepr='', offset=21, starts_line=None, is_jump_target=False), - Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=24, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=27, starts_line=7, is_jump_target=False), - Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=30, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='b', argrepr='b', offset=33, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval='', argrepr="''", offset=36, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval=1, argrepr='1', offset=39, starts_line=None, is_jump_target=False), - Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=42, starts_line=None, is_jump_target=False), - Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=45, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval='Hello world!', argrepr="'Hello world!'", offset=48, starts_line=None, is_jump_target=False), - Instruction(opname='CALL_FUNCTION', opcode=131, arg=7, argval=7, argrepr='7 positional, 0 keyword pair', offset=51, starts_line=None, is_jump_target=False), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=55, starts_line=8, is_jump_target=False), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval=(3, 4), argrepr='(3, 4)', offset=0, starts_line=2, is_jump_target=False), + Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False), + Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=6, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=8, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=10, starts_line=None, is_jump_target=False), + Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='', offset=12, starts_line=None, is_jump_target=False), + Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=14, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=16, starts_line=7, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=18, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='b', argrepr='b', offset=20, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval='', argrepr="''", offset=22, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval=1, argrepr='1', offset=24, starts_line=None, is_jump_target=False), + Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=26, starts_line=None, is_jump_target=False), + Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=28, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval='Hello world!', argrepr="'Hello world!'", offset=30, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=7, argval=7, argrepr='', offset=32, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=34, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=36, starts_line=8, is_jump_target=False), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=38, starts_line=None, is_jump_target=False), ] expected_opinfo_f = [ - Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=5, argrepr='5', offset=0, starts_line=3, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=6, argrepr='6', offset=3, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CLOSURE', opcode=135, arg=2, argval='a', argrepr='a', offset=6, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='b', argrepr='b', offset=9, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='c', argrepr='c', offset=12, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='d', argrepr='d', offset=15, starts_line=None, is_jump_target=False), - Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=18, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=21, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=24, starts_line=None, is_jump_target=False), - Instruction(opname='MAKE_CLOSURE', opcode=134, arg=2, argval=2, argrepr='', offset=27, starts_line=None, is_jump_target=False), - Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=30, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=33, starts_line=5, is_jump_target=False), - Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='a', argrepr='a', offset=36, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='b', argrepr='b', offset=39, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='c', argrepr='c', offset=42, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='d', argrepr='d', offset=45, starts_line=None, is_jump_target=False), - Instruction(opname='CALL_FUNCTION', opcode=131, arg=4, argval=4, argrepr='4 positional, 0 keyword pair', offset=48, starts_line=None, is_jump_target=False), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=51, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=52, starts_line=6, is_jump_target=False), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=55, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=(5, 6), argrepr='(5, 6)', offset=0, starts_line=3, is_jump_target=False), + Instruction(opname='LOAD_CLOSURE', opcode=135, arg=2, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False), + Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=10, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=12, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=14, starts_line=None, is_jump_target=False), + Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='', offset=16, starts_line=None, is_jump_target=False), + Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=18, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=20, starts_line=5, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='a', argrepr='a', offset=22, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='b', argrepr='b', offset=24, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='c', argrepr='c', offset=26, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='d', argrepr='d', offset=28, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=4, argval=4, argrepr='', offset=30, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=32, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=34, starts_line=6, is_jump_target=False), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=36, starts_line=None, is_jump_target=False), ] expected_opinfo_inner = [ Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=0, starts_line=4, is_jump_target=False), - Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=3, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='b', argrepr='b', offset=6, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='c', argrepr='c', offset=9, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='d', argrepr='d', offset=12, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=15, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=18, starts_line=None, is_jump_target=False), - Instruction(opname='CALL_FUNCTION', opcode=131, arg=6, argval=6, argrepr='6 positional, 0 keyword pair', offset=21, starts_line=None, is_jump_target=False), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=24, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=25, starts_line=None, is_jump_target=False), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=28, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=10, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=12, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=6, argval=6, argrepr='', offset=14, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=16, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=18, starts_line=None, is_jump_target=False), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=20, starts_line=None, is_jump_target=False), ] expected_opinfo_jumpy = [ - Instruction(opname='SETUP_LOOP', opcode=120, arg=68, argval=71, argrepr='to 71', offset=0, starts_line=3, is_jump_target=False), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='range', argrepr='range', offset=3, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=6, starts_line=None, is_jump_target=False), - Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=9, starts_line=None, is_jump_target=False), - Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=12, starts_line=None, is_jump_target=False), - Instruction(opname='FOR_ITER', opcode=93, arg=44, argval=60, argrepr='to 60', offset=13, starts_line=None, is_jump_target=True), - Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=16, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=19, starts_line=4, is_jump_target=False), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=22, starts_line=None, is_jump_target=False), - Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=25, starts_line=None, is_jump_target=False), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=28, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=29, starts_line=5, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=32, starts_line=None, is_jump_target=False), - Instruction(opname='COMPARE_OP', opcode=107, arg=0, argval='<', argrepr='<', offset=35, starts_line=None, is_jump_target=False), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=44, argval=44, argrepr='', offset=38, starts_line=None, is_jump_target=False), - Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=13, argval=13, argrepr='', offset=41, starts_line=6, is_jump_target=False), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=44, starts_line=7, is_jump_target=True), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=47, starts_line=None, is_jump_target=False), - Instruction(opname='COMPARE_OP', opcode=107, arg=4, argval='>', argrepr='>', offset=50, starts_line=None, is_jump_target=False), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=13, argval=13, argrepr='', offset=53, starts_line=None, is_jump_target=False), - Instruction(opname='BREAK_LOOP', opcode=80, arg=None, argval=None, argrepr='', offset=56, starts_line=8, is_jump_target=False), - Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=13, argval=13, argrepr='', offset=57, starts_line=None, is_jump_target=False), - Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=60, starts_line=None, is_jump_target=True), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=61, starts_line=10, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=64, starts_line=None, is_jump_target=False), - Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=67, starts_line=None, is_jump_target=False), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=70, starts_line=None, is_jump_target=False), - Instruction(opname='SETUP_LOOP', opcode=120, arg=68, argval=142, argrepr='to 142', offset=71, starts_line=11, is_jump_target=True), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=74, starts_line=None, is_jump_target=True), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=131, argval=131, argrepr='', offset=77, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=80, starts_line=12, is_jump_target=False), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=83, starts_line=None, is_jump_target=False), - Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=86, starts_line=None, is_jump_target=False), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=89, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=90, starts_line=13, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=93, starts_line=None, is_jump_target=False), - Instruction(opname='INPLACE_SUBTRACT', opcode=56, arg=None, argval=None, argrepr='', offset=96, starts_line=None, is_jump_target=False), - Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=97, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=100, starts_line=14, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=103, starts_line=None, is_jump_target=False), - Instruction(opname='COMPARE_OP', opcode=107, arg=4, argval='>', argrepr='>', offset=106, starts_line=None, is_jump_target=False), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=115, argval=115, argrepr='', offset=109, starts_line=None, is_jump_target=False), - Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=74, argval=74, argrepr='', offset=112, starts_line=15, is_jump_target=False), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=115, starts_line=16, is_jump_target=True), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=118, starts_line=None, is_jump_target=False), - Instruction(opname='COMPARE_OP', opcode=107, arg=0, argval='<', argrepr='<', offset=121, starts_line=None, is_jump_target=False), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=74, argval=74, argrepr='', offset=124, starts_line=None, is_jump_target=False), - Instruction(opname='BREAK_LOOP', opcode=80, arg=None, argval=None, argrepr='', offset=127, starts_line=17, is_jump_target=False), - Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=74, argval=74, argrepr='', offset=128, starts_line=None, is_jump_target=False), - Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=131, starts_line=None, is_jump_target=True), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=132, starts_line=19, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=135, starts_line=None, is_jump_target=False), - Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=138, starts_line=None, is_jump_target=False), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=141, starts_line=None, is_jump_target=False), - Instruction(opname='SETUP_FINALLY', opcode=122, arg=73, argval=218, argrepr='to 218', offset=142, starts_line=20, is_jump_target=True), - Instruction(opname='SETUP_EXCEPT', opcode=121, arg=12, argval=160, argrepr='to 160', offset=145, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=148, starts_line=21, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=151, starts_line=None, is_jump_target=False), - Instruction(opname='BINARY_TRUE_DIVIDE', opcode=27, arg=None, argval=None, argrepr='', offset=154, starts_line=None, is_jump_target=False), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=155, starts_line=None, is_jump_target=False), - Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=156, starts_line=None, is_jump_target=False), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=28, argval=188, argrepr='to 188', offset=157, starts_line=None, is_jump_target=False), - Instruction(opname='DUP_TOP', opcode=4, arg=None, argval=None, argrepr='', offset=160, starts_line=22, is_jump_target=True), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=2, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=161, starts_line=None, is_jump_target=False), - Instruction(opname='COMPARE_OP', opcode=107, arg=10, argval='exception match', argrepr='exception match', offset=164, starts_line=None, is_jump_target=False), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=187, argval=187, argrepr='', offset=167, starts_line=None, is_jump_target=False), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=170, starts_line=None, is_jump_target=False), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=171, starts_line=None, is_jump_target=False), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=172, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=173, starts_line=23, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=176, starts_line=None, is_jump_target=False), - Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=179, starts_line=None, is_jump_target=False), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=182, starts_line=None, is_jump_target=False), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=183, starts_line=None, is_jump_target=False), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=27, argval=214, argrepr='to 214', offset=184, starts_line=None, is_jump_target=False), - Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=187, starts_line=None, is_jump_target=True), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=188, starts_line=25, is_jump_target=True), - Instruction(opname='SETUP_WITH', opcode=143, arg=17, argval=211, argrepr='to 211', offset=191, starts_line=None, is_jump_target=False), - Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=194, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=197, starts_line=26, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Never reach this', argrepr="'Never reach this'", offset=200, starts_line=None, is_jump_target=False), - Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=203, starts_line=None, is_jump_target=False), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=206, starts_line=None, is_jump_target=False), - Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=207, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=208, starts_line=None, is_jump_target=False), - Instruction(opname='WITH_CLEANUP_START', opcode=81, arg=None, argval=None, argrepr='', offset=211, starts_line=None, is_jump_target=True), - Instruction(opname='WITH_CLEANUP_FINISH', opcode=82, arg=None, argval=None, argrepr='', offset=212, starts_line=None, is_jump_target=False), - Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=213, starts_line=None, is_jump_target=False), - Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=214, starts_line=None, is_jump_target=True), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=215, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=218, starts_line=28, is_jump_target=True), - Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=221, starts_line=None, is_jump_target=False), - Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=224, starts_line=None, is_jump_target=False), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=227, starts_line=None, is_jump_target=False), - Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=228, starts_line=None, is_jump_target=False), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=229, starts_line=None, is_jump_target=False), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=232, starts_line=None, is_jump_target=False), + Instruction(opname='SETUP_LOOP', opcode=120, arg=52, argval=54, argrepr='to 54', offset=0, starts_line=3, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='range', argrepr='range', offset=2, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=4, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=6, starts_line=None, is_jump_target=False), + Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=8, starts_line=None, is_jump_target=False), + Instruction(opname='FOR_ITER', opcode=93, arg=32, argval=44, argrepr='to 44', offset=10, starts_line=None, is_jump_target=True), + Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=12, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=14, starts_line=4, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=16, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=18, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=20, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=22, starts_line=5, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=24, starts_line=None, is_jump_target=False), + Instruction(opname='COMPARE_OP', opcode=107, arg=0, argval='<', argrepr='<', offset=26, starts_line=None, is_jump_target=False), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=32, argval=32, argrepr='', offset=28, starts_line=None, is_jump_target=False), + Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=10, argval=10, argrepr='', offset=30, starts_line=6, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=32, starts_line=7, is_jump_target=True), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=34, starts_line=None, is_jump_target=False), + Instruction(opname='COMPARE_OP', opcode=107, arg=4, argval='>', argrepr='>', offset=36, starts_line=None, is_jump_target=False), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=10, argval=10, argrepr='', offset=38, starts_line=None, is_jump_target=False), + Instruction(opname='BREAK_LOOP', opcode=80, arg=None, argval=None, argrepr='', offset=40, starts_line=8, is_jump_target=False), + Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=10, argval=10, argrepr='', offset=42, starts_line=None, is_jump_target=False), + Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=44, starts_line=None, is_jump_target=True), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=46, starts_line=10, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=48, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=50, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=52, starts_line=None, is_jump_target=False), + Instruction(opname='SETUP_LOOP', opcode=120, arg=52, argval=108, argrepr='to 108', offset=54, starts_line=11, is_jump_target=True), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=56, starts_line=None, is_jump_target=True), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=98, argval=98, argrepr='', offset=58, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=60, starts_line=12, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=62, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=64, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=66, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=68, starts_line=13, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=70, starts_line=None, is_jump_target=False), + Instruction(opname='INPLACE_SUBTRACT', opcode=56, arg=None, argval=None, argrepr='', offset=72, starts_line=None, is_jump_target=False), + Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=74, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=76, starts_line=14, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=78, starts_line=None, is_jump_target=False), + Instruction(opname='COMPARE_OP', opcode=107, arg=4, argval='>', argrepr='>', offset=80, starts_line=None, is_jump_target=False), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=86, argval=86, argrepr='', offset=82, starts_line=None, is_jump_target=False), + Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=56, argval=56, argrepr='', offset=84, starts_line=15, is_jump_target=False), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=86, starts_line=16, is_jump_target=True), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=88, starts_line=None, is_jump_target=False), + Instruction(opname='COMPARE_OP', opcode=107, arg=0, argval='<', argrepr='<', offset=90, starts_line=None, is_jump_target=False), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=56, argval=56, argrepr='', offset=92, starts_line=None, is_jump_target=False), + Instruction(opname='BREAK_LOOP', opcode=80, arg=None, argval=None, argrepr='', offset=94, starts_line=17, is_jump_target=False), + Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=56, argval=56, argrepr='', offset=96, starts_line=None, is_jump_target=False), + Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=98, starts_line=None, is_jump_target=True), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=100, starts_line=19, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=102, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=104, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=106, starts_line=None, is_jump_target=False), + Instruction(opname='SETUP_FINALLY', opcode=122, arg=70, argval=180, argrepr='to 180', offset=108, starts_line=20, is_jump_target=True), + Instruction(opname='SETUP_EXCEPT', opcode=121, arg=12, argval=124, argrepr='to 124', offset=110, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=112, starts_line=21, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=114, starts_line=None, is_jump_target=False), + Instruction(opname='BINARY_TRUE_DIVIDE', opcode=27, arg=None, argval=None, argrepr='', offset=116, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=118, starts_line=None, is_jump_target=False), + Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=120, starts_line=None, is_jump_target=False), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=28, argval=152, argrepr='to 152', offset=122, starts_line=None, is_jump_target=False), + Instruction(opname='DUP_TOP', opcode=4, arg=None, argval=None, argrepr='', offset=124, starts_line=22, is_jump_target=True), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=2, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=126, starts_line=None, is_jump_target=False), + Instruction(opname='COMPARE_OP', opcode=107, arg=10, argval='exception match', argrepr='exception match', offset=128, starts_line=None, is_jump_target=False), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=150, argval=150, argrepr='', offset=130, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=132, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=134, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=136, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=138, starts_line=23, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=140, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=142, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=144, starts_line=None, is_jump_target=False), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=146, starts_line=None, is_jump_target=False), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=26, argval=176, argrepr='to 176', offset=148, starts_line=None, is_jump_target=False), + Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=150, starts_line=None, is_jump_target=True), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=152, starts_line=25, is_jump_target=True), + Instruction(opname='SETUP_WITH', opcode=143, arg=14, argval=170, argrepr='to 170', offset=154, starts_line=None, is_jump_target=False), + Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=156, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=158, starts_line=26, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Never reach this', argrepr="'Never reach this'", offset=160, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=162, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=164, starts_line=None, is_jump_target=False), + Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=166, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=168, starts_line=None, is_jump_target=False), + Instruction(opname='WITH_CLEANUP_START', opcode=81, arg=None, argval=None, argrepr='', offset=170, starts_line=None, is_jump_target=True), + Instruction(opname='WITH_CLEANUP_FINISH', opcode=82, arg=None, argval=None, argrepr='', offset=172, starts_line=None, is_jump_target=False), + Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=174, starts_line=None, is_jump_target=False), + Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=176, starts_line=None, is_jump_target=True), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=178, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=180, starts_line=28, is_jump_target=True), + Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=182, starts_line=None, is_jump_target=False), + Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=184, starts_line=None, is_jump_target=False), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=186, starts_line=None, is_jump_target=False), + Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=188, starts_line=None, is_jump_target=False), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=190, starts_line=None, is_jump_target=False), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=192, starts_line=None, is_jump_target=False), ] # One last piece of inspect fodder to check the default line number handling def simple(): pass expected_opinfo_simple = [ Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=0, starts_line=simple.__code__.co_firstlineno, is_jump_target=False), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=3, starts_line=None, is_jump_target=False) + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=2, starts_line=None, is_jump_target=False) ] diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 33163b9387..2258c6b1ba 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -291,7 +291,7 @@ constructor: ... ... Non-example text. ... - ... >>> print('another\example') + ... >>> print('another\\example') ... another ... example ... ''' @@ -1876,7 +1876,6 @@ if not hasattr(sys, 'gettrace') or not sys.gettrace(): To demonstrate this, we'll create a fake standard input that captures our debugger input: - >>> import tempfile >>> real_stdin = sys.stdin >>> sys.stdin = _FakeInput([ ... 'print(x)', # print data defined by the example @@ -1917,7 +1916,7 @@ if not hasattr(sys, 'gettrace') or not sys.gettrace(): ... finally: ... sys.stdin = real_stdin --Return-- - > <doctest test.test_doctest.test_pdb_set_trace[8]>(3)calls_set_trace()->None + > <doctest test.test_doctest.test_pdb_set_trace[7]>(3)calls_set_trace()->None -> import pdb; pdb.set_trace() (Pdb) print(y) 2 @@ -2798,7 +2797,6 @@ text files). ... _ = f.write(" 'abc def'\n") ... _ = f.write("\n") ... _ = f.write(' \"\"\"\n') - ... import shutil ... rc1, out1, err1 = script_helper.assert_python_failure( ... '-m', 'doctest', fn, fn2) ... rc2, out2, err2 = script_helper.assert_python_ok( @@ -2942,12 +2940,11 @@ Invalid doctest option: def test_main(): # Check the doctest cases in doctest itself: ret = support.run_doctest(doctest, verbosity=True) + # Check the doctest cases defined here: from test import test_doctest support.run_doctest(test_doctest, verbosity=True) -import sys, re, io - def test_coverage(coverdir): trace = support.import_module('trace') tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,], diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py new file mode 100644 index 0000000000..47a501018a --- /dev/null +++ b/Lib/test/test_dtrace.py @@ -0,0 +1,178 @@ +import dis +import os.path +import re +import subprocess +import sys +import types +import unittest + +from test.support import findfile, run_unittest + + +def abspath(filename): + return os.path.abspath(findfile(filename, subdir="dtracedata")) + + +def normalize_trace_output(output): + """Normalize DTrace output for comparison. + + DTrace keeps a per-CPU buffer, and when showing the fired probes, buffers + are concatenated. So if the operating system moves our thread around, the + straight result can be "non-causal". So we add timestamps to the probe + firing, sort by that field, then strip it from the output""" + + # When compiling with '--with-pydebug', strip '[# refs]' debug output. + output = re.sub(r"\[[0-9]+ refs\]", "", output) + try: + result = [ + row.split("\t") + for row in output.splitlines() + if row and not row.startswith('#') + ] + result.sort(key=lambda row: int(row[0])) + result = [row[1] for row in result] + return "\n".join(result) + except (IndexError, ValueError): + raise AssertionError( + "tracer produced unparseable output:\n{}".format(output) + ) + + +class TraceBackend: + EXTENSION = None + COMMAND = None + COMMAND_ARGS = [] + + def run_case(self, name, optimize_python=None): + actual_output = normalize_trace_output(self.trace_python( + script_file=abspath(name + self.EXTENSION), + python_file=abspath(name + ".py"), + optimize_python=optimize_python)) + + with open(abspath(name + self.EXTENSION + ".expected")) as f: + expected_output = f.read().rstrip() + + return (expected_output, actual_output) + + def generate_trace_command(self, script_file, subcommand=None): + command = self.COMMAND + [script_file] + if subcommand: + command += ["-c", subcommand] + return command + + def trace(self, script_file, subcommand=None): + command = self.generate_trace_command(script_file, subcommand) + stdout, _ = subprocess.Popen(command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True).communicate() + return stdout + + def trace_python(self, script_file, python_file, optimize_python=None): + python_flags = [] + if optimize_python: + python_flags.extend(["-O"] * optimize_python) + subcommand = " ".join([sys.executable] + python_flags + [python_file]) + return self.trace(script_file, subcommand) + + def assert_usable(self): + try: + output = self.trace(abspath("assert_usable" + self.EXTENSION)) + output = output.strip() + except (FileNotFoundError, PermissionError) as fnfe: + output = str(fnfe) + if output != "probe: success": + raise unittest.SkipTest( + "{}(1) failed: {}".format(self.COMMAND[0], output) + ) + + +class DTraceBackend(TraceBackend): + EXTENSION = ".d" + COMMAND = ["dtrace", "-q", "-s"] + + +class SystemTapBackend(TraceBackend): + EXTENSION = ".stp" + COMMAND = ["stap", "-g"] + + +class TraceTests(unittest.TestCase): + # unittest.TestCase options + maxDiff = None + + # TraceTests options + backend = None + optimize_python = 0 + + @classmethod + def setUpClass(self): + self.backend.assert_usable() + + def run_case(self, name): + actual_output, expected_output = self.backend.run_case( + name, optimize_python=self.optimize_python) + self.assertEqual(actual_output, expected_output) + + def test_function_entry_return(self): + self.run_case("call_stack") + + def test_verify_call_opcodes(self): + """Ensure our call stack test hits all function call opcodes""" + + opcodes = set(["CALL_FUNCTION", "CALL_FUNCTION_EX", "CALL_FUNCTION_KW"]) + + with open(abspath("call_stack.py")) as f: + code_string = f.read() + + def get_function_instructions(funcname): + # Recompile with appropriate optimization setting + code = compile(source=code_string, + filename="<string>", + mode="exec", + optimize=self.optimize_python) + + for c in code.co_consts: + if isinstance(c, types.CodeType) and c.co_name == funcname: + return dis.get_instructions(c) + return [] + + for instruction in get_function_instructions('start'): + opcodes.discard(instruction.opname) + + self.assertEqual(set(), opcodes) + + def test_gc(self): + self.run_case("gc") + + def test_line(self): + self.run_case("line") + + +class DTraceNormalTests(TraceTests): + backend = DTraceBackend() + optimize_python = 0 + + +class DTraceOptimizedTests(TraceTests): + backend = DTraceBackend() + optimize_python = 2 + + +class SystemTapNormalTests(TraceTests): + backend = SystemTapBackend() + optimize_python = 0 + + +class SystemTapOptimizedTests(TraceTests): + backend = SystemTapBackend() + optimize_python = 2 + + +def test_main(): + run_unittest(DTraceNormalTests, DTraceOptimizedTests, SystemTapNormalTests, + SystemTapOptimizedTests) + + +if __name__ == '__main__': + test_main() diff --git a/Lib/test/test_dynamic.py b/Lib/test/test_dynamic.py index 5080ec9f79..3ae090fd66 100644 --- a/Lib/test/test_dynamic.py +++ b/Lib/test/test_dynamic.py @@ -1,7 +1,6 @@ # Test the most dynamic corner cases of Python's runtime semantics. import builtins -import contextlib import unittest from test.support import swap_item, swap_attr diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index aabad835a0..25f86d31de 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -1,7 +1,5 @@ import os import signal -import subprocess -import sys import unittest from test import support @@ -16,14 +14,8 @@ class EINTRTests(unittest.TestCase): # Run the tester in a sub-process, to make sure there is only one # thread (for reliable signal delivery). tester = support.findfile("eintr_tester.py", subdir="eintrdata") - - if support.verbose: - args = [sys.executable, tester] - with subprocess.Popen(args) as proc: - exitcode = proc.wait() - self.assertEqual(exitcode, 0) - else: - script_helper.assert_python_ok(tester) + # use -u to try to get the full output if the test hangs or crash + script_helper.assert_python_ok("-u", tester) if __name__ == "__main__": diff --git a/Lib/test/test_email/__init__.py b/Lib/test/test_email/__init__.py index d4a03ee071..888751e9e6 100644 --- a/Lib/test/test_email/__init__.py +++ b/Lib/test/test_email/__init__.py @@ -1,5 +1,4 @@ import os -import sys import unittest import collections import email diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index f7ac0e3ded..26ece6911f 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -581,7 +581,7 @@ class TestParser(TestParserMixin, TestEmailBase): def test_get_comment_quoted_parens(self): self._test_get_x(parser.get_comment, - '(foo\) \(\)bar)', '(foo\) \(\)bar)', ' ', [], '', ['foo) ()bar']) + r'(foo\) \(\)bar)', r'(foo\) \(\)bar)', ' ', [], '', ['foo) ()bar']) def test_get_comment_non_printable(self): self._test_get_x(parser.get_comment, @@ -625,7 +625,7 @@ class TestParser(TestParserMixin, TestEmailBase): def test_get_comment_qs_in_nested_comment(self): comment = self._test_get_x(parser.get_comment, - '(foo (b\)))', '(foo (b\)))', ' ', [], '', ['foo (b\))']) + r'(foo (b\)))', r'(foo (b\)))', ' ', [], '', [r'foo (b\))']) self.assertEqual(comment[2].content, 'b)') # get_cfws diff --git a/Lib/test/test_email/test_asian_codecs.py b/Lib/test/test_email/test_asian_codecs.py index f9cd7d9b4e..1e0caeeaed 100644 --- a/Lib/test/test_email/test_asian_codecs.py +++ b/Lib/test/test_email/test_asian_codecs.py @@ -3,7 +3,6 @@ # email package unit tests for (optional) Asian codecs import unittest -from test.support import run_unittest from test.test_email import TestEmailBase from email.charset import Charset diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 9b86a2aee9..f97ccc6711 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -32,6 +32,7 @@ from email.mime.image import MIMEImage from email.mime.base import MIMEBase from email.mime.message import MIMEMessage from email.mime.multipart import MIMEMultipart +from email.mime.nonmultipart import MIMENonMultipart from email import utils from email import errors from email import encoders @@ -2078,7 +2079,13 @@ YXNkZg== --===============0012394164==--""") self.assertEqual(m.get_payload(0).get_payload(), 'YXNkZg==') + def test_mimebase_default_policy(self): + m = MIMEBase('multipart', 'mixed') + self.assertIs(m.policy, email.policy.compat32) + def test_mimebase_custom_policy(self): + m = MIMEBase('multipart', 'mixed', policy=email.policy.default) + self.assertIs(m.policy, email.policy.default) # Test some badly formatted messages class TestNonConformant(TestEmailBase): @@ -2680,6 +2687,19 @@ message 2 msg = MIMEMultipart() self.assertTrue(msg.is_multipart()) + def test_multipart_default_policy(self): + msg = MIMEMultipart() + msg['To'] = 'a@b.com' + msg['To'] = 'c@d.com' + self.assertEqual(msg.get_all('to'), ['a@b.com', 'c@d.com']) + + def test_multipart_custom_policy(self): + msg = MIMEMultipart(policy=email.policy.default) + msg['To'] = 'a@b.com' + with self.assertRaises(ValueError) as cm: + msg['To'] = 'c@d.com' + self.assertEqual(str(cm.exception), + 'There may be at most 1 To headers in a message') # A general test of parser->model->generator idempotency. IOW, read a message # in, parse it into a message object tree, then without touching the tree, @@ -3033,7 +3053,7 @@ class TestMiscellaneous(TestEmailBase): def test_escape_backslashes(self): self.assertEqual( - utils.formataddr(('Arthur \Backslash\ Foobar', 'person@dom.ain')), + utils.formataddr((r'Arthur \Backslash\ Foobar', 'person@dom.ain')), r'"Arthur \\Backslash\\ Foobar" <person@dom.ain>') a = r'Arthur \Backslash\ Foobar' b = 'person@dom.ain' @@ -3340,6 +3360,27 @@ multipart/report g.flatten(msg, linesep='\r\n') self.assertEqual(s.getvalue(), msgtxt) + def test_mime_classes_policy_argument(self): + with openfile('audiotest.au', 'rb') as fp: + audiodata = fp.read() + with openfile('PyBanner048.gif', 'rb') as fp: + bindata = fp.read() + classes = [ + (MIMEApplication, ('',)), + (MIMEAudio, (audiodata,)), + (MIMEImage, (bindata,)), + (MIMEMessage, (Message(),)), + (MIMENonMultipart, ('multipart', 'mixed')), + (MIMEText, ('',)), + ] + for cls, constructor in classes: + with self.subTest(cls=cls.__name__, policy='compat32'): + m = cls(*constructor) + self.assertIs(m.policy, email.policy.compat32) + with self.subTest(cls=cls.__name__, policy='default'): + m = cls(*constructor, policy=email.policy.default) + self.assertIs(m.policy, email.policy.default) + # Test the iterator/generators class TestIterators(TestEmailBase): @@ -3448,7 +3489,6 @@ Do you like this message? class TestFeedParsers(TestEmailBase): def parse(self, chunks): - from email.feedparser import FeedParser feedparser = FeedParser() for chunk in chunks: feedparser.feed(chunk) diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index b1cbce26d5..7c8877fdcb 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -143,7 +143,7 @@ class TestGeneratorBase: def test_set_mangle_from_via_policy(self): source = textwrap.dedent("""\ Subject: test that - from is mangeld in the body! + from is mangled in the body! From time to time I write a rhyme. """) diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index 55ecdea9aa..af836dc972 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -1,7 +1,6 @@ import datetime import textwrap import unittest -import types from email import errors from email import policy from email.message import Message diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py index 434516226c..f3a57df9e9 100644 --- a/Lib/test/test_email/test_message.py +++ b/Lib/test/test_email/test_message.py @@ -764,6 +764,26 @@ class TestEmailMessage(TestEmailMessageBase, TestEmailBase): m.set_content(content_manager=cm) self.assertEqual(m['MIME-Version'], '1.0') + def test_as_string_uses_max_header_length_by_default(self): + m = self._str_msg('Subject: long line' + ' ab'*50 + '\n\n') + self.assertEqual(len(m.as_string().strip().splitlines()), 3) + + def test_as_string_allows_maxheaderlen(self): + m = self._str_msg('Subject: long line' + ' ab'*50 + '\n\n') + self.assertEqual(len(m.as_string(maxheaderlen=0).strip().splitlines()), + 1) + self.assertEqual(len(m.as_string(maxheaderlen=34).strip().splitlines()), + 6) + + def test_str_defaults_to_policy_max_line_length(self): + m = self._str_msg('Subject: long line' + ' ab'*50 + '\n\n') + self.assertEqual(len(str(m).strip().splitlines()), 3) + + def test_str_defaults_to_utf8(self): + m = EmailMessage() + m['Subject'] = 'unicöde' + self.assertEqual(str(m), 'Subject: unicöde\n\n') + class TestMIMEPart(TestEmailMessageBase, TestEmailBase): # Doing the full test run here may seem a bit redundant, since the two diff --git a/Lib/test/test_email/test_parser.py b/Lib/test/test_email/test_parser.py index 8ddc176389..06c86408ab 100644 --- a/Lib/test/test_email/test_parser.py +++ b/Lib/test/test_email/test_parser.py @@ -1,7 +1,7 @@ import io import email import unittest -from email.message import Message +from email.message import Message, EmailMessage from email.policy import default from test.test_email import TestEmailBase @@ -39,38 +39,71 @@ class TestParserBase: # The unicode line splitter splits on unicode linebreaks, which are # more numerous than allowed by the email RFCs; make sure we are only # splitting on those two. - msg = self.parser( - "Next-Line: not\x85broken\r\n" - "Null: not\x00broken\r\n" - "Vertical-Tab: not\vbroken\r\n" - "Form-Feed: not\fbroken\r\n" - "File-Separator: not\x1Cbroken\r\n" - "Group-Separator: not\x1Dbroken\r\n" - "Record-Separator: not\x1Ebroken\r\n" - "Line-Separator: not\u2028broken\r\n" - "Paragraph-Separator: not\u2029broken\r\n" - "\r\n", - policy=default, - ) - self.assertEqual(msg.items(), [ - ("Next-Line", "not\x85broken"), - ("Null", "not\x00broken"), - ("Vertical-Tab", "not\vbroken"), - ("Form-Feed", "not\fbroken"), - ("File-Separator", "not\x1Cbroken"), - ("Group-Separator", "not\x1Dbroken"), - ("Record-Separator", "not\x1Ebroken"), - ("Line-Separator", "not\u2028broken"), - ("Paragraph-Separator", "not\u2029broken"), - ]) - self.assertEqual(msg.get_payload(), "") + for parser in self.parsers: + with self.subTest(parser=parser.__name__): + msg = parser( + "Next-Line: not\x85broken\r\n" + "Null: not\x00broken\r\n" + "Vertical-Tab: not\vbroken\r\n" + "Form-Feed: not\fbroken\r\n" + "File-Separator: not\x1Cbroken\r\n" + "Group-Separator: not\x1Dbroken\r\n" + "Record-Separator: not\x1Ebroken\r\n" + "Line-Separator: not\u2028broken\r\n" + "Paragraph-Separator: not\u2029broken\r\n" + "\r\n", + policy=default, + ) + self.assertEqual(msg.items(), [ + ("Next-Line", "not\x85broken"), + ("Null", "not\x00broken"), + ("Vertical-Tab", "not\vbroken"), + ("Form-Feed", "not\fbroken"), + ("File-Separator", "not\x1Cbroken"), + ("Group-Separator", "not\x1Dbroken"), + ("Record-Separator", "not\x1Ebroken"), + ("Line-Separator", "not\u2028broken"), + ("Paragraph-Separator", "not\u2029broken"), + ]) + self.assertEqual(msg.get_payload(), "") + + class MyMessage(EmailMessage): + pass + + def test_custom_message_factory_on_policy(self): + for parser in self.parsers: + with self.subTest(parser=parser.__name__): + MyPolicy = default.clone(message_factory=self.MyMessage) + msg = parser("To: foo\n\ntest", policy=MyPolicy) + self.assertIsInstance(msg, self.MyMessage) + + def test_factory_arg_overrides_policy(self): + for parser in self.parsers: + with self.subTest(parser=parser.__name__): + MyPolicy = default.clone(message_factory=self.MyMessage) + msg = parser("To: foo\n\ntest", Message, policy=MyPolicy) + self.assertNotIsInstance(msg, self.MyMessage) + self.assertIsInstance(msg, Message) + +# Play some games to get nice output in subTest. This code could be clearer +# if staticmethod supported __name__. + +def message_from_file(s, *args, **kw): + f = io.StringIO(s) + return email.message_from_file(f, *args, **kw) class TestParser(TestParserBase, TestEmailBase): - parser = staticmethod(email.message_from_string) + parsers = (email.message_from_string, message_from_file) + +def message_from_bytes(s, *args, **kw): + return email.message_from_bytes(s.encode(), *args, **kw) + +def message_from_binary_file(s, *args, **kw): + f = io.BytesIO(s.encode()) + return email.message_from_binary_file(f, *args, **kw) class TestBytesParser(TestParserBase, TestEmailBase): - def parser(self, s, *args, **kw): - return email.message_from_bytes(s.encode(), *args, **kw) + parsers = (message_from_bytes, message_from_binary_file) if __name__ == '__main__': diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py index 70ac4db8b0..8fecb8a5fc 100644 --- a/Lib/test/test_email/test_policy.py +++ b/Lib/test/test_email/test_policy.py @@ -5,6 +5,7 @@ import unittest import email.policy import email.parser import email.generator +import email.message from email import headerregistry def make_defaults(base_defaults, differences): @@ -23,6 +24,7 @@ class PolicyAPITests(unittest.TestCase): 'cte_type': '8bit', 'raise_on_defect': False, 'mangle_from_': True, + 'message_factory': None, } # These default values are the ones set on email.policy.default. # If any of these defaults change, the docs must be updated. @@ -34,6 +36,7 @@ class PolicyAPITests(unittest.TestCase): 'refold_source': 'long', 'content_manager': email.policy.EmailPolicy.content_manager, 'mangle_from_': False, + 'message_factory': email.message.EmailMessage, }) # For each policy under test, we give here what we expect the defaults to @@ -62,20 +65,22 @@ class PolicyAPITests(unittest.TestCase): def test_defaults(self): for policy, expected in self.policies.items(): for attr, value in expected.items(): - self.assertEqual(getattr(policy, attr), value, - ("change {} docs/docstrings if defaults have " - "changed").format(policy)) + with self.subTest(policy=policy, attr=attr): + self.assertEqual(getattr(policy, attr), value, + ("change {} docs/docstrings if defaults have " + "changed").format(policy)) def test_all_attributes_covered(self): for policy, expected in self.policies.items(): for attr in dir(policy): - if (attr.startswith('_') or - isinstance(getattr(email.policy.EmailPolicy, attr), - types.FunctionType)): - continue - else: - self.assertIn(attr, expected, - "{} is not fully tested".format(attr)) + with self.subTest(policy=policy, attr=attr): + if (attr.startswith('_') or + isinstance(getattr(email.policy.EmailPolicy, attr), + types.FunctionType)): + continue + else: + self.assertIn(attr, expected, + "{} is not fully tested".format(attr)) def test_abc(self): with self.assertRaises(TypeError) as cm: @@ -237,6 +242,9 @@ class PolicyAPITests(unittest.TestCase): # wins), but that the order still works (right overrides left). +class TestException(Exception): + pass + class TestPolicyPropagation(unittest.TestCase): # The abstract methods are used by the parser but not by the wrapper @@ -244,40 +252,40 @@ class TestPolicyPropagation(unittest.TestCase): # policy was actually propagated all the way to feedparser. class MyPolicy(email.policy.Policy): def badmethod(self, *args, **kw): - raise Exception("test") + raise TestException("test") fold = fold_binary = header_fetch_parser = badmethod header_source_parse = header_store_parse = badmethod def test_message_from_string(self): - with self.assertRaisesRegex(Exception, "^test$"): + with self.assertRaisesRegex(TestException, "^test$"): email.message_from_string("Subject: test\n\n", policy=self.MyPolicy) def test_message_from_bytes(self): - with self.assertRaisesRegex(Exception, "^test$"): + with self.assertRaisesRegex(TestException, "^test$"): email.message_from_bytes(b"Subject: test\n\n", policy=self.MyPolicy) def test_message_from_file(self): f = io.StringIO('Subject: test\n\n') - with self.assertRaisesRegex(Exception, "^test$"): + with self.assertRaisesRegex(TestException, "^test$"): email.message_from_file(f, policy=self.MyPolicy) def test_message_from_binary_file(self): f = io.BytesIO(b'Subject: test\n\n') - with self.assertRaisesRegex(Exception, "^test$"): + with self.assertRaisesRegex(TestException, "^test$"): email.message_from_binary_file(f, policy=self.MyPolicy) # These are redundant, but we need them for black-box completeness. def test_parser(self): p = email.parser.Parser(policy=self.MyPolicy) - with self.assertRaisesRegex(Exception, "^test$"): + with self.assertRaisesRegex(TestException, "^test$"): p.parsestr('Subject: test\n\n') def test_bytes_parser(self): p = email.parser.BytesParser(policy=self.MyPolicy) - with self.assertRaisesRegex(Exception, "^test$"): + with self.assertRaisesRegex(TestException, "^test$"): p.parsebytes(b'Subject: test\n\n') # Now that we've established that all the parse methods get the diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index a78ca14886..9b04c18b0e 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -9,19 +9,6 @@ import sys import ensurepip import ensurepip._uninstall -# pip currently requires ssl support, so we ensure we handle -# it being missing (http://bugs.python.org/issue19744) -ensurepip_no_ssl = test.support.import_fresh_module("ensurepip", - blocked=["ssl"]) -try: - import ssl -except ImportError: - def requires_usable_pip(f): - deco = unittest.skip(ensurepip._MISSING_SSL_MESSAGE) - return deco(f) -else: - def requires_usable_pip(f): - return f class TestEnsurePipVersion(unittest.TestCase): @@ -47,7 +34,6 @@ class EnsurepipMixin: class TestBootstrap(EnsurepipMixin, unittest.TestCase): - @requires_usable_pip def test_basic_bootstrapping(self): ensurepip.bootstrap() @@ -62,7 +48,6 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): additional_paths = self.run_pip.call_args[0][1] self.assertEqual(len(additional_paths), 2) - @requires_usable_pip def test_bootstrapping_with_root(self): ensurepip.bootstrap(root="/foo/bar/") @@ -75,7 +60,6 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): unittest.mock.ANY, ) - @requires_usable_pip def test_bootstrapping_with_user(self): ensurepip.bootstrap(user=True) @@ -87,7 +71,6 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): unittest.mock.ANY, ) - @requires_usable_pip def test_bootstrapping_with_upgrade(self): ensurepip.bootstrap(upgrade=True) @@ -99,7 +82,6 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): unittest.mock.ANY, ) - @requires_usable_pip def test_bootstrapping_with_verbosity_1(self): ensurepip.bootstrap(verbosity=1) @@ -111,7 +93,6 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): unittest.mock.ANY, ) - @requires_usable_pip def test_bootstrapping_with_verbosity_2(self): ensurepip.bootstrap(verbosity=2) @@ -123,7 +104,6 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): unittest.mock.ANY, ) - @requires_usable_pip def test_bootstrapping_with_verbosity_3(self): ensurepip.bootstrap(verbosity=3) @@ -135,17 +115,14 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): unittest.mock.ANY, ) - @requires_usable_pip def test_bootstrapping_with_regular_install(self): ensurepip.bootstrap() self.assertEqual(self.os_environ["ENSUREPIP_OPTIONS"], "install") - @requires_usable_pip def test_bootstrapping_with_alt_install(self): ensurepip.bootstrap(altinstall=True) self.assertEqual(self.os_environ["ENSUREPIP_OPTIONS"], "altinstall") - @requires_usable_pip def test_bootstrapping_with_default_pip(self): ensurepip.bootstrap(default_pip=True) self.assertNotIn("ENSUREPIP_OPTIONS", self.os_environ) @@ -155,7 +132,6 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): ensurepip.bootstrap(altinstall=True, default_pip=True) self.assertFalse(self.run_pip.called) - @requires_usable_pip def test_pip_environment_variables_removed(self): # ensurepip deliberately ignores all pip environment variables # See http://bugs.python.org/issue19734 for details @@ -163,7 +139,6 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): ensurepip.bootstrap() self.assertNotIn("PIP_THIS_SHOULD_GO_AWAY", self.os_environ) - @requires_usable_pip def test_pip_config_file_disabled(self): # ensurepip deliberately ignores the pip config file # See http://bugs.python.org/issue20053 for details @@ -205,7 +180,6 @@ class TestUninstall(EnsurepipMixin, unittest.TestCase): self.assertFalse(self.run_pip.called) - @requires_usable_pip def test_uninstall(self): with fake_pip(): ensurepip._uninstall_helper() @@ -217,7 +191,6 @@ class TestUninstall(EnsurepipMixin, unittest.TestCase): ] ) - @requires_usable_pip def test_uninstall_with_verbosity_1(self): with fake_pip(): ensurepip._uninstall_helper(verbosity=1) @@ -229,7 +202,6 @@ class TestUninstall(EnsurepipMixin, unittest.TestCase): ] ) - @requires_usable_pip def test_uninstall_with_verbosity_2(self): with fake_pip(): ensurepip._uninstall_helper(verbosity=2) @@ -241,7 +213,6 @@ class TestUninstall(EnsurepipMixin, unittest.TestCase): ] ) - @requires_usable_pip def test_uninstall_with_verbosity_3(self): with fake_pip(): ensurepip._uninstall_helper(verbosity=3) @@ -253,7 +224,6 @@ class TestUninstall(EnsurepipMixin, unittest.TestCase): ] ) - @requires_usable_pip def test_pip_environment_variables_removed(self): # ensurepip deliberately ignores all pip environment variables # See http://bugs.python.org/issue19734 for details @@ -262,7 +232,6 @@ class TestUninstall(EnsurepipMixin, unittest.TestCase): ensurepip._uninstall_helper() self.assertNotIn("PIP_THIS_SHOULD_GO_AWAY", self.os_environ) - @requires_usable_pip def test_pip_config_file_disabled(self): # ensurepip deliberately ignores the pip config file # See http://bugs.python.org/issue20053 for details @@ -271,44 +240,12 @@ class TestUninstall(EnsurepipMixin, unittest.TestCase): self.assertEqual(self.os_environ["PIP_CONFIG_FILE"], os.devnull) -class TestMissingSSL(EnsurepipMixin, unittest.TestCase): - - def setUp(self): - sys.modules["ensurepip"] = ensurepip_no_ssl - @self.addCleanup - def restore_module(): - sys.modules["ensurepip"] = ensurepip - super().setUp() - - def test_bootstrap_requires_ssl(self): - self.os_environ["PIP_THIS_SHOULD_STAY"] = "test fodder" - with self.assertRaisesRegex(RuntimeError, "requires SSL/TLS"): - ensurepip_no_ssl.bootstrap() - self.assertFalse(self.run_pip.called) - self.assertIn("PIP_THIS_SHOULD_STAY", self.os_environ) - - def test_uninstall_requires_ssl(self): - self.os_environ["PIP_THIS_SHOULD_STAY"] = "test fodder" - with self.assertRaisesRegex(RuntimeError, "requires SSL/TLS"): - with fake_pip(): - ensurepip_no_ssl._uninstall_helper() - self.assertFalse(self.run_pip.called) - self.assertIn("PIP_THIS_SHOULD_STAY", self.os_environ) - - def test_main_exits_early_with_warning(self): - with test.support.captured_stderr() as stderr: - ensurepip_no_ssl._main(["--version"]) - warning = stderr.getvalue().strip() - self.assertTrue(warning.endswith("requires SSL/TLS"), warning) - self.assertFalse(self.run_pip.called) - # Basic testing of the main functions and their argument parsing EXPECTED_VERSION_OUTPUT = "pip " + ensurepip._PIP_VERSION class TestBootstrappingMainFunction(EnsurepipMixin, unittest.TestCase): - @requires_usable_pip def test_bootstrap_version(self): with test.support.captured_stdout() as stdout: with self.assertRaises(SystemExit): @@ -317,7 +254,6 @@ class TestBootstrappingMainFunction(EnsurepipMixin, unittest.TestCase): self.assertEqual(result, EXPECTED_VERSION_OUTPUT) self.assertFalse(self.run_pip.called) - @requires_usable_pip def test_basic_bootstrapping(self): ensurepip._main([]) @@ -342,7 +278,6 @@ class TestUninstallationMainFunction(EnsurepipMixin, unittest.TestCase): self.assertEqual(result, EXPECTED_VERSION_OUTPUT) self.assertFalse(self.run_pip.called) - @requires_usable_pip def test_basic_uninstall(self): with fake_pip(): ensurepip._uninstall._main([]) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 630b155e5b..13a89fc2db 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -3,9 +3,15 @@ import inspect import pydoc import unittest from collections import OrderedDict -from enum import Enum, IntEnum, EnumMeta, unique +from enum import Enum, IntEnum, EnumMeta, Flag, IntFlag, unique, auto from io import StringIO from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL +from test import support +try: + import threading +except ImportError: + threading = None + # for pickle tests try: @@ -32,6 +38,14 @@ try: except Exception as exc: FloatStooges = exc +try: + class FlagStooges(Flag): + LARRY = 1 + CURLY = 2 + MOE = 3 +except Exception as exc: + FlagStooges = exc + # for pickle test and subclass tests try: class StrEnum(str, Enum): @@ -60,9 +74,9 @@ except Exception as exc: # for doctests try: class Fruit(Enum): - tomato = 1 - banana = 2 - cherry = 3 + TOMATO = 1 + BANANA = 2 + CHERRY = 3 except Exception: pass @@ -104,6 +118,7 @@ class TestHelpers(unittest.TestCase): '__', '___', '____', '_____',): self.assertFalse(enum._is_dunder(s)) +# tests class TestEnum(unittest.TestCase): @@ -283,6 +298,28 @@ class TestEnum(unittest.TestCase): class Wrong(Enum): _any_name_ = 9 + def test_bool(self): + # plain Enum members are always True + class Logic(Enum): + true = True + false = False + self.assertTrue(Logic.true) + self.assertTrue(Logic.false) + # unless overridden + class RealLogic(Enum): + true = True + false = False + def __bool__(self): + return bool(self._value_) + self.assertTrue(RealLogic.true) + self.assertFalse(RealLogic.false) + # mixed Enums depend on mixed-in type + class IntLogic(int, Enum): + true = 1 + false = 0 + self.assertTrue(IntLogic.true) + self.assertFalse(IntLogic.false) + def test_contains(self): Season = self.Season self.assertIn(Season.AUTUMN, Season) @@ -1547,6 +1584,780 @@ class TestEnum(unittest.TestCase): self.assertEqual(LabelledList.unprocessed, 1) self.assertEqual(LabelledList(1), LabelledList.unprocessed) + def test_auto_number(self): + class Color(Enum): + red = auto() + blue = auto() + green = auto() + + self.assertEqual(list(Color), [Color.red, Color.blue, Color.green]) + self.assertEqual(Color.red.value, 1) + self.assertEqual(Color.blue.value, 2) + self.assertEqual(Color.green.value, 3) + + def test_auto_name(self): + class Color(Enum): + def _generate_next_value_(name, start, count, last): + return name + red = auto() + blue = auto() + green = auto() + + self.assertEqual(list(Color), [Color.red, Color.blue, Color.green]) + self.assertEqual(Color.red.value, 'red') + self.assertEqual(Color.blue.value, 'blue') + self.assertEqual(Color.green.value, 'green') + + def test_auto_name_inherit(self): + class AutoNameEnum(Enum): + def _generate_next_value_(name, start, count, last): + return name + class Color(AutoNameEnum): + red = auto() + blue = auto() + green = auto() + + self.assertEqual(list(Color), [Color.red, Color.blue, Color.green]) + self.assertEqual(Color.red.value, 'red') + self.assertEqual(Color.blue.value, 'blue') + self.assertEqual(Color.green.value, 'green') + + def test_auto_garbage(self): + class Color(Enum): + red = 'red' + blue = auto() + self.assertEqual(Color.blue.value, 1) + + def test_auto_garbage_corrected(self): + class Color(Enum): + red = 'red' + blue = 2 + green = auto() + + self.assertEqual(list(Color), [Color.red, Color.blue, Color.green]) + self.assertEqual(Color.red.value, 'red') + self.assertEqual(Color.blue.value, 2) + self.assertEqual(Color.green.value, 3) + + def test_duplicate_auto(self): + class Dupes(Enum): + first = primero = auto() + second = auto() + third = auto() + self.assertEqual([Dupes.first, Dupes.second, Dupes.third], list(Dupes)) + + +class TestOrder(unittest.TestCase): + + def test_same_members(self): + class Color(Enum): + _order_ = 'red green blue' + red = 1 + green = 2 + blue = 3 + + def test_same_members_with_aliases(self): + class Color(Enum): + _order_ = 'red green blue' + red = 1 + green = 2 + blue = 3 + verde = green + + def test_same_members_wrong_order(self): + with self.assertRaisesRegex(TypeError, 'member order does not match _order_'): + class Color(Enum): + _order_ = 'red green blue' + red = 1 + blue = 3 + green = 2 + + def test_order_has_extra_members(self): + with self.assertRaisesRegex(TypeError, 'member order does not match _order_'): + class Color(Enum): + _order_ = 'red green blue purple' + red = 1 + green = 2 + blue = 3 + + def test_order_has_extra_members_with_aliases(self): + with self.assertRaisesRegex(TypeError, 'member order does not match _order_'): + class Color(Enum): + _order_ = 'red green blue purple' + red = 1 + green = 2 + blue = 3 + verde = green + + def test_enum_has_extra_members(self): + with self.assertRaisesRegex(TypeError, 'member order does not match _order_'): + class Color(Enum): + _order_ = 'red green blue' + red = 1 + green = 2 + blue = 3 + purple = 4 + + def test_enum_has_extra_members_with_aliases(self): + with self.assertRaisesRegex(TypeError, 'member order does not match _order_'): + class Color(Enum): + _order_ = 'red green blue' + red = 1 + green = 2 + blue = 3 + purple = 4 + verde = green + + +class TestFlag(unittest.TestCase): + """Tests of the Flags.""" + + class Perm(Flag): + R, W, X = 4, 2, 1 + + class Open(Flag): + RO = 0 + WO = 1 + RW = 2 + AC = 3 + CE = 1<<19 + + def test_str(self): + Perm = self.Perm + self.assertEqual(str(Perm.R), 'Perm.R') + self.assertEqual(str(Perm.W), 'Perm.W') + self.assertEqual(str(Perm.X), 'Perm.X') + self.assertEqual(str(Perm.R | Perm.W), 'Perm.R|W') + self.assertEqual(str(Perm.R | Perm.W | Perm.X), 'Perm.R|W|X') + self.assertEqual(str(Perm(0)), 'Perm.0') + self.assertEqual(str(~Perm.R), 'Perm.W|X') + self.assertEqual(str(~Perm.W), 'Perm.R|X') + self.assertEqual(str(~Perm.X), 'Perm.R|W') + self.assertEqual(str(~(Perm.R | Perm.W)), 'Perm.X') + self.assertEqual(str(~(Perm.R | Perm.W | Perm.X)), 'Perm.0') + self.assertEqual(str(Perm(~0)), 'Perm.R|W|X') + + Open = self.Open + self.assertEqual(str(Open.RO), 'Open.RO') + self.assertEqual(str(Open.WO), 'Open.WO') + self.assertEqual(str(Open.AC), 'Open.AC') + self.assertEqual(str(Open.RO | Open.CE), 'Open.CE') + self.assertEqual(str(Open.WO | Open.CE), 'Open.CE|WO') + self.assertEqual(str(~Open.RO), 'Open.CE|AC|RW|WO') + self.assertEqual(str(~Open.WO), 'Open.CE|RW') + self.assertEqual(str(~Open.AC), 'Open.CE') + self.assertEqual(str(~(Open.RO | Open.CE)), 'Open.AC') + self.assertEqual(str(~(Open.WO | Open.CE)), 'Open.RW') + + def test_repr(self): + Perm = self.Perm + self.assertEqual(repr(Perm.R), '<Perm.R: 4>') + self.assertEqual(repr(Perm.W), '<Perm.W: 2>') + self.assertEqual(repr(Perm.X), '<Perm.X: 1>') + self.assertEqual(repr(Perm.R | Perm.W), '<Perm.R|W: 6>') + self.assertEqual(repr(Perm.R | Perm.W | Perm.X), '<Perm.R|W|X: 7>') + self.assertEqual(repr(Perm(0)), '<Perm.0: 0>') + self.assertEqual(repr(~Perm.R), '<Perm.W|X: 3>') + self.assertEqual(repr(~Perm.W), '<Perm.R|X: 5>') + self.assertEqual(repr(~Perm.X), '<Perm.R|W: 6>') + self.assertEqual(repr(~(Perm.R | Perm.W)), '<Perm.X: 1>') + self.assertEqual(repr(~(Perm.R | Perm.W | Perm.X)), '<Perm.0: 0>') + self.assertEqual(repr(Perm(~0)), '<Perm.R|W|X: 7>') + + Open = self.Open + self.assertEqual(repr(Open.RO), '<Open.RO: 0>') + self.assertEqual(repr(Open.WO), '<Open.WO: 1>') + self.assertEqual(repr(Open.AC), '<Open.AC: 3>') + self.assertEqual(repr(Open.RO | Open.CE), '<Open.CE: 524288>') + self.assertEqual(repr(Open.WO | Open.CE), '<Open.CE|WO: 524289>') + self.assertEqual(repr(~Open.RO), '<Open.CE|AC|RW|WO: 524291>') + self.assertEqual(repr(~Open.WO), '<Open.CE|RW: 524290>') + self.assertEqual(repr(~Open.AC), '<Open.CE: 524288>') + self.assertEqual(repr(~(Open.RO | Open.CE)), '<Open.AC: 3>') + self.assertEqual(repr(~(Open.WO | Open.CE)), '<Open.RW: 2>') + + def test_or(self): + Perm = self.Perm + for i in Perm: + for j in Perm: + self.assertEqual((i | j), Perm(i.value | j.value)) + self.assertEqual((i | j).value, i.value | j.value) + self.assertIs(type(i | j), Perm) + for i in Perm: + self.assertIs(i | i, i) + Open = self.Open + self.assertIs(Open.RO | Open.CE, Open.CE) + + def test_and(self): + Perm = self.Perm + RW = Perm.R | Perm.W + RX = Perm.R | Perm.X + WX = Perm.W | Perm.X + RWX = Perm.R | Perm.W | Perm.X + values = list(Perm) + [RW, RX, WX, RWX, Perm(0)] + for i in values: + for j in values: + self.assertEqual((i & j).value, i.value & j.value) + self.assertIs(type(i & j), Perm) + for i in Perm: + self.assertIs(i & i, i) + self.assertIs(i & RWX, i) + self.assertIs(RWX & i, i) + Open = self.Open + self.assertIs(Open.RO & Open.CE, Open.RO) + + def test_xor(self): + Perm = self.Perm + for i in Perm: + for j in Perm: + self.assertEqual((i ^ j).value, i.value ^ j.value) + self.assertIs(type(i ^ j), Perm) + for i in Perm: + self.assertIs(i ^ Perm(0), i) + self.assertIs(Perm(0) ^ i, i) + Open = self.Open + self.assertIs(Open.RO ^ Open.CE, Open.CE) + self.assertIs(Open.CE ^ Open.CE, Open.RO) + + def test_invert(self): + Perm = self.Perm + RW = Perm.R | Perm.W + RX = Perm.R | Perm.X + WX = Perm.W | Perm.X + RWX = Perm.R | Perm.W | Perm.X + values = list(Perm) + [RW, RX, WX, RWX, Perm(0)] + for i in values: + self.assertIs(type(~i), Perm) + self.assertEqual(~~i, i) + for i in Perm: + self.assertIs(~~i, i) + Open = self.Open + self.assertIs(Open.WO & ~Open.WO, Open.RO) + self.assertIs((Open.WO|Open.CE) & ~Open.WO, Open.CE) + + def test_bool(self): + Perm = self.Perm + for f in Perm: + self.assertTrue(f) + Open = self.Open + for f in Open: + self.assertEqual(bool(f.value), bool(f)) + + def test_programatic_function_string(self): + Perm = Flag('Perm', 'R W X') + lst = list(Perm) + self.assertEqual(len(lst), len(Perm)) + self.assertEqual(len(Perm), 3, Perm) + self.assertEqual(lst, [Perm.R, Perm.W, Perm.X]) + for i, n in enumerate('R W X'.split()): + v = 1<<i + e = Perm(v) + self.assertEqual(e.value, v) + self.assertEqual(type(e.value), int) + self.assertEqual(e.name, n) + self.assertIn(e, Perm) + self.assertIs(type(e), Perm) + + def test_programatic_function_string_with_start(self): + Perm = Flag('Perm', 'R W X', start=8) + lst = list(Perm) + self.assertEqual(len(lst), len(Perm)) + self.assertEqual(len(Perm), 3, Perm) + self.assertEqual(lst, [Perm.R, Perm.W, Perm.X]) + for i, n in enumerate('R W X'.split()): + v = 8<<i + e = Perm(v) + self.assertEqual(e.value, v) + self.assertEqual(type(e.value), int) + self.assertEqual(e.name, n) + self.assertIn(e, Perm) + self.assertIs(type(e), Perm) + + def test_programatic_function_string_list(self): + Perm = Flag('Perm', ['R', 'W', 'X']) + lst = list(Perm) + self.assertEqual(len(lst), len(Perm)) + self.assertEqual(len(Perm), 3, Perm) + self.assertEqual(lst, [Perm.R, Perm.W, Perm.X]) + for i, n in enumerate('R W X'.split()): + v = 1<<i + e = Perm(v) + self.assertEqual(e.value, v) + self.assertEqual(type(e.value), int) + self.assertEqual(e.name, n) + self.assertIn(e, Perm) + self.assertIs(type(e), Perm) + + def test_programatic_function_iterable(self): + Perm = Flag('Perm', (('R', 2), ('W', 8), ('X', 32))) + lst = list(Perm) + self.assertEqual(len(lst), len(Perm)) + self.assertEqual(len(Perm), 3, Perm) + self.assertEqual(lst, [Perm.R, Perm.W, Perm.X]) + for i, n in enumerate('R W X'.split()): + v = 1<<(2*i+1) + e = Perm(v) + self.assertEqual(e.value, v) + self.assertEqual(type(e.value), int) + self.assertEqual(e.name, n) + self.assertIn(e, Perm) + self.assertIs(type(e), Perm) + + def test_programatic_function_from_dict(self): + Perm = Flag('Perm', OrderedDict((('R', 2), ('W', 8), ('X', 32)))) + lst = list(Perm) + self.assertEqual(len(lst), len(Perm)) + self.assertEqual(len(Perm), 3, Perm) + self.assertEqual(lst, [Perm.R, Perm.W, Perm.X]) + for i, n in enumerate('R W X'.split()): + v = 1<<(2*i+1) + e = Perm(v) + self.assertEqual(e.value, v) + self.assertEqual(type(e.value), int) + self.assertEqual(e.name, n) + self.assertIn(e, Perm) + self.assertIs(type(e), Perm) + + def test_pickle(self): + if isinstance(FlagStooges, Exception): + raise FlagStooges + test_pickle_dump_load(self.assertIs, FlagStooges.CURLY|FlagStooges.MOE) + test_pickle_dump_load(self.assertIs, FlagStooges) + + def test_containment(self): + Perm = self.Perm + R, W, X = Perm + RW = R | W + RX = R | X + WX = W | X + RWX = R | W | X + self.assertTrue(R in RW) + self.assertTrue(R in RX) + self.assertTrue(R in RWX) + self.assertTrue(W in RW) + self.assertTrue(W in WX) + self.assertTrue(W in RWX) + self.assertTrue(X in RX) + self.assertTrue(X in WX) + self.assertTrue(X in RWX) + self.assertFalse(R in WX) + self.assertFalse(W in RX) + self.assertFalse(X in RW) + + def test_auto_number(self): + class Color(Flag): + red = auto() + blue = auto() + green = auto() + + self.assertEqual(list(Color), [Color.red, Color.blue, Color.green]) + self.assertEqual(Color.red.value, 1) + self.assertEqual(Color.blue.value, 2) + self.assertEqual(Color.green.value, 4) + + def test_auto_number_garbage(self): + with self.assertRaisesRegex(TypeError, 'Invalid Flag value: .not an int.'): + class Color(Flag): + red = 'not an int' + blue = auto() + + def test_cascading_failure(self): + class Bizarre(Flag): + c = 3 + d = 4 + f = 6 + # Bizarre.c | Bizarre.d + self.assertRaisesRegex(ValueError, "5 is not a valid Bizarre", Bizarre, 5) + self.assertRaisesRegex(ValueError, "5 is not a valid Bizarre", Bizarre, 5) + self.assertRaisesRegex(ValueError, "2 is not a valid Bizarre", Bizarre, 2) + self.assertRaisesRegex(ValueError, "2 is not a valid Bizarre", Bizarre, 2) + self.assertRaisesRegex(ValueError, "1 is not a valid Bizarre", Bizarre, 1) + self.assertRaisesRegex(ValueError, "1 is not a valid Bizarre", Bizarre, 1) + + def test_duplicate_auto(self): + class Dupes(Enum): + first = primero = auto() + second = auto() + third = auto() + self.assertEqual([Dupes.first, Dupes.second, Dupes.third], list(Dupes)) + + def test_bizarre(self): + class Bizarre(Flag): + b = 3 + c = 4 + d = 6 + self.assertEqual(repr(Bizarre(7)), '<Bizarre.d|c|b: 7>') + + @unittest.skipUnless(threading, 'Threading required for this test.') + @support.reap_threads + def test_unique_composite(self): + # override __eq__ to be identity only + class TestFlag(Flag): + one = auto() + two = auto() + three = auto() + four = auto() + five = auto() + six = auto() + seven = auto() + eight = auto() + def __eq__(self, other): + return self is other + def __hash__(self): + return hash(self._value_) + # have multiple threads competing to complete the composite members + seen = set() + failed = False + def cycle_enum(): + nonlocal failed + try: + for i in range(256): + seen.add(TestFlag(i)) + except Exception: + failed = True + threads = [ + threading.Thread(target=cycle_enum) + for _ in range(8) + ] + with support.start_threads(threads): + pass + # check that only 248 members were created + self.assertFalse( + failed, + 'at least one thread failed while creating composite members') + self.assertEqual(256, len(seen), 'too many composite members created') + + +class TestIntFlag(unittest.TestCase): + """Tests of the IntFlags.""" + + class Perm(IntFlag): + X = 1 << 0 + W = 1 << 1 + R = 1 << 2 + + class Open(IntFlag): + RO = 0 + WO = 1 + RW = 2 + AC = 3 + CE = 1<<19 + + def test_type(self): + Perm = self.Perm + Open = self.Open + for f in Perm: + self.assertTrue(isinstance(f, Perm)) + self.assertEqual(f, f.value) + self.assertTrue(isinstance(Perm.W | Perm.X, Perm)) + self.assertEqual(Perm.W | Perm.X, 3) + for f in Open: + self.assertTrue(isinstance(f, Open)) + self.assertEqual(f, f.value) + self.assertTrue(isinstance(Open.WO | Open.RW, Open)) + self.assertEqual(Open.WO | Open.RW, 3) + + + def test_str(self): + Perm = self.Perm + self.assertEqual(str(Perm.R), 'Perm.R') + self.assertEqual(str(Perm.W), 'Perm.W') + self.assertEqual(str(Perm.X), 'Perm.X') + self.assertEqual(str(Perm.R | Perm.W), 'Perm.R|W') + self.assertEqual(str(Perm.R | Perm.W | Perm.X), 'Perm.R|W|X') + self.assertEqual(str(Perm.R | 8), 'Perm.8|R') + self.assertEqual(str(Perm(0)), 'Perm.0') + self.assertEqual(str(Perm(8)), 'Perm.8') + self.assertEqual(str(~Perm.R), 'Perm.W|X') + self.assertEqual(str(~Perm.W), 'Perm.R|X') + self.assertEqual(str(~Perm.X), 'Perm.R|W') + self.assertEqual(str(~(Perm.R | Perm.W)), 'Perm.X') + self.assertEqual(str(~(Perm.R | Perm.W | Perm.X)), 'Perm.-8') + self.assertEqual(str(~(Perm.R | 8)), 'Perm.W|X') + self.assertEqual(str(Perm(~0)), 'Perm.R|W|X') + self.assertEqual(str(Perm(~8)), 'Perm.R|W|X') + + Open = self.Open + self.assertEqual(str(Open.RO), 'Open.RO') + self.assertEqual(str(Open.WO), 'Open.WO') + self.assertEqual(str(Open.AC), 'Open.AC') + self.assertEqual(str(Open.RO | Open.CE), 'Open.CE') + self.assertEqual(str(Open.WO | Open.CE), 'Open.CE|WO') + self.assertEqual(str(Open(4)), 'Open.4') + self.assertEqual(str(~Open.RO), 'Open.CE|AC|RW|WO') + self.assertEqual(str(~Open.WO), 'Open.CE|RW') + self.assertEqual(str(~Open.AC), 'Open.CE') + self.assertEqual(str(~(Open.RO | Open.CE)), 'Open.AC|RW|WO') + self.assertEqual(str(~(Open.WO | Open.CE)), 'Open.RW') + self.assertEqual(str(Open(~4)), 'Open.CE|AC|RW|WO') + + def test_repr(self): + Perm = self.Perm + self.assertEqual(repr(Perm.R), '<Perm.R: 4>') + self.assertEqual(repr(Perm.W), '<Perm.W: 2>') + self.assertEqual(repr(Perm.X), '<Perm.X: 1>') + self.assertEqual(repr(Perm.R | Perm.W), '<Perm.R|W: 6>') + self.assertEqual(repr(Perm.R | Perm.W | Perm.X), '<Perm.R|W|X: 7>') + self.assertEqual(repr(Perm.R | 8), '<Perm.8|R: 12>') + self.assertEqual(repr(Perm(0)), '<Perm.0: 0>') + self.assertEqual(repr(Perm(8)), '<Perm.8: 8>') + self.assertEqual(repr(~Perm.R), '<Perm.W|X: -5>') + self.assertEqual(repr(~Perm.W), '<Perm.R|X: -3>') + self.assertEqual(repr(~Perm.X), '<Perm.R|W: -2>') + self.assertEqual(repr(~(Perm.R | Perm.W)), '<Perm.X: -7>') + self.assertEqual(repr(~(Perm.R | Perm.W | Perm.X)), '<Perm.-8: -8>') + self.assertEqual(repr(~(Perm.R | 8)), '<Perm.W|X: -13>') + self.assertEqual(repr(Perm(~0)), '<Perm.R|W|X: -1>') + self.assertEqual(repr(Perm(~8)), '<Perm.R|W|X: -9>') + + Open = self.Open + self.assertEqual(repr(Open.RO), '<Open.RO: 0>') + self.assertEqual(repr(Open.WO), '<Open.WO: 1>') + self.assertEqual(repr(Open.AC), '<Open.AC: 3>') + self.assertEqual(repr(Open.RO | Open.CE), '<Open.CE: 524288>') + self.assertEqual(repr(Open.WO | Open.CE), '<Open.CE|WO: 524289>') + self.assertEqual(repr(Open(4)), '<Open.4: 4>') + self.assertEqual(repr(~Open.RO), '<Open.CE|AC|RW|WO: -1>') + self.assertEqual(repr(~Open.WO), '<Open.CE|RW: -2>') + self.assertEqual(repr(~Open.AC), '<Open.CE: -4>') + self.assertEqual(repr(~(Open.RO | Open.CE)), '<Open.AC|RW|WO: -524289>') + self.assertEqual(repr(~(Open.WO | Open.CE)), '<Open.RW: -524290>') + self.assertEqual(repr(Open(~4)), '<Open.CE|AC|RW|WO: -5>') + + def test_or(self): + Perm = self.Perm + for i in Perm: + for j in Perm: + self.assertEqual(i | j, i.value | j.value) + self.assertEqual((i | j).value, i.value | j.value) + self.assertIs(type(i | j), Perm) + for j in range(8): + self.assertEqual(i | j, i.value | j) + self.assertEqual((i | j).value, i.value | j) + self.assertIs(type(i | j), Perm) + self.assertEqual(j | i, j | i.value) + self.assertEqual((j | i).value, j | i.value) + self.assertIs(type(j | i), Perm) + for i in Perm: + self.assertIs(i | i, i) + self.assertIs(i | 0, i) + self.assertIs(0 | i, i) + Open = self.Open + self.assertIs(Open.RO | Open.CE, Open.CE) + + def test_and(self): + Perm = self.Perm + RW = Perm.R | Perm.W + RX = Perm.R | Perm.X + WX = Perm.W | Perm.X + RWX = Perm.R | Perm.W | Perm.X + values = list(Perm) + [RW, RX, WX, RWX, Perm(0)] + for i in values: + for j in values: + self.assertEqual(i & j, i.value & j.value, 'i is %r, j is %r' % (i, j)) + self.assertEqual((i & j).value, i.value & j.value, 'i is %r, j is %r' % (i, j)) + self.assertIs(type(i & j), Perm, 'i is %r, j is %r' % (i, j)) + for j in range(8): + self.assertEqual(i & j, i.value & j) + self.assertEqual((i & j).value, i.value & j) + self.assertIs(type(i & j), Perm) + self.assertEqual(j & i, j & i.value) + self.assertEqual((j & i).value, j & i.value) + self.assertIs(type(j & i), Perm) + for i in Perm: + self.assertIs(i & i, i) + self.assertIs(i & 7, i) + self.assertIs(7 & i, i) + Open = self.Open + self.assertIs(Open.RO & Open.CE, Open.RO) + + def test_xor(self): + Perm = self.Perm + for i in Perm: + for j in Perm: + self.assertEqual(i ^ j, i.value ^ j.value) + self.assertEqual((i ^ j).value, i.value ^ j.value) + self.assertIs(type(i ^ j), Perm) + for j in range(8): + self.assertEqual(i ^ j, i.value ^ j) + self.assertEqual((i ^ j).value, i.value ^ j) + self.assertIs(type(i ^ j), Perm) + self.assertEqual(j ^ i, j ^ i.value) + self.assertEqual((j ^ i).value, j ^ i.value) + self.assertIs(type(j ^ i), Perm) + for i in Perm: + self.assertIs(i ^ 0, i) + self.assertIs(0 ^ i, i) + Open = self.Open + self.assertIs(Open.RO ^ Open.CE, Open.CE) + self.assertIs(Open.CE ^ Open.CE, Open.RO) + + def test_invert(self): + Perm = self.Perm + RW = Perm.R | Perm.W + RX = Perm.R | Perm.X + WX = Perm.W | Perm.X + RWX = Perm.R | Perm.W | Perm.X + values = list(Perm) + [RW, RX, WX, RWX, Perm(0)] + for i in values: + self.assertEqual(~i, ~i.value) + self.assertEqual((~i).value, ~i.value) + self.assertIs(type(~i), Perm) + self.assertEqual(~~i, i) + for i in Perm: + self.assertIs(~~i, i) + Open = self.Open + self.assertIs(Open.WO & ~Open.WO, Open.RO) + self.assertIs((Open.WO|Open.CE) & ~Open.WO, Open.CE) + + def test_programatic_function_string(self): + Perm = IntFlag('Perm', 'R W X') + lst = list(Perm) + self.assertEqual(len(lst), len(Perm)) + self.assertEqual(len(Perm), 3, Perm) + self.assertEqual(lst, [Perm.R, Perm.W, Perm.X]) + for i, n in enumerate('R W X'.split()): + v = 1<<i + e = Perm(v) + self.assertEqual(e.value, v) + self.assertEqual(type(e.value), int) + self.assertEqual(e, v) + self.assertEqual(e.name, n) + self.assertIn(e, Perm) + self.assertIs(type(e), Perm) + + def test_programatic_function_string_with_start(self): + Perm = IntFlag('Perm', 'R W X', start=8) + lst = list(Perm) + self.assertEqual(len(lst), len(Perm)) + self.assertEqual(len(Perm), 3, Perm) + self.assertEqual(lst, [Perm.R, Perm.W, Perm.X]) + for i, n in enumerate('R W X'.split()): + v = 8<<i + e = Perm(v) + self.assertEqual(e.value, v) + self.assertEqual(type(e.value), int) + self.assertEqual(e, v) + self.assertEqual(e.name, n) + self.assertIn(e, Perm) + self.assertIs(type(e), Perm) + + def test_programatic_function_string_list(self): + Perm = IntFlag('Perm', ['R', 'W', 'X']) + lst = list(Perm) + self.assertEqual(len(lst), len(Perm)) + self.assertEqual(len(Perm), 3, Perm) + self.assertEqual(lst, [Perm.R, Perm.W, Perm.X]) + for i, n in enumerate('R W X'.split()): + v = 1<<i + e = Perm(v) + self.assertEqual(e.value, v) + self.assertEqual(type(e.value), int) + self.assertEqual(e, v) + self.assertEqual(e.name, n) + self.assertIn(e, Perm) + self.assertIs(type(e), Perm) + + def test_programatic_function_iterable(self): + Perm = IntFlag('Perm', (('R', 2), ('W', 8), ('X', 32))) + lst = list(Perm) + self.assertEqual(len(lst), len(Perm)) + self.assertEqual(len(Perm), 3, Perm) + self.assertEqual(lst, [Perm.R, Perm.W, Perm.X]) + for i, n in enumerate('R W X'.split()): + v = 1<<(2*i+1) + e = Perm(v) + self.assertEqual(e.value, v) + self.assertEqual(type(e.value), int) + self.assertEqual(e, v) + self.assertEqual(e.name, n) + self.assertIn(e, Perm) + self.assertIs(type(e), Perm) + + def test_programatic_function_from_dict(self): + Perm = IntFlag('Perm', OrderedDict((('R', 2), ('W', 8), ('X', 32)))) + lst = list(Perm) + self.assertEqual(len(lst), len(Perm)) + self.assertEqual(len(Perm), 3, Perm) + self.assertEqual(lst, [Perm.R, Perm.W, Perm.X]) + for i, n in enumerate('R W X'.split()): + v = 1<<(2*i+1) + e = Perm(v) + self.assertEqual(e.value, v) + self.assertEqual(type(e.value), int) + self.assertEqual(e, v) + self.assertEqual(e.name, n) + self.assertIn(e, Perm) + self.assertIs(type(e), Perm) + + + def test_containment(self): + Perm = self.Perm + R, W, X = Perm + RW = R | W + RX = R | X + WX = W | X + RWX = R | W | X + self.assertTrue(R in RW) + self.assertTrue(R in RX) + self.assertTrue(R in RWX) + self.assertTrue(W in RW) + self.assertTrue(W in WX) + self.assertTrue(W in RWX) + self.assertTrue(X in RX) + self.assertTrue(X in WX) + self.assertTrue(X in RWX) + self.assertFalse(R in WX) + self.assertFalse(W in RX) + self.assertFalse(X in RW) + + def test_bool(self): + Perm = self.Perm + for f in Perm: + self.assertTrue(f) + Open = self.Open + for f in Open: + self.assertEqual(bool(f.value), bool(f)) + + @unittest.skipUnless(threading, 'Threading required for this test.') + @support.reap_threads + def test_unique_composite(self): + # override __eq__ to be identity only + class TestFlag(IntFlag): + one = auto() + two = auto() + three = auto() + four = auto() + five = auto() + six = auto() + seven = auto() + eight = auto() + def __eq__(self, other): + return self is other + def __hash__(self): + return hash(self._value_) + # have multiple threads competing to complete the composite members + seen = set() + failed = False + def cycle_enum(): + nonlocal failed + try: + for i in range(256): + seen.add(TestFlag(i)) + except Exception: + failed = True + threads = [ + threading.Thread(target=cycle_enum) + for _ in range(8) + ] + with support.start_threads(threads): + pass + # check that only 248 members were created + self.assertFalse( + failed, + 'at least one thread failed while creating composite members') + self.assertEqual(256, len(seen), 'too many composite members created') + class TestUnique(unittest.TestCase): @@ -1739,5 +2550,49 @@ class TestStdLib(unittest.TestCase): if failed: self.fail("result does not equal expected, see print above") + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, enum) + + +# These are unordered here on purpose to ensure that declaration order +# makes no difference. +CONVERT_TEST_NAME_D = 5 +CONVERT_TEST_NAME_C = 5 +CONVERT_TEST_NAME_B = 5 +CONVERT_TEST_NAME_A = 5 # This one should sort first. +CONVERT_TEST_NAME_E = 5 +CONVERT_TEST_NAME_F = 5 + +class TestIntEnumConvert(unittest.TestCase): + def test_convert_value_lookup_priority(self): + test_type = enum.IntEnum._convert( + 'UnittestConvert', + ('test.test_enum', '__main__')[__name__=='__main__'], + filter=lambda x: x.startswith('CONVERT_TEST_')) + # We don't want the reverse lookup value to vary when there are + # multiple possible names for a given value. It should always + # report the first lexigraphical name in that case. + self.assertEqual(test_type(5).name, 'CONVERT_TEST_NAME_A') + + def test_convert(self): + test_type = enum.IntEnum._convert( + 'UnittestConvert', + ('test.test_enum', '__main__')[__name__=='__main__'], + filter=lambda x: x.startswith('CONVERT_TEST_')) + # Ensure that test_type has all of the desired names and values. + self.assertEqual(test_type.CONVERT_TEST_NAME_F, + test_type.CONVERT_TEST_NAME_A) + self.assertEqual(test_type.CONVERT_TEST_NAME_B, 5) + self.assertEqual(test_type.CONVERT_TEST_NAME_C, 5) + self.assertEqual(test_type.CONVERT_TEST_NAME_D, 5) + self.assertEqual(test_type.CONVERT_TEST_NAME_E, 5) + # Ensure that test_type only picked up names matching the filter. + self.assertEqual([name for name in dir(test_type) + if name[0:2] not in ('CO', '__')], + [], msg='Names other than CONVERT_TEST_* found.') + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_enumerate.py b/Lib/test/test_enumerate.py index 2630cf2d50..e455adee50 100644 --- a/Lib/test/test_enumerate.py +++ b/Lib/test/test_enumerate.py @@ -223,7 +223,7 @@ class TestReversed(unittest.TestCase, PickleTest): def test_objmethods(self): # Objects must have __len__() and __getitem__() implemented. class NoLen(object): - def __getitem__(self): return 1 + def __getitem__(self, i): return 1 nl = NoLen() self.assertRaises(TypeError, reversed, nl) @@ -232,6 +232,13 @@ class TestReversed(unittest.TestCase, PickleTest): ngi = NoGetItem() self.assertRaises(TypeError, reversed, ngi) + class Blocked(object): + def __getitem__(self, i): return 1 + def __len__(self): return 2 + __reversed__ = None + b = Blocked() + self.assertRaises(TypeError, reversed, b) + def test_pickle(self): for data in 'abc', range(5), tuple(enumerate('abc')), range(1,17,5): self.check_pickle(reversed(data), list(data)[::-1]) diff --git a/Lib/test/test_epoll.py b/Lib/test/test_epoll.py index a7359e9c29..a7aff8a59e 100644 --- a/Lib/test/test_epoll.py +++ b/Lib/test/test_epoll.py @@ -28,7 +28,6 @@ import socket import time import unittest -from test import support if not hasattr(select, "epoll"): raise unittest.SkipTest("test works only on Linux 2.6") diff --git a/Lib/test/test_pep3151.py b/Lib/test/test_exception_hierarchy.py index 7b0d465425..8649596790 100644 --- a/Lib/test/test_pep3151.py +++ b/Lib/test/test_exception_hierarchy.py @@ -2,7 +2,6 @@ import builtins import os import select import socket -import sys import unittest import errno from errno import EEXIST diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 96c3a48c31..48379222c3 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1109,7 +1109,6 @@ class ImportErrorTests(unittest.TestCase): with self.assertRaisesRegex(TypeError, msg): ImportError(invalid='keyword') - msg = "'invalid|another' is an invalid keyword argument for this function" with self.assertRaisesRegex(TypeError, msg): ImportError('test', invalid='keyword', another=True) diff --git a/Lib/test/test_extcall.py b/Lib/test/test_extcall.py index 9cb0d38669..043df01311 100644 --- a/Lib/test/test_extcall.py +++ b/Lib/test/test_extcall.py @@ -238,6 +238,11 @@ What about willful misconduct? ... TypeError: h() argument after * must be an iterable, not function + >>> h(*[1], *h) + Traceback (most recent call last): + ... + TypeError: h() argument after * must be an iterable, not function + >>> dir(*h) Traceback (most recent call last): ... diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 1562eecbd6..bdd8d1a2a6 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -7,7 +7,7 @@ import signal import subprocess import sys from test import support -from test.support import script_helper +from test.support import script_helper, is_android, requires_android_level import tempfile import unittest from textwrap import dedent @@ -23,6 +23,7 @@ except ImportError: _testcapi = None TIMEOUT = 0.5 +MS_WINDOWS = (os.name == 'nt') def expected_traceback(lineno1, lineno2, header, min_count=1): regex = header @@ -41,6 +42,10 @@ def temporary_filename(): finally: support.unlink(filename) +def requires_raise(test): + return (test if not is_android else + requires_android_level(24, 'raise() is buggy')(test)) + class FaultHandlerTests(unittest.TestCase): def get_output(self, code, filename=None, fd=None): """ @@ -58,8 +63,9 @@ class FaultHandlerTests(unittest.TestCase): pass_fds.append(fd) with support.SuppressCrashReport(): process = script_helper.spawn_python('-c', code, pass_fds=pass_fds) - stdout, stderr = process.communicate() - exitcode = process.wait() + with process: + stdout, stderr = process.communicate() + exitcode = process.wait() output = support.strip_python_stderr(stdout) output = output.decode('ascii', 'backslashreplace') if filename: @@ -73,14 +79,11 @@ class FaultHandlerTests(unittest.TestCase): with open(fd, "rb", closefd=False) as fp: output = fp.read() output = output.decode('ascii', 'backslashreplace') - output = re.sub('Current thread 0x[0-9a-f]+', - 'Current thread XXX', - output) return output.splitlines(), exitcode - def check_fatal_error(self, code, line_number, name_regex, - filename=None, all_threads=True, other_regex=None, - fd=None): + def check_error(self, code, line_number, fatal_error, *, + filename=None, all_threads=True, other_regex=None, + fd=None, know_current_thread=True): """ Check that the fault handler for fatal errors is enabled and check the traceback from the child process output. @@ -88,19 +91,22 @@ class FaultHandlerTests(unittest.TestCase): Raise an error if the output doesn't match the expected format. """ if all_threads: - header = 'Current thread XXX (most recent call first)' + if know_current_thread: + header = 'Current thread 0x[0-9a-f]+' + else: + header = 'Thread 0x[0-9a-f]+' else: - header = 'Stack (most recent call first)' - regex = """ - ^Fatal Python error: {name} + header = 'Stack' + regex = r""" + ^{fatal_error} - {header}: + {header} \(most recent call first\): File "<string>", line {lineno} in <module> """ regex = dedent(regex.format( lineno=line_number, - name=name_regex, - header=re.escape(header))).strip() + fatal_error=fatal_error, + header=header)).strip() if other_regex: regex += '|' + other_regex output, exitcode = self.get_output(code, filename=filename, fd=fd) @@ -108,26 +114,57 @@ class FaultHandlerTests(unittest.TestCase): self.assertRegex(output, regex) self.assertNotEqual(exitcode, 0) + def check_fatal_error(self, code, line_number, name_regex, **kw): + fatal_error = 'Fatal Python error: %s' % name_regex + self.check_error(code, line_number, fatal_error, **kw) + + def check_windows_exception(self, code, line_number, name_regex, **kw): + fatal_error = 'Windows fatal exception: %s' % name_regex + self.check_error(code, line_number, fatal_error, **kw) + @unittest.skipIf(sys.platform.startswith('aix'), "the first page of memory is a mapped read-only on AIX") def test_read_null(self): + if not MS_WINDOWS: + self.check_fatal_error(""" + import faulthandler + faulthandler.enable() + faulthandler._read_null() + """, + 3, + # Issue #12700: Read NULL raises SIGILL on Mac OS X Lion + '(?:Segmentation fault' + '|Bus error' + '|Illegal instruction)') + else: + self.check_windows_exception(""" + import faulthandler + faulthandler.enable() + faulthandler._read_null() + """, + 3, + 'access violation') + + @requires_raise + def test_sigsegv(self): self.check_fatal_error(""" import faulthandler faulthandler.enable() - faulthandler._read_null() + faulthandler._sigsegv() """, 3, - # Issue #12700: Read NULL raises SIGILL on Mac OS X Lion - '(?:Segmentation fault|Bus error|Illegal instruction)') + 'Segmentation fault') - def test_sigsegv(self): + @unittest.skipIf(not HAVE_THREADS, 'need threads') + def test_fatal_error_c_thread(self): self.check_fatal_error(""" import faulthandler faulthandler.enable() - faulthandler._sigsegv() + faulthandler._fatal_error_c_thread() """, 3, - 'Segmentation fault') + 'in new thread', + know_current_thread=False) def test_sigabrt(self): self.check_fatal_error(""" @@ -151,6 +188,7 @@ class FaultHandlerTests(unittest.TestCase): @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(signal, 'SIGBUS'), 'need signal.SIGBUS') + @requires_raise def test_sigbus(self): self.check_fatal_error(""" import _testcapi @@ -165,6 +203,7 @@ class FaultHandlerTests(unittest.TestCase): @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(signal, 'SIGILL'), 'need signal.SIGILL') + @requires_raise def test_sigill(self): self.check_fatal_error(""" import _testcapi @@ -208,6 +247,7 @@ class FaultHandlerTests(unittest.TestCase): '(?:Segmentation fault|Bus error)', other_regex='unable to raise a stack overflow') + @requires_raise def test_gil_released(self): self.check_fatal_error(""" import faulthandler @@ -217,6 +257,7 @@ class FaultHandlerTests(unittest.TestCase): 3, 'Segmentation fault') + @requires_raise def test_enable_file(self): with temporary_filename() as filename: self.check_fatal_error(""" @@ -231,6 +272,7 @@ class FaultHandlerTests(unittest.TestCase): @unittest.skipIf(sys.platform == "win32", "subprocess doesn't support pass_fds on Windows") + @requires_raise def test_enable_fd(self): with tempfile.TemporaryFile('wb+') as fp: fd = fp.fileno() @@ -244,6 +286,7 @@ class FaultHandlerTests(unittest.TestCase): 'Segmentation fault', fd=fd) + @requires_raise def test_enable_single_thread(self): self.check_fatal_error(""" import faulthandler @@ -254,6 +297,7 @@ class FaultHandlerTests(unittest.TestCase): 'Segmentation fault', all_threads=False) + @requires_raise def test_disable(self): code = """ import faulthandler @@ -458,14 +502,14 @@ class FaultHandlerTests(unittest.TestCase): lineno = 8 else: lineno = 10 - regex = """ + regex = r""" ^Thread 0x[0-9a-f]+ \(most recent call first\): (?: File ".*threading.py", line [0-9]+ in [_a-z]+ ){{1,3}} File "<string>", line 23 in run File ".*threading.py", line [0-9]+ in _bootstrap_inner File ".*threading.py", line [0-9]+ in _bootstrap - Current thread XXX \(most recent call first\): + Current thread 0x[0-9a-f]+ \(most recent call first\): File "<string>", line {lineno} in dump File "<string>", line 28 in <module>$ """ @@ -637,9 +681,9 @@ class FaultHandlerTests(unittest.TestCase): trace = '\n'.join(trace) if not unregister: if all_threads: - regex = 'Current thread XXX \(most recent call first\):\n' + regex = r'Current thread 0x[0-9a-f]+ \(most recent call first\):\n' else: - regex = 'Stack \(most recent call first\):\n' + regex = r'Stack \(most recent call first\):\n' regex = expected_traceback(14, 32, regex) self.assertRegex(trace, regex) else: @@ -696,6 +740,22 @@ class FaultHandlerTests(unittest.TestCase): with self.check_stderr_none(): faulthandler.register(signal.SIGUSR1) + @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') + def test_raise_exception(self): + for exc, name in ( + ('EXCEPTION_ACCESS_VIOLATION', 'access violation'), + ('EXCEPTION_INT_DIVIDE_BY_ZERO', 'int divide by zero'), + ('EXCEPTION_STACK_OVERFLOW', 'stack overflow'), + ): + self.check_windows_exception(f""" + import faulthandler + faulthandler.enable() + faulthandler._raise_exception(faulthandler._{exc}) + """, + 3, + name) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py index 4e392b770c..65be30f0b2 100644 --- a/Lib/test/test_file.py +++ b/Lib/test/test_file.py @@ -7,7 +7,7 @@ from weakref import proxy import io import _pyio as pyio -from test.support import TESTFN, run_unittest +from test.support import TESTFN from collections import UserList class AutoFileTests: @@ -139,7 +139,7 @@ class OtherFileTests: def testModeStrings(self): # check invalid mode strings - for mode in ("", "aU", "wU+"): + for mode in ("", "aU", "wU+", "U+", "+U", "rU+"): try: f = self.open(TESTFN, mode) except ValueError: diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index 784bc92d23..565633fccc 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -22,8 +22,9 @@ except ImportError: from io import BytesIO, StringIO from fileinput import FileInput, hook_encoded -from test.support import verbose, TESTFN, run_unittest, check_warnings +from test.support import verbose, TESTFN, check_warnings from test.support import unlink as safe_unlink +from test import support from unittest import mock @@ -92,7 +93,11 @@ class BufferSizesTests(unittest.TestCase): t2 = writeTmp(2, ["Line %s of file 2\n" % (i+1) for i in range(10)]) t3 = writeTmp(3, ["Line %s of file 3\n" % (i+1) for i in range(5)]) t4 = writeTmp(4, ["Line %s of file 4\n" % (i+1) for i in range(1)]) - self.buffer_size_test(t1, t2, t3, t4, bs, round) + if bs: + with self.assertWarns(DeprecationWarning): + self.buffer_size_test(t1, t2, t3, t4, bs, round) + else: + self.buffer_size_test(t1, t2, t3, t4, bs, round) finally: remove_tempfiles(t1, t2, t3, t4) @@ -940,7 +945,8 @@ class Test_hook_encoded(unittest.TestCase): def test(self): encoding = object() - result = fileinput.hook_encoded(encoding) + errors = object() + result = fileinput.hook_encoded(encoding, errors=errors) fake_open = InvocationRecorder() original_open = builtins.open @@ -958,8 +964,26 @@ class Test_hook_encoded(unittest.TestCase): self.assertIs(args[0], filename) self.assertIs(args[1], mode) self.assertIs(kwargs.pop('encoding'), encoding) + self.assertIs(kwargs.pop('errors'), errors) self.assertFalse(kwargs) + def test_errors(self): + with open(TESTFN, 'wb') as f: + f.write(b'\x80abc') + self.addCleanup(safe_unlink, TESTFN) + + def check(errors, expected_lines): + with FileInput(files=TESTFN, mode='r', + openhook=hook_encoded('utf-8', errors=errors)) as fi: + lines = list(fi) + self.assertEqual(lines, expected_lines) + + check('ignore', ['abc']) + with self.assertRaises(UnicodeDecodeError): + check('strict', ['abc']) + check('replace', ['\ufffdabc']) + check('backslashreplace', ['\\x80abc']) + def test_modes(self): with open(TESTFN, 'wb') as f: # UTF-7 is a convenient, seldom used encoding @@ -981,5 +1005,11 @@ class Test_hook_encoded(unittest.TestCase): check('rb', ['A\n', 'B\r\n', 'C\r', 'D\u20ac']) +class MiscTest(unittest.TestCase): + + def test_all(self): + support.check__all__(self, fileinput) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index cb1f6db8fc..ac8473db50 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -1,6 +1,4 @@ - import fractions -import math import operator import os import random @@ -10,6 +8,8 @@ import time import unittest from test import support +from test.test_grammar import (VALID_UNDERSCORE_LITERALS, + INVALID_UNDERSCORE_LITERALS) from math import isinf, isnan, copysign, ldexp INF = float("inf") @@ -61,6 +61,27 @@ class GeneralFloatCases(unittest.TestCase): float(b'.' + b'1'*1000) float('.' + '1'*1000) + def test_underscores(self): + for lit in VALID_UNDERSCORE_LITERALS: + if not any(ch in lit for ch in 'jJxXoObB'): + self.assertEqual(float(lit), eval(lit)) + self.assertEqual(float(lit), float(lit.replace('_', ''))) + for lit in INVALID_UNDERSCORE_LITERALS: + if lit in ('0_7', '09_99'): # octals are not recognized here + continue + if not any(ch in lit for ch in 'jJxXoObB'): + self.assertRaises(ValueError, float, lit) + # Additional test cases; nan and inf are never valid as literals, + # only in the float() constructor, but we don't allow underscores + # in or around them. + self.assertRaises(ValueError, float, '_NaN') + self.assertRaises(ValueError, float, 'Na_N') + self.assertRaises(ValueError, float, 'IN_F') + self.assertRaises(ValueError, float, '-_INF') + self.assertRaises(ValueError, float, '-INF_') + # Check that we handle bytes values correctly. + self.assertRaises(ValueError, float, b'0_.\xff9') + def test_non_numeric_input_types(self): # Test possible non-numeric types for the argument x, including # subclasses of the explicitly documented accepted types. @@ -162,11 +183,12 @@ class GeneralFloatCases(unittest.TestCase): def __float__(self): return float(str(self)) + 1 - self.assertAlmostEqual(float(Foo1()), 42.) - self.assertAlmostEqual(float(Foo2()), 42.) - self.assertAlmostEqual(float(Foo3(21)), 42.) + self.assertEqual(float(Foo1()), 42.) + self.assertEqual(float(Foo2()), 42.) + with self.assertWarns(DeprecationWarning): + self.assertEqual(float(Foo3(21)), 42.) self.assertRaises(TypeError, float, Foo4(42)) - self.assertAlmostEqual(float(FooStr('8')), 9.) + self.assertEqual(float(FooStr('8')), 9.) class Foo5: def __float__(self): @@ -177,10 +199,14 @@ class GeneralFloatCases(unittest.TestCase): class F: def __float__(self): return OtherFloatSubclass(42.) - self.assertAlmostEqual(float(F()), 42.) - self.assertIs(type(float(F())), OtherFloatSubclass) - self.assertAlmostEqual(FloatSubclass(F()), 42.) - self.assertIs(type(FloatSubclass(F())), FloatSubclass) + with self.assertWarns(DeprecationWarning): + self.assertEqual(float(F()), 42.) + with self.assertWarns(DeprecationWarning): + self.assertIs(type(float(F())), float) + with self.assertWarns(DeprecationWarning): + self.assertEqual(FloatSubclass(F()), 42.) + with self.assertWarns(DeprecationWarning): + self.assertIs(type(FloatSubclass(F())), FloatSubclass) def test_is_integer(self): self.assertFalse((1.1).is_integer()) diff --git a/Lib/test/test_fnmatch.py b/Lib/test/test_fnmatch.py index fa37f90c27..fb7424624b 100644 --- a/Lib/test/test_fnmatch.py +++ b/Lib/test/test_fnmatch.py @@ -62,14 +62,14 @@ class FnmatchTestCase(unittest.TestCase): class TranslateTestCase(unittest.TestCase): def test_translate(self): - self.assertEqual(translate('*'), '.*\Z(?ms)') - self.assertEqual(translate('?'), '.\Z(?ms)') - self.assertEqual(translate('a?b*'), 'a.b.*\Z(?ms)') - self.assertEqual(translate('[abc]'), '[abc]\Z(?ms)') - self.assertEqual(translate('[]]'), '[]]\Z(?ms)') - self.assertEqual(translate('[!x]'), '[^x]\Z(?ms)') - self.assertEqual(translate('[^x]'), '[\\^x]\Z(?ms)') - self.assertEqual(translate('[x'), '\\[x\Z(?ms)') + self.assertEqual(translate('*'), r'(?s:.*)\Z') + self.assertEqual(translate('?'), r'(?s:.)\Z') + self.assertEqual(translate('a?b*'), r'(?s:a.b.*)\Z') + self.assertEqual(translate('[abc]'), r'(?s:[abc])\Z') + self.assertEqual(translate('[]]'), r'(?s:[]])\Z') + self.assertEqual(translate('[!x]'), r'(?s:[^x])\Z') + self.assertEqual(translate('[^x]'), r'(?s:[\^x])\Z') + self.assertEqual(translate('[x'), r'(?s:\[x)\Z') class FilterTestCase(unittest.TestCase): diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 565bb79d7e..83eb29faf8 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -278,7 +278,7 @@ class FormatTest(unittest.TestCase): test_exc('%d', '1', TypeError, "%d format: a number is required, not str") test_exc('%x', '1', TypeError, "%x format: an integer is required, not str") test_exc('%x', 3.14, TypeError, "%x format: an integer is required, not float") - test_exc('%g', '1', TypeError, "a float is required") + test_exc('%g', '1', TypeError, "must be real number, not str") test_exc('no format', '1', TypeError, "not all arguments converted during string formatting") test_exc('%c', -1, OverflowError, "%c arg not in range(0x110000)") @@ -304,6 +304,8 @@ class FormatTest(unittest.TestCase): testcommon(b"%c", 7, b"\x07") testcommon(b"%c", b"Z", b"Z") testcommon(b"%c", bytearray(b"Z"), b"Z") + testcommon(b"%5c", 65, b" A") + testcommon(b"%-5c", 65, b"A ") # %b will insert a series of bytes, either from a type that supports # the Py_buffer protocol, or something that has a __bytes__ method class FakeBytes(object): diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 9df4a54fcb..7905c367ba 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -150,6 +150,7 @@ class FractionTest(unittest.TestCase): self.assertRaises(TypeError, F, "3/2", 3) self.assertRaises(TypeError, F, 3, 0j) self.assertRaises(TypeError, F, 3, 1j) + self.assertRaises(TypeError, F, 1, 2, 3) @requires_IEEE_754 def testInitFromFloat(self): @@ -263,13 +264,13 @@ class FractionTest(unittest.TestCase): nan = inf - inf # bug 16469: error types should be consistent with float -> int self.assertRaisesMessage( - OverflowError, "Cannot convert inf to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_float, inf) self.assertRaisesMessage( - OverflowError, "Cannot convert -inf to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_float, -inf) self.assertRaisesMessage( - ValueError, "Cannot convert nan to Fraction.", + ValueError, "cannot convert NaN to integer ratio", F.from_float, nan) def testFromDecimal(self): @@ -284,16 +285,16 @@ class FractionTest(unittest.TestCase): # bug 16469: error types should be consistent with decimal -> int self.assertRaisesMessage( - OverflowError, "Cannot convert Infinity to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_decimal, Decimal("inf")) self.assertRaisesMessage( - OverflowError, "Cannot convert -Infinity to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_decimal, Decimal("-inf")) self.assertRaisesMessage( - ValueError, "Cannot convert NaN to Fraction.", + ValueError, "cannot convert NaN to integer ratio", F.from_decimal, Decimal("nan")) self.assertRaisesMessage( - ValueError, "Cannot convert sNaN to Fraction.", + ValueError, "cannot convert NaN to integer ratio", F.from_decimal, Decimal("snan")) def testLimitDenominator(self): diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py new file mode 100644 index 0000000000..708ed25b52 --- /dev/null +++ b/Lib/test/test_fstring.py @@ -0,0 +1,760 @@ +import ast +import types +import decimal +import unittest + +a_global = 'global variable' + +# You could argue that I'm too strict in looking for specific error +# values with assertRaisesRegex, but without it it's way too easy to +# make a syntax error in the test strings. Especially with all of the +# triple quotes, raw strings, backslashes, etc. I think it's a +# worthwhile tradeoff. When I switched to this method, I found many +# examples where I wasn't testing what I thought I was. + +class TestCase(unittest.TestCase): + def assertAllRaise(self, exception_type, regex, error_strings): + for str in error_strings: + with self.subTest(str=str): + with self.assertRaisesRegex(exception_type, regex): + eval(str) + + def test__format__lookup(self): + # Make sure __format__ is looked up on the type, not the instance. + class X: + def __format__(self, spec): + return 'class' + + x = X() + + # Add a bound __format__ method to the 'y' instance, but not + # the 'x' instance. + y = X() + y.__format__ = types.MethodType(lambda self, spec: 'instance', y) + + self.assertEqual(f'{y}', format(y)) + self.assertEqual(f'{y}', 'class') + self.assertEqual(format(x), format(y)) + + # __format__ is not called this way, but still make sure it + # returns what we expect (so we can make sure we're bypassing + # it). + self.assertEqual(x.__format__(''), 'class') + self.assertEqual(y.__format__(''), 'instance') + + # This is how __format__ is actually called. + self.assertEqual(type(x).__format__(x, ''), 'class') + self.assertEqual(type(y).__format__(y, ''), 'class') + + def test_ast(self): + # Inspired by http://bugs.python.org/issue24975 + class X: + def __init__(self): + self.called = False + def __call__(self): + self.called = True + return 4 + x = X() + expr = """ +a = 10 +f'{a * x()}'""" + t = ast.parse(expr) + c = compile(t, '', 'exec') + + # Make sure x was not called. + self.assertFalse(x.called) + + # Actually run the code. + exec(c) + + # Make sure x was called. + self.assertTrue(x.called) + + def test_docstring(self): + def f(): + f'''Not a docstring''' + self.assertIsNone(f.__doc__) + def g(): + '''Not a docstring''' \ + f'' + self.assertIsNone(g.__doc__) + + def test_literal_eval(self): + with self.assertRaisesRegex(ValueError, 'malformed node or string'): + ast.literal_eval("f'x'") + + def test_ast_compile_time_concat(self): + x = [''] + + expr = """x[0] = 'foo' f'{3}'""" + t = ast.parse(expr) + c = compile(t, '', 'exec') + exec(c) + self.assertEqual(x[0], 'foo3') + + def test_compile_time_concat_errors(self): + self.assertAllRaise(SyntaxError, + 'cannot mix bytes and nonbytes literals', + [r"""f'' b''""", + r"""b'' f''""", + ]) + + def test_literal(self): + self.assertEqual(f'', '') + self.assertEqual(f'a', 'a') + self.assertEqual(f' ', ' ') + + def test_unterminated_string(self): + self.assertAllRaise(SyntaxError, 'f-string: unterminated string', + [r"""f'{"x'""", + r"""f'{"x}'""", + r"""f'{("x'""", + r"""f'{("x}'""", + ]) + + def test_mismatched_parens(self): + self.assertAllRaise(SyntaxError, 'f-string: mismatched', + ["f'{((}'", + ]) + + def test_double_braces(self): + self.assertEqual(f'{{', '{') + self.assertEqual(f'a{{', 'a{') + self.assertEqual(f'{{b', '{b') + self.assertEqual(f'a{{b', 'a{b') + self.assertEqual(f'}}', '}') + self.assertEqual(f'a}}', 'a}') + self.assertEqual(f'}}b', '}b') + self.assertEqual(f'a}}b', 'a}b') + self.assertEqual(f'{{}}', '{}') + self.assertEqual(f'a{{}}', 'a{}') + self.assertEqual(f'{{b}}', '{b}') + self.assertEqual(f'{{}}c', '{}c') + self.assertEqual(f'a{{b}}', 'a{b}') + self.assertEqual(f'a{{}}c', 'a{}c') + self.assertEqual(f'{{b}}c', '{b}c') + self.assertEqual(f'a{{b}}c', 'a{b}c') + + self.assertEqual(f'{{{10}', '{10') + self.assertEqual(f'}}{10}', '}10') + self.assertEqual(f'}}{{{10}', '}{10') + self.assertEqual(f'}}a{{{10}', '}a{10') + + self.assertEqual(f'{10}{{', '10{') + self.assertEqual(f'{10}}}', '10}') + self.assertEqual(f'{10}}}{{', '10}{') + self.assertEqual(f'{10}}}a{{' '}', '10}a{}') + + # Inside of strings, don't interpret doubled brackets. + self.assertEqual(f'{"{{}}"}', '{{}}') + + self.assertAllRaise(TypeError, 'unhashable type', + ["f'{ {{}} }'", # dict in a set + ]) + + def test_compile_time_concat(self): + x = 'def' + self.assertEqual('abc' f'## {x}ghi', 'abc## defghi') + self.assertEqual('abc' f'{x}' 'ghi', 'abcdefghi') + self.assertEqual('abc' f'{x}' 'gh' f'i{x:4}', 'abcdefghidef ') + self.assertEqual('{x}' f'{x}', '{x}def') + self.assertEqual('{x' f'{x}', '{xdef') + self.assertEqual('{x}' f'{x}', '{x}def') + self.assertEqual('{{x}}' f'{x}', '{{x}}def') + self.assertEqual('{{x' f'{x}', '{{xdef') + self.assertEqual('x}}' f'{x}', 'x}}def') + self.assertEqual(f'{x}' 'x}}', 'defx}}') + self.assertEqual(f'{x}' '', 'def') + self.assertEqual('' f'{x}' '', 'def') + self.assertEqual('' f'{x}', 'def') + self.assertEqual(f'{x}' '2', 'def2') + self.assertEqual('1' f'{x}' '2', '1def2') + self.assertEqual('1' f'{x}', '1def') + self.assertEqual(f'{x}' f'-{x}', 'def-def') + self.assertEqual('' f'', '') + self.assertEqual('' f'' '', '') + self.assertEqual('' f'' '' f'', '') + self.assertEqual(f'', '') + self.assertEqual(f'' '', '') + self.assertEqual(f'' '' f'', '') + self.assertEqual(f'' '' f'' '', '') + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3' f'}'", # can't concat to get a valid f-string + ]) + + def test_comments(self): + # These aren't comments, since they're in strings. + d = {'#': 'hash'} + self.assertEqual(f'{"#"}', '#') + self.assertEqual(f'{d["#"]}', 'hash') + + self.assertAllRaise(SyntaxError, "f-string expression part cannot include '#'", + ["f'{1#}'", # error because the expression becomes "(1#)" + "f'{3(#)}'", + "f'{#}'", + "f'{)#}'", # When wrapped in parens, this becomes + # '()#)'. Make sure that doesn't compile. + ]) + + def test_many_expressions(self): + # Create a string with many expressions in it. Note that + # because we have a space in here as a literal, we're actually + # going to use twice as many ast nodes: one for each literal + # plus one for each expression. + def build_fstr(n, extra=''): + return "f'" + ('{x} ' * n) + extra + "'" + + x = 'X' + width = 1 + + # Test around 256. + for i in range(250, 260): + self.assertEqual(eval(build_fstr(i)), (x+' ')*i) + + # Test concatenating 2 largs fstrings. + self.assertEqual(eval(build_fstr(255)*256), (x+' ')*(255*256)) + + s = build_fstr(253, '{x:{width}} ') + self.assertEqual(eval(s), (x+' ')*254) + + # Test lots of expressions and constants, concatenated. + s = "f'{1}' 'x' 'y'" * 1024 + self.assertEqual(eval(s), '1xy' * 1024) + + def test_format_specifier_expressions(self): + width = 10 + precision = 4 + value = decimal.Decimal('12.34567') + self.assertEqual(f'result: {value:{width}.{precision}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result: 12.35') + self.assertEqual(f'{10:#{1}0x}', ' 0xa') + self.assertEqual(f'{10:{"#"}1{0}{"x"}}', ' 0xa') + self.assertEqual(f'{-10:-{"#"}1{0}x}', ' -0xa') + self.assertEqual(f'{-10:{"-"}#{1}0{"x"}}', ' -0xa') + self.assertEqual(f'{10:#{3 != {4:5} and width}x}', ' 0xa') + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["""f'{"s"!r{":10"}}'""", + + # This looks like a nested format spec. + ]) + + self.assertAllRaise(SyntaxError, "invalid syntax", + [# Invalid syntax inside a nested spec. + "f'{4:{/5}}'", + ]) + + self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply", + [# Can't nest format specifiers. + "f'result: {value:{width:{0}}.{precision:1}}'", + ]) + + self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character', + [# No expansion inside conversion or for + # the : or ! itself. + """f'{"s"!{"r"}}'""", + ]) + + def test_side_effect_order(self): + class X: + def __init__(self): + self.i = 0 + def __format__(self, spec): + self.i += 1 + return str(self.i) + + x = X() + self.assertEqual(f'{x} {x}', '1 2') + + def test_missing_expression(self): + self.assertAllRaise(SyntaxError, 'f-string: empty expression not allowed', + ["f'{}'", + "f'{ }'" + "f' {} '", + "f'{!r}'", + "f'{ !r}'", + "f'{10:{ }}'", + "f' { } '", + + # Catch the empty expression before the + # invalid conversion. + "f'{!x}'", + "f'{ !xr}'", + "f'{!x:}'", + "f'{!x:a}'", + "f'{ !xr:}'", + "f'{ !xr:a}'", + + "f'{!}'", + "f'{:}'", + + # We find the empty expression before the + # missing closing brace. + "f'{!'", + "f'{!s:'", + "f'{:'", + "f'{:x'", + ]) + + def test_parens_in_expressions(self): + self.assertEqual(f'{3,}', '(3,)') + + # Add these because when an expression is evaluated, parens + # are added around it. But we shouldn't go from an invalid + # expression to a valid one. The added parens are just + # supposed to allow whitespace (including newlines). + self.assertAllRaise(SyntaxError, 'invalid syntax', + ["f'{,}'", + "f'{,}'", # this is (,), which is an error + ]) + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3)+(4}'", + ]) + + self.assertAllRaise(SyntaxError, 'EOL while scanning string literal', + ["f'{\n}'", + ]) + + def test_backslashes_in_string_part(self): + self.assertEqual(f'\t', '\t') + self.assertEqual(r'\t', '\\t') + self.assertEqual(rf'\t', '\\t') + self.assertEqual(f'{2}\t', '2\t') + self.assertEqual(f'{2}\t{3}', '2\t3') + self.assertEqual(f'\t{3}', '\t3') + + self.assertEqual(f'\u0394', '\u0394') + self.assertEqual(r'\u0394', '\\u0394') + self.assertEqual(rf'\u0394', '\\u0394') + self.assertEqual(f'{2}\u0394', '2\u0394') + self.assertEqual(f'{2}\u0394{3}', '2\u03943') + self.assertEqual(f'\u0394{3}', '\u03943') + + self.assertEqual(f'\U00000394', '\u0394') + self.assertEqual(r'\U00000394', '\\U00000394') + self.assertEqual(rf'\U00000394', '\\U00000394') + self.assertEqual(f'{2}\U00000394', '2\u0394') + self.assertEqual(f'{2}\U00000394{3}', '2\u03943') + self.assertEqual(f'\U00000394{3}', '\u03943') + + self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}', '\u0394') + self.assertEqual(f'{2}\N{GREEK CAPITAL LETTER DELTA}', '2\u0394') + self.assertEqual(f'{2}\N{GREEK CAPITAL LETTER DELTA}{3}', '2\u03943') + self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}{3}', '\u03943') + self.assertEqual(f'2\N{GREEK CAPITAL LETTER DELTA}', '2\u0394') + self.assertEqual(f'2\N{GREEK CAPITAL LETTER DELTA}3', '2\u03943') + self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}3', '\u03943') + + self.assertEqual(f'\x20', ' ') + self.assertEqual(r'\x20', '\\x20') + self.assertEqual(rf'\x20', '\\x20') + self.assertEqual(f'{2}\x20', '2 ') + self.assertEqual(f'{2}\x20{3}', '2 3') + self.assertEqual(f'\x20{3}', ' 3') + + self.assertEqual(f'2\x20', '2 ') + self.assertEqual(f'2\x203', '2 3') + self.assertEqual(f'\x203', ' 3') + + def test_misformed_unicode_character_name(self): + # These test are needed because unicode names are parsed + # differently inside f-strings. + self.assertAllRaise(SyntaxError, r"\(unicode error\) 'unicodeescape' codec can't decode bytes in position .*: malformed \\N character escape", + [r"f'\N'", + r"f'\N{'", + r"f'\N{GREEK CAPITAL LETTER DELTA'", + + # Here are the non-f-string versions, + # which should give the same errors. + r"'\N'", + r"'\N{'", + r"'\N{GREEK CAPITAL LETTER DELTA'", + ]) + + def test_no_backslashes_in_expression_part(self): + self.assertAllRaise(SyntaxError, 'f-string expression part cannot include a backslash', + [r"f'{\'a\'}'", + r"f'{\t3}'", + r"f'{\}'", + r"rf'{\'a\'}'", + r"rf'{\t3}'", + r"rf'{\}'", + r"""rf'{"\N{LEFT CURLY BRACKET}"}'""", + r"f'{\n}'", + ]) + + def test_no_escapes_for_braces(self): + """ + Only literal curly braces begin an expression. + """ + # \x7b is '{'. + self.assertEqual(f'\x7b1+1}}', '{1+1}') + self.assertEqual(f'\x7b1+1', '{1+1') + self.assertEqual(f'\u007b1+1', '{1+1') + self.assertEqual(f'\N{LEFT CURLY BRACKET}1+1\N{RIGHT CURLY BRACKET}', '{1+1}') + + def test_newlines_in_expressions(self): + self.assertEqual(f'{0}', '0') + self.assertEqual(rf'''{3+ +4}''', '7') + + def test_lambda(self): + x = 5 + self.assertEqual(f'{(lambda y:x*y)("8")!r}', "'88888'") + self.assertEqual(f'{(lambda y:x*y)("8")!r:10}', "'88888' ") + self.assertEqual(f'{(lambda y:x*y)("8"):10}', "88888 ") + + # lambda doesn't work without parens, because the colon + # makes the parser think it's a format_spec + self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', + ["f'{lambda x:x}'", + ]) + + def test_yield(self): + # Not terribly useful, but make sure the yield turns + # a function into a generator + def fn(y): + f'y:{yield y*2}' + + g = fn(4) + self.assertEqual(next(g), 8) + + def test_yield_send(self): + def fn(x): + yield f'x:{yield (lambda i: x * i)}' + + g = fn(10) + the_lambda = next(g) + self.assertEqual(the_lambda(4), 40) + self.assertEqual(g.send('string'), 'x:string') + + def test_expressions_with_triple_quoted_strings(self): + self.assertEqual(f"{'''x'''}", 'x') + self.assertEqual(f"{'''eric's'''}", "eric's") + + # Test concatenation within an expression + self.assertEqual(f'{"x" """eric"s""" "y"}', 'xeric"sy') + self.assertEqual(f'{"x" """eric"s"""}', 'xeric"s') + self.assertEqual(f'{"""eric"s""" "y"}', 'eric"sy') + self.assertEqual(f'{"""x""" """eric"s""" "y"}', 'xeric"sy') + self.assertEqual(f'{"""x""" """eric"s""" """y"""}', 'xeric"sy') + self.assertEqual(f'{r"""x""" """eric"s""" """y"""}', 'xeric"sy') + + def test_multiple_vars(self): + x = 98 + y = 'abc' + self.assertEqual(f'{x}{y}', '98abc') + + self.assertEqual(f'X{x}{y}', 'X98abc') + self.assertEqual(f'{x}X{y}', '98Xabc') + self.assertEqual(f'{x}{y}X', '98abcX') + + self.assertEqual(f'X{x}Y{y}', 'X98Yabc') + self.assertEqual(f'X{x}{y}Y', 'X98abcY') + self.assertEqual(f'{x}X{y}Y', '98XabcY') + + self.assertEqual(f'X{x}Y{y}Z', 'X98YabcZ') + + def test_closure(self): + def outer(x): + def inner(): + return f'x:{x}' + return inner + + self.assertEqual(outer('987')(), 'x:987') + self.assertEqual(outer(7)(), 'x:7') + + def test_arguments(self): + y = 2 + def f(x, width): + return f'x={x*y:{width}}' + + self.assertEqual(f('foo', 10), 'x=foofoo ') + x = 'bar' + self.assertEqual(f(10, 10), 'x= 20') + + def test_locals(self): + value = 123 + self.assertEqual(f'v:{value}', 'v:123') + + def test_missing_variable(self): + with self.assertRaises(NameError): + f'v:{value}' + + def test_missing_format_spec(self): + class O: + def __format__(self, spec): + if not spec: + return '*' + return spec + + self.assertEqual(f'{O():x}', 'x') + self.assertEqual(f'{O()}', '*') + self.assertEqual(f'{O():}', '*') + + self.assertEqual(f'{3:}', '3') + self.assertEqual(f'{3!s:}', '3') + + def test_global(self): + self.assertEqual(f'g:{a_global}', 'g:global variable') + self.assertEqual(f'g:{a_global!r}', "g:'global variable'") + + a_local = 'local variable' + self.assertEqual(f'g:{a_global} l:{a_local}', + 'g:global variable l:local variable') + self.assertEqual(f'g:{a_global!r}', + "g:'global variable'") + self.assertEqual(f'g:{a_global} l:{a_local!r}', + "g:global variable l:'local variable'") + + self.assertIn("module 'unittest' from", f'{unittest}') + + def test_shadowed_global(self): + a_global = 'really a local' + self.assertEqual(f'g:{a_global}', 'g:really a local') + self.assertEqual(f'g:{a_global!r}', "g:'really a local'") + + a_local = 'local variable' + self.assertEqual(f'g:{a_global} l:{a_local}', + 'g:really a local l:local variable') + self.assertEqual(f'g:{a_global!r}', + "g:'really a local'") + self.assertEqual(f'g:{a_global} l:{a_local!r}', + "g:really a local l:'local variable'") + + def test_call(self): + def foo(x): + return 'x=' + str(x) + + self.assertEqual(f'{foo(10)}', 'x=10') + + def test_nested_fstrings(self): + y = 5 + self.assertEqual(f'{f"{0}"*3}', '000') + self.assertEqual(f'{f"{y}"*3}', '555') + + def test_invalid_string_prefixes(self): + self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', + ["fu''", + "uf''", + "Fu''", + "fU''", + "Uf''", + "uF''", + "ufr''", + "urf''", + "fur''", + "fru''", + "rfu''", + "ruf''", + "FUR''", + "Fur''", + "fb''", + "fB''", + "Fb''", + "FB''", + "bf''", + "bF''", + "Bf''", + "BF''", + ]) + + def test_leading_trailing_spaces(self): + self.assertEqual(f'{ 3}', '3') + self.assertEqual(f'{ 3}', '3') + self.assertEqual(f'{3 }', '3') + self.assertEqual(f'{3 }', '3') + + self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]}}', + 'expr={1: 2}') + self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]} }', + 'expr={1: 2}') + + def test_not_equal(self): + # There's a special test for this because there's a special + # case in the f-string parser to look for != as not ending an + # expression. Normally it would, while looking for !s or !r. + + self.assertEqual(f'{3!=4}', 'True') + self.assertEqual(f'{3!=4:}', 'True') + self.assertEqual(f'{3!=4!s}', 'True') + self.assertEqual(f'{3!=4!s:.3}', 'Tru') + + def test_conversions(self): + self.assertEqual(f'{3.14:10.10}', ' 3.14') + self.assertEqual(f'{3.14!s:10.10}', '3.14 ') + self.assertEqual(f'{3.14!r:10.10}', '3.14 ') + self.assertEqual(f'{3.14!a:10.10}', '3.14 ') + + self.assertEqual(f'{"a"}', 'a') + self.assertEqual(f'{"a"!r}', "'a'") + self.assertEqual(f'{"a"!a}', "'a'") + + # Not a conversion. + self.assertEqual(f'{"a!r"}', "a!r") + + # Not a conversion, but show that ! is allowed in a format spec. + self.assertEqual(f'{3.14:!<10.10}', '3.14!!!!!!') + + self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character', + ["f'{3!g}'", + "f'{3!A}'", + "f'{3!3}'", + "f'{3!G}'", + "f'{3!!}'", + "f'{3!:}'", + "f'{3! s}'", # no space before conversion char + ]) + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{x!s{y}}'", + "f'{3!ss}'", + "f'{3!ss:}'", + "f'{3!ss:s}'", + ]) + + def test_assignment(self): + self.assertAllRaise(SyntaxError, 'invalid syntax', + ["f'' = 3", + "f'{0}' = x", + "f'{x}' = x", + ]) + + def test_del(self): + self.assertAllRaise(SyntaxError, 'invalid syntax', + ["del f''", + "del '' f''", + ]) + + def test_mismatched_braces(self): + self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed", + ["f'{{}'", + "f'{{}}}'", + "f'}'", + "f'x}'", + "f'x}x'", + r"f'\u007b}'", + + # Can't have { or } in a format spec. + "f'{3:}>10}'", + "f'{3:}}>10}'", + ]) + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3:{{>10}'", + "f'{3'", + "f'{3!'", + "f'{3:'", + "f'{3!s'", + "f'{3!s:'", + "f'{3!s:3'", + "f'x{'", + "f'x{x'", + "f'{x'", + "f'{3:s'", + "f'{{{'", + "f'{{}}{'", + "f'{'", + ]) + + # But these are just normal strings. + self.assertEqual(f'{"{"}', '{') + self.assertEqual(f'{"}"}', '}') + self.assertEqual(f'{3:{"}"}>10}', '}}}}}}}}}3') + self.assertEqual(f'{2:{"{"}>10}', '{{{{{{{{{2') + + def test_if_conditional(self): + # There's special logic in compile.c to test if the + # conditional for an if (and while) are constants. Exercise + # that code. + + def test_fstring(x, expected): + flag = 0 + if f'{x}': + flag = 1 + else: + flag = 2 + self.assertEqual(flag, expected) + + def test_concat_empty(x, expected): + flag = 0 + if '' f'{x}': + flag = 1 + else: + flag = 2 + self.assertEqual(flag, expected) + + def test_concat_non_empty(x, expected): + flag = 0 + if ' ' f'{x}': + flag = 1 + else: + flag = 2 + self.assertEqual(flag, expected) + + test_fstring('', 2) + test_fstring(' ', 1) + + test_concat_empty('', 2) + test_concat_empty(' ', 1) + + test_concat_non_empty('', 1) + test_concat_non_empty(' ', 1) + + def test_empty_format_specifier(self): + x = 'test' + self.assertEqual(f'{x}', 'test') + self.assertEqual(f'{x:}', 'test') + self.assertEqual(f'{x!s:}', 'test') + self.assertEqual(f'{x!r:}', "'test'") + + def test_str_format_differences(self): + d = {'a': 'string', + 0: 'integer', + } + a = 0 + self.assertEqual(f'{d[0]}', 'integer') + self.assertEqual(f'{d["a"]}', 'string') + self.assertEqual(f'{d[a]}', 'integer') + self.assertEqual('{d[a]}'.format(d=d), 'string') + self.assertEqual('{d[0]}'.format(d=d), 'integer') + + def test_invalid_expressions(self): + self.assertAllRaise(SyntaxError, 'invalid syntax', + [r"f'{a[4)}'", + r"f'{a(4]}'", + ]) + + def test_errors(self): + # see issue 26287 + self.assertAllRaise(TypeError, 'unsupported', + [r"f'{(lambda: 0):x}'", + r"f'{(0,):x}'", + ]) + self.assertAllRaise(ValueError, 'Unknown format code', + [r"f'{1000:j}'", + r"f'{1000:j}'", + ]) + + def test_loop(self): + for i in range(1000): + self.assertEqual(f'i:{i}', 'i:' + str(i)) + + def test_dict(self): + d = {'"': 'dquote', + "'": 'squote', + 'foo': 'bar', + } + self.assertEqual(f'''{d["'"]}''', 'squote') + self.assertEqual(f"""{d['"']}""", 'dquote') + + self.assertEqual(f'{d["foo"]}', 'bar') + self.assertEqual(f"{d['foo']}", 'bar') + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index aef66da98a..12fabc5e8b 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -311,10 +311,12 @@ if ssl is not None: _ssl_closing = False def secure_connection(self): - socket = ssl.wrap_socket(self.socket, suppress_ragged_eofs=False, - certfile=CERTFILE, server_side=True, - do_handshake_on_connect=False, - ssl_version=ssl.PROTOCOL_SSLv23) + context = ssl.SSLContext() + context.load_cert_chain(CERTFILE) + socket = context.wrap_socket(self.socket, + suppress_ragged_eofs=False, + server_side=True, + do_handshake_on_connect=False) self.del_channel() self.set_socket(socket) self._ssl_accepting = True @@ -1049,10 +1051,19 @@ class TestTimeouts(TestCase): ftp.close() +class MiscTestCase(TestCase): + def test__all__(self): + blacklist = {'MSG_OOB', 'FTP_PORT', 'MAXLINE', 'CRLF', 'B_CRLF', + 'Error', 'parse150', 'parse227', 'parse229', 'parse257', + 'print_line', 'ftpcp', 'test'} + support.check__all__(self, ftplib, blacklist=blacklist) + + def test_main(): tests = [TestFTPClass, TestTimeouts, TestIPv6Environment, - TestTLS_FTPClassMixin, TestTLS_FTPClass] + TestTLS_FTPClassMixin, TestTLS_FTPClass, + MiscTestCase] thread_info = support.threading_setup() try: diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 9ccd0cafc4..824549b803 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -10,6 +10,7 @@ from test import support import time import unittest from weakref import proxy +import contextlib try: import threading except ImportError: @@ -22,6 +23,14 @@ c_functools = support.import_fresh_module('functools', fresh=['_functools']) decimal = support.import_fresh_module('decimal', fresh=['_decimal']) +@contextlib.contextmanager +def replaced_module(name, replacement): + original_module = sys.modules[name] + sys.modules[name] = replacement + try: + yield + finally: + sys.modules[name] = original_module def capture(*args, **kw): """capture all positional and keyword arguments""" @@ -169,58 +178,35 @@ class TestPartial: p2.new_attr = 'spam' self.assertEqual(p2.new_attr, 'spam') - -@unittest.skipUnless(c_functools, 'requires the C _functools module') -class TestPartialC(TestPartial, unittest.TestCase): - if c_functools: - partial = c_functools.partial - - def test_attributes_unwritable(self): - # attributes should not be writable - p = self.partial(capture, 1, 2, a=10, b=20) - self.assertRaises(AttributeError, setattr, p, 'func', map) - self.assertRaises(AttributeError, setattr, p, 'args', (1, 2)) - self.assertRaises(AttributeError, setattr, p, 'keywords', dict(a=1, b=2)) - - p = self.partial(hex) - try: - del p.__dict__ - except TypeError: - pass - else: - self.fail('partial object allowed __dict__ to be deleted') - def test_repr(self): args = (object(), object()) args_repr = ', '.join(repr(a) for a in args) kwargs = {'a': object(), 'b': object()} kwargs_reprs = ['a={a!r}, b={b!r}'.format_map(kwargs), 'b={b!r}, a={a!r}'.format_map(kwargs)] - if self.partial is c_functools.partial: + if self.partial in (c_functools.partial, py_functools.partial): name = 'functools.partial' else: name = self.partial.__name__ f = self.partial(capture) - self.assertEqual('{}({!r})'.format(name, capture), - repr(f)) + self.assertEqual(f'{name}({capture!r})', repr(f)) f = self.partial(capture, *args) - self.assertEqual('{}({!r}, {})'.format(name, capture, args_repr), - repr(f)) + self.assertEqual(f'{name}({capture!r}, {args_repr})', repr(f)) f = self.partial(capture, **kwargs) self.assertIn(repr(f), - ['{}({!r}, {})'.format(name, capture, kwargs_repr) + [f'{name}({capture!r}, {kwargs_repr})' for kwargs_repr in kwargs_reprs]) f = self.partial(capture, *args, **kwargs) self.assertIn(repr(f), - ['{}({!r}, {}, {})'.format(name, capture, args_repr, kwargs_repr) + [f'{name}({capture!r}, {args_repr}, {kwargs_repr})' for kwargs_repr in kwargs_reprs]) def test_recursive_repr(self): - if self.partial is c_functools.partial: + if self.partial in (c_functools.partial, py_functools.partial): name = 'functools.partial' else: name = self.partial.__name__ @@ -228,30 +214,31 @@ class TestPartialC(TestPartial, unittest.TestCase): f = self.partial(capture) f.__setstate__((f, (), {}, {})) try: - self.assertEqual(repr(f), '%s(%s(...))' % (name, name)) + self.assertEqual(repr(f), '%s(...)' % (name,)) finally: f.__setstate__((capture, (), {}, {})) f = self.partial(capture) f.__setstate__((capture, (f,), {}, {})) try: - self.assertEqual(repr(f), '%s(%r, %s(...))' % (name, capture, name)) + self.assertEqual(repr(f), '%s(%r, ...)' % (name, capture,)) finally: f.__setstate__((capture, (), {}, {})) f = self.partial(capture) f.__setstate__((capture, (), {'a': f}, {})) try: - self.assertEqual(repr(f), '%s(%r, a=%s(...))' % (name, capture, name)) + self.assertEqual(repr(f), '%s(%r, a=...)' % (name, capture,)) finally: f.__setstate__((capture, (), {}, {})) def test_pickle(self): - f = self.partial(signature, ['asdf'], bar=[True]) - f.attr = [] - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - f_copy = pickle.loads(pickle.dumps(f, proto)) - self.assertEqual(signature(f_copy), signature(f)) + with self.AllowPickle(): + f = self.partial(signature, ['asdf'], bar=[True]) + f.attr = [] + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f_copy = pickle.loads(pickle.dumps(f, proto)) + self.assertEqual(signature(f_copy), signature(f)) def test_copy(self): f = self.partial(signature, ['asdf'], bar=[True]) @@ -276,11 +263,13 @@ class TestPartialC(TestPartial, unittest.TestCase): def test_setstate(self): f = self.partial(signature) f.__setstate__((capture, (1,), dict(a=10), dict(attr=[]))) + self.assertEqual(signature(f), (capture, (1,), dict(a=10), dict(attr=[]))) self.assertEqual(f(2, b=20), ((1, 2), {'a': 10, 'b': 20})) f.__setstate__((capture, (1,), dict(a=10), None)) + self.assertEqual(signature(f), (capture, (1,), dict(a=10), {})) self.assertEqual(f(2, b=20), ((1, 2), {'a': 10, 'b': 20})) @@ -327,38 +316,39 @@ class TestPartialC(TestPartial, unittest.TestCase): self.assertIs(type(r[0]), tuple) def test_recursive_pickle(self): - f = self.partial(capture) - f.__setstate__((f, (), {}, {})) - try: - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(RecursionError): - pickle.dumps(f, proto) - finally: - f.__setstate__((capture, (), {}, {})) - - f = self.partial(capture) - f.__setstate__((capture, (f,), {}, {})) - try: - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - f_copy = pickle.loads(pickle.dumps(f, proto)) - try: - self.assertIs(f_copy.args[0], f_copy) - finally: - f_copy.__setstate__((capture, (), {}, {})) - finally: - f.__setstate__((capture, (), {}, {})) - - f = self.partial(capture) - f.__setstate__((capture, (), {'a': f}, {})) - try: - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - f_copy = pickle.loads(pickle.dumps(f, proto)) - try: - self.assertIs(f_copy.keywords['a'], f_copy) - finally: - f_copy.__setstate__((capture, (), {}, {})) - finally: - f.__setstate__((capture, (), {}, {})) + with self.AllowPickle(): + f = self.partial(capture) + f.__setstate__((f, (), {}, {})) + try: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(RecursionError): + pickle.dumps(f, proto) + finally: + f.__setstate__((capture, (), {}, {})) + + f = self.partial(capture) + f.__setstate__((capture, (f,), {}, {})) + try: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f_copy = pickle.loads(pickle.dumps(f, proto)) + try: + self.assertIs(f_copy.args[0], f_copy) + finally: + f_copy.__setstate__((capture, (), {}, {})) + finally: + f.__setstate__((capture, (), {}, {})) + + f = self.partial(capture) + f.__setstate__((capture, (), {'a': f}, {})) + try: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f_copy = pickle.loads(pickle.dumps(f, proto)) + try: + self.assertIs(f_copy.keywords['a'], f_copy) + finally: + f_copy.__setstate__((capture, (), {}, {})) + finally: + f.__setstate__((capture, (), {}, {})) # Issue 6083: Reference counting bug def test_setstate_refcount(self): @@ -377,24 +367,60 @@ class TestPartialC(TestPartial, unittest.TestCase): f = self.partial(object) self.assertRaises(TypeError, f.__setstate__, BadSequence()) +@unittest.skipUnless(c_functools, 'requires the C _functools module') +class TestPartialC(TestPartial, unittest.TestCase): + if c_functools: + partial = c_functools.partial + + class AllowPickle: + def __enter__(self): + return self + def __exit__(self, type, value, tb): + return False + + def test_attributes_unwritable(self): + # attributes should not be writable + p = self.partial(capture, 1, 2, a=10, b=20) + self.assertRaises(AttributeError, setattr, p, 'func', map) + self.assertRaises(AttributeError, setattr, p, 'args', (1, 2)) + self.assertRaises(AttributeError, setattr, p, 'keywords', dict(a=1, b=2)) + + p = self.partial(hex) + try: + del p.__dict__ + except TypeError: + pass + else: + self.fail('partial object allowed __dict__ to be deleted') class TestPartialPy(TestPartial, unittest.TestCase): - partial = staticmethod(py_functools.partial) + partial = py_functools.partial + class AllowPickle: + def __init__(self): + self._cm = replaced_module("functools", py_functools) + def __enter__(self): + return self._cm.__enter__() + def __exit__(self, type, value, tb): + return self._cm.__exit__(type, value, tb) if c_functools: - class PartialSubclass(c_functools.partial): + class CPartialSubclass(c_functools.partial): pass +class PyPartialSubclass(py_functools.partial): + pass @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestPartialCSubclass(TestPartialC): if c_functools: - partial = PartialSubclass + partial = CPartialSubclass # partial subclasses are not optimized for nested calls test_nested_optimization = None +class TestPartialPySubclass(TestPartialPy): + partial = PyPartialSubclass class TestPartialMethod(unittest.TestCase): @@ -685,9 +711,10 @@ class TestWraps(TestUpdateWrapper): self.assertEqual(wrapper.attr, 'This is a different test') self.assertEqual(wrapper.dict_attr, f.dict_attr) - +@unittest.skipUnless(c_functools, 'requires the C _functools module') class TestReduce(unittest.TestCase): - func = functools.reduce + if c_functools: + func = c_functools.reduce def test_reduce(self): class Squares: @@ -1280,6 +1307,16 @@ class TestLRU: self.assertEqual(fib.cache_info(), self.module._CacheInfo(hits=0, misses=0, maxsize=None, currsize=0)) + def test_kwargs_order(self): + # PEP 468: Preserving Keyword Argument Order + @self.module.lru_cache(maxsize=10) + def f(**kwargs): + return list(kwargs.items()) + self.assertEqual(f(a=1, b=2), [('a', 1), ('b', 2)]) + self.assertEqual(f(b=2, a=1), [('b', 2), ('a', 1)]) + self.assertEqual(f.cache_info(), + self.module._CacheInfo(hits=0, misses=2, maxsize=10, currsize=2)) + def test_lru_cache_decoration(self): def f(zomg: 'zomg_annotation'): """f doc string""" @@ -1309,7 +1346,7 @@ class TestLRU: f.cache_clear() orig_si = sys.getswitchinterval() - sys.setswitchinterval(1e-6) + support.setswitchinterval(1e-6) try: # create n threads in order to fill cache threads = [threading.Thread(target=full, args=[k]) @@ -1595,13 +1632,15 @@ class TestSingleDispatch(unittest.TestCase): bases = [c.Sequence, c.MutableMapping, c.Mapping, c.Set] for haystack in permutations(bases): m = mro(dict, haystack) - self.assertEqual(m, [dict, c.MutableMapping, c.Mapping, c.Sized, - c.Iterable, c.Container, object]) + self.assertEqual(m, [dict, c.MutableMapping, c.Mapping, + c.Collection, c.Sized, c.Iterable, + c.Container, object]) bases = [c.Container, c.Mapping, c.MutableMapping, c.OrderedDict] for haystack in permutations(bases): m = mro(c.ChainMap, haystack) self.assertEqual(m, [c.ChainMap, c.MutableMapping, c.Mapping, - c.Sized, c.Iterable, c.Container, object]) + c.Collection, c.Sized, c.Iterable, + c.Container, object]) # If there's a generic function with implementations registered for # both Sized and Container, passing a defaultdict to it results in an @@ -1622,9 +1661,9 @@ class TestSingleDispatch(unittest.TestCase): bases = [c.MutableSequence, c.MutableMapping] for haystack in permutations(bases): m = mro(D, bases) - self.assertEqual(m, [D, c.MutableSequence, c.Sequence, - c.defaultdict, dict, c.MutableMapping, - c.Mapping, c.Sized, c.Iterable, c.Container, + self.assertEqual(m, [D, c.MutableSequence, c.Sequence, c.Reversible, + c.defaultdict, dict, c.MutableMapping, c.Mapping, + c.Collection, c.Sized, c.Iterable, c.Container, object]) # Container and Callable are registered on different base classes and @@ -1637,7 +1676,8 @@ class TestSingleDispatch(unittest.TestCase): for haystack in permutations(bases): m = mro(C, haystack) self.assertEqual(m, [C, c.Callable, c.defaultdict, dict, c.Mapping, - c.Sized, c.Iterable, c.Container, object]) + c.Collection, c.Sized, c.Iterable, + c.Container, object]) def test_register_abc(self): c = collections diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index 7b072ad897..2f1c410b1b 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -5,7 +5,7 @@ from test import support import os import re -rx = re.compile('\((\S+).py, line (\d+)') +rx = re.compile(r'\((\S+).py, line (\d+)') def get_error_location(msg): mo = rx.search(str(msg)) diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index a4d684b40e..e7274998d5 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -684,7 +684,6 @@ class GCTests(unittest.TestCase): # Create a reference cycle through the __main__ module and check # it gets collected at interpreter shutdown. code = """if 1: - import weakref class C: def __del__(self): print('__del__ called') @@ -699,7 +698,6 @@ class GCTests(unittest.TestCase): # Same as above, but with a non-__main__ module. with temp_dir() as script_dir: module = """if 1: - import weakref class C: def __del__(self): print('__del__ called') diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 3db10b664e..60f1d92846 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -5,7 +5,6 @@ import os import re -import pprint import subprocess import sys import sysconfig @@ -111,6 +110,7 @@ HAS_PYUP_PYDOWN = gdb_has_frame_select() BREAKPOINT_FN='builtin_id' +@unittest.skipIf(support.PGO, "not useful for PGO") class DebuggerTests(unittest.TestCase): """Test that the debugger can debug Python.""" @@ -177,6 +177,7 @@ class DebuggerTests(unittest.TestCase): args = ['--eval-command=%s' % cmd for cmd in commands] args += ["--args", sys.executable] + args.extend(subprocess._args_from_interpreter_flags()) if not import_site: # -S suppresses the default 'import site' @@ -240,7 +241,7 @@ class DebuggerTests(unittest.TestCase): # gdb can insert additional '\n' and space characters in various places # in its output, depending on the width of the terminal it's connected # to (using its "wrap_here" function) - m = re.match('.*#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)\)\s+at\s+\S*Python/bltinmodule.c.*', + m = re.match(r'.*#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)\)\s+at\s+\S*Python/bltinmodule.c.*', gdb_output, re.DOTALL) if not m: self.fail('Unexpected gdb output: %r\n%s' % (gdb_output, gdb_output)) @@ -292,7 +293,8 @@ class PrettyPrintTests(DebuggerTests): 'Verify the pretty-printing of dictionaries' self.assertGdbRepr({}) self.assertGdbRepr({'foo': 'bar'}, "{'foo': 'bar'}") - self.assertGdbRepr({'foo': 'bar', 'douglas': 42}, "{'douglas': 42, 'foo': 'bar'}") + # Python preserves insertion order since 3.6 + self.assertGdbRepr({'foo': 'bar', 'douglas': 42}, "{'foo': 'bar', 'douglas': 42}") def test_lists(self): 'Verify the pretty-printing of lists' @@ -355,9 +357,12 @@ class PrettyPrintTests(DebuggerTests): 'Verify the pretty-printing of sets' if (gdb_major_version, gdb_minor_version) < (7, 3): self.skipTest("pretty-printing of sets needs gdb 7.3 or later") - self.assertGdbRepr(set(), 'set()') - self.assertGdbRepr(set(['a', 'b']), "{'a', 'b'}") - self.assertGdbRepr(set([4, 5, 6]), "{4, 5, 6}") + self.assertGdbRepr(set(), "set()") + self.assertGdbRepr(set(['a']), "{'a'}") + # PYTHONHASHSEED is need to get the exact frozenset item order + if not sys.flags.ignore_environment: + self.assertGdbRepr(set(['a', 'b']), "{'a', 'b'}") + self.assertGdbRepr(set([4, 5, 6]), "{4, 5, 6}") # Ensure that we handle sets containing the "dummy" key value, # which happens on deletion: @@ -370,9 +375,12 @@ id(s)''') 'Verify the pretty-printing of frozensets' if (gdb_major_version, gdb_minor_version) < (7, 3): self.skipTest("pretty-printing of frozensets needs gdb 7.3 or later") - self.assertGdbRepr(frozenset(), 'frozenset()') - self.assertGdbRepr(frozenset(['a', 'b']), "frozenset({'a', 'b'})") - self.assertGdbRepr(frozenset([4, 5, 6]), "frozenset({4, 5, 6})") + self.assertGdbRepr(frozenset(), "frozenset()") + self.assertGdbRepr(frozenset(['a']), "frozenset({'a'})") + # PYTHONHASHSEED is need to get the exact frozenset item order + if not sys.flags.ignore_environment: + self.assertGdbRepr(frozenset(['a', 'b']), "frozenset({'a', 'b'})") + self.assertGdbRepr(frozenset([4, 5, 6]), "frozenset({4, 5, 6})") def test_exceptions(self): # Test a RuntimeError @@ -501,6 +509,10 @@ id(foo)''') def test_builtins_help(self): 'Ensure that the new-style class _Helper in site.py can be handled' + + if sys.flags.no_site: + self.skipTest("need site module, but -S option was used") + # (this was the issue causing tracebacks in # http://bugs.python.org/issue8032#msg100537 ) gdb_repr, gdb_output = self.get_gdb_repr('id(__builtins__.help)', import_site=True) @@ -536,7 +548,7 @@ class Foo: foo = Foo() foo.an_attr = foo id(foo)''') - self.assertTrue(re.match('<Foo\(an_attr=<\.\.\.>\) at remote 0x-?[0-9a-f]+>', + self.assertTrue(re.match(r'<Foo\(an_attr=<\.\.\.>\) at remote 0x-?[0-9a-f]+>', gdb_repr), 'Unexpected gdb representation: %r\n%s' % \ (gdb_repr, gdb_output)) @@ -549,7 +561,7 @@ class Foo(object): foo = Foo() foo.an_attr = foo id(foo)''') - self.assertTrue(re.match('<Foo\(an_attr=<\.\.\.>\) at remote 0x-?[0-9a-f]+>', + self.assertTrue(re.match(r'<Foo\(an_attr=<\.\.\.>\) at remote 0x-?[0-9a-f]+>', gdb_repr), 'Unexpected gdb representation: %r\n%s' % \ (gdb_repr, gdb_output)) @@ -563,7 +575,7 @@ b = Foo() a.an_attr = b b.an_attr = a id(a)''') - self.assertTrue(re.match('<Foo\(an_attr=<Foo\(an_attr=<\.\.\.>\) at remote 0x-?[0-9a-f]+>\) at remote 0x-?[0-9a-f]+>', + self.assertTrue(re.match(r'<Foo\(an_attr=<Foo\(an_attr=<\.\.\.>\) at remote 0x-?[0-9a-f]+>\) at remote 0x-?[0-9a-f]+>', gdb_repr), 'Unexpected gdb representation: %r\n%s' % \ (gdb_repr, gdb_output)) @@ -598,7 +610,7 @@ id(a)''') def test_builtin_method(self): gdb_repr, gdb_output = self.get_gdb_repr('import sys; id(sys.stdout.readlines)') - self.assertTrue(re.match('<built-in method readlines of _io.TextIOWrapper object at remote 0x-?[0-9a-f]+>', + self.assertTrue(re.match(r'<built-in method readlines of _io.TextIOWrapper object at remote 0x-?[0-9a-f]+>', gdb_repr), 'Unexpected gdb representation: %r\n%s' % \ (gdb_repr, gdb_output)) @@ -613,7 +625,7 @@ id(foo.__code__)''', breakpoint='builtin_id', cmds_after_breakpoint=['print (PyFrameObject*)(((PyCodeObject*)v)->co_zombieframe)'] ) - self.assertTrue(re.match('.*\s+\$1 =\s+Frame 0x-?[0-9a-f]+, for file <string>, line 3, in foo \(\)\s+.*', + self.assertTrue(re.match(r'.*\s+\$1 =\s+Frame 0x-?[0-9a-f]+, for file <string>, line 3, in foo \(\)\s+.*', gdb_output, re.DOTALL), 'Unexpected gdb representation: %r\n%s' % (gdb_output, gdb_output)) @@ -667,7 +679,7 @@ class StackNavigationTests(DebuggerTests): def test_pyup_command(self): 'Verify that the "py-up" command works' bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-up']) + cmds_after_breakpoint=['py-up', 'py-up']) self.assertMultilineMatches(bt, r'''^.* #[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\) @@ -686,7 +698,7 @@ $''') def test_up_at_top(self): 'Verify handling of "py-up" at the top of the stack' bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-up'] * 4) + cmds_after_breakpoint=['py-up'] * 5) self.assertEndsWith(bt, 'Unable to find an older python frame\n') @@ -696,7 +708,7 @@ $''') def test_up_then_down(self): 'Verify "py-up" followed by "py-down"' bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-up', 'py-down']) + cmds_after_breakpoint=['py-up', 'py-up', 'py-down']) self.assertMultilineMatches(bt, r'''^.* #[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\) @@ -715,6 +727,7 @@ class PyBtTests(DebuggerTests): self.assertMultilineMatches(bt, r'''^.* Traceback \(most recent call first\): + <built-in method id of module object .*> File ".*gdb_sample.py", line 10, in baz id\(42\) File ".*gdb_sample.py", line 7, in bar @@ -841,7 +854,7 @@ class PyPrintTests(DebuggerTests): def test_basic_command(self): 'Verify that the "py-print" command works' bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-print args']) + cmds_after_breakpoint=['py-up', 'py-print args']) self.assertMultilineMatches(bt, r".*\nlocal 'args' = \(1, 2, 3\)\n.*") @@ -850,7 +863,7 @@ class PyPrintTests(DebuggerTests): @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") def test_print_after_up(self): bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-up', 'py-print c', 'py-print b', 'py-print a']) + cmds_after_breakpoint=['py-up', 'py-up', 'py-print c', 'py-print b', 'py-print a']) self.assertMultilineMatches(bt, r".*\nlocal 'c' = 3\nlocal 'b' = 2\nlocal 'a' = 1\n.*") @@ -858,7 +871,7 @@ class PyPrintTests(DebuggerTests): "Python was compiled with optimizations") def test_printing_global(self): bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-print __name__']) + cmds_after_breakpoint=['py-up', 'py-print __name__']) self.assertMultilineMatches(bt, r".*\nglobal '__name__' = '__main__'\n.*") @@ -866,7 +879,7 @@ class PyPrintTests(DebuggerTests): "Python was compiled with optimizations") def test_printing_builtin(self): bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-print len']) + cmds_after_breakpoint=['py-up', 'py-print len']) self.assertMultilineMatches(bt, r".*\nbuiltin 'len' = <built-in method len of module object at remote 0x-?[0-9a-f]+>\n.*") @@ -875,7 +888,7 @@ class PyLocalsTests(DebuggerTests): "Python was compiled with optimizations") def test_basic_command(self): bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-locals']) + cmds_after_breakpoint=['py-up', 'py-locals']) self.assertMultilineMatches(bt, r".*\nargs = \(1, 2, 3\)\n.*") @@ -884,7 +897,7 @@ class PyLocalsTests(DebuggerTests): "Python was compiled with optimizations") def test_locals_after_up(self): bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-up', 'py-locals']) + cmds_after_breakpoint=['py-up', 'py-up', 'py-locals']) self.assertMultilineMatches(bt, r".*\na = 1\nb = 2\nc = 3\n.*") diff --git a/Lib/test/test_pep479.py b/Lib/test/test_generator_stop.py index bc235ceb00..bc235ceb00 100644 --- a/Lib/test/test_pep479.py +++ b/Lib/test/test_generator_stop.py diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index cd6a43d753..f81c82f705 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -245,11 +245,11 @@ class ExceptionTest(unittest.TestCase): yield with self.assertRaises(StopIteration), \ - self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + self.assertWarnsRegex(DeprecationWarning, "StopIteration"): next(gen()) - with self.assertRaisesRegex(PendingDeprecationWarning, + with self.assertRaisesRegex(DeprecationWarning, "generator .* raised StopIteration"), \ warnings.catch_warnings(): @@ -268,7 +268,7 @@ class ExceptionTest(unittest.TestCase): g = f() self.assertEqual(next(g), 1) - with self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): with self.assertRaises(StopIteration): next(g) diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index b77d1d79a1..f698e13f68 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -8,13 +8,12 @@ import sys import unittest import warnings from test import support +android_not_root = support.android_not_root -def safe_rmdir(dirname): - try: - os.rmdir(dirname) - except OSError: - pass +def create_file(filename, data=b'foo'): + with open(filename, 'xb', 0) as fp: + fp.write(data) class GenericTest: @@ -97,52 +96,47 @@ class GenericTest: self.assertNotEqual(s1[n:n+1], s2[n:n+1]) def test_getsize(self): - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - self.assertEqual(self.pathmodule.getsize(support.TESTFN), 3) - finally: - if not f.closed: - f.close() - support.unlink(support.TESTFN) + filename = support.TESTFN + self.addCleanup(support.unlink, filename) - def test_time(self): - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - f = open(support.TESTFN, "ab") + create_file(filename, b'Hello') + self.assertEqual(self.pathmodule.getsize(filename), 5) + os.remove(filename) + + create_file(filename, b'Hello World!') + self.assertEqual(self.pathmodule.getsize(filename), 12) + + def test_filetime(self): + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + create_file(filename, b'foo') + + with open(filename, "ab", 0) as f: f.write(b"bar") - f.close() - f = open(support.TESTFN, "rb") - d = f.read() - f.close() - self.assertEqual(d, b"foobar") - - self.assertLessEqual( - self.pathmodule.getctime(support.TESTFN), - self.pathmodule.getmtime(support.TESTFN) - ) - finally: - if not f.closed: - f.close() - support.unlink(support.TESTFN) + + with open(filename, "rb", 0) as f: + data = f.read() + self.assertEqual(data, b"foobar") + + self.assertLessEqual( + self.pathmodule.getctime(filename), + self.pathmodule.getmtime(filename) + ) def test_exists(self): - self.assertIs(self.pathmodule.exists(support.TESTFN), False) - f = open(support.TESTFN, "wb") - try: + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + self.assertIs(self.pathmodule.exists(filename), False) + + with open(filename, "xb") as f: f.write(b"foo") - f.close() - self.assertIs(self.pathmodule.exists(support.TESTFN), True) - if not self.pathmodule == genericpath: - self.assertIs(self.pathmodule.lexists(support.TESTFN), - True) - finally: - if not f.close(): - f.close() - support.unlink(support.TESTFN) + + self.assertIs(self.pathmodule.exists(filename), True) + + if not self.pathmodule == genericpath: + self.assertIs(self.pathmodule.lexists(filename), True) @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_exists_fd(self): @@ -154,119 +148,126 @@ class GenericTest: os.close(w) self.assertFalse(self.pathmodule.exists(r)) - def test_isdir(self): - self.assertIs(self.pathmodule.isdir(support.TESTFN), False) - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - self.assertIs(self.pathmodule.isdir(support.TESTFN), False) - os.remove(support.TESTFN) - os.mkdir(support.TESTFN) - self.assertIs(self.pathmodule.isdir(support.TESTFN), True) - os.rmdir(support.TESTFN) - finally: - if not f.close(): - f.close() - support.unlink(support.TESTFN) - safe_rmdir(support.TESTFN) - - def test_isfile(self): - self.assertIs(self.pathmodule.isfile(support.TESTFN), False) - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - self.assertIs(self.pathmodule.isfile(support.TESTFN), True) - os.remove(support.TESTFN) - os.mkdir(support.TESTFN) - self.assertIs(self.pathmodule.isfile(support.TESTFN), False) - os.rmdir(support.TESTFN) - finally: - if not f.close(): - f.close() - support.unlink(support.TESTFN) - safe_rmdir(support.TESTFN) + def test_isdir_file(self): + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + self.assertIs(self.pathmodule.isdir(filename), False) + + create_file(filename) + self.assertIs(self.pathmodule.isdir(filename), False) + + def test_isdir_dir(self): + filename = support.TESTFN + self.addCleanup(support.rmdir, filename) + self.assertIs(self.pathmodule.isdir(filename), False) - @staticmethod - def _create_file(filename): - with open(filename, 'wb') as f: - f.write(b'foo') + os.mkdir(filename) + self.assertIs(self.pathmodule.isdir(filename), True) + + def test_isfile_file(self): + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + self.assertIs(self.pathmodule.isfile(filename), False) + + create_file(filename) + self.assertIs(self.pathmodule.isfile(filename), True) + + def test_isfile_dir(self): + filename = support.TESTFN + self.addCleanup(support.rmdir, filename) + self.assertIs(self.pathmodule.isfile(filename), False) + + os.mkdir(filename) + self.assertIs(self.pathmodule.isfile(filename), False) def test_samefile(self): - try: - test_fn = support.TESTFN + "1" - self._create_file(test_fn) - self.assertTrue(self.pathmodule.samefile(test_fn, test_fn)) - self.assertRaises(TypeError, self.pathmodule.samefile) - finally: - os.remove(test_fn) + file1 = support.TESTFN + file2 = support.TESTFN + "2" + self.addCleanup(support.unlink, file1) + self.addCleanup(support.unlink, file2) + + create_file(file1) + self.assertTrue(self.pathmodule.samefile(file1, file1)) + + create_file(file2) + self.assertFalse(self.pathmodule.samefile(file1, file2)) + + self.assertRaises(TypeError, self.pathmodule.samefile) + + def _test_samefile_on_link_func(self, func): + test_fn1 = support.TESTFN + test_fn2 = support.TESTFN + "2" + self.addCleanup(support.unlink, test_fn1) + self.addCleanup(support.unlink, test_fn2) + + create_file(test_fn1) + + func(test_fn1, test_fn2) + self.assertTrue(self.pathmodule.samefile(test_fn1, test_fn2)) + os.remove(test_fn2) + + create_file(test_fn2) + self.assertFalse(self.pathmodule.samefile(test_fn1, test_fn2)) @support.skip_unless_symlink def test_samefile_on_symlink(self): self._test_samefile_on_link_func(os.symlink) + @unittest.skipIf(android_not_root, "hard links not allowed, non root user") def test_samefile_on_link(self): self._test_samefile_on_link_func(os.link) - def _test_samefile_on_link_func(self, func): - try: - test_fn1 = support.TESTFN + "1" - test_fn2 = support.TESTFN + "2" - self._create_file(test_fn1) + def test_samestat(self): + test_fn1 = support.TESTFN + test_fn2 = support.TESTFN + "2" + self.addCleanup(support.unlink, test_fn1) + self.addCleanup(support.unlink, test_fn2) - func(test_fn1, test_fn2) - self.assertTrue(self.pathmodule.samefile(test_fn1, test_fn2)) - os.remove(test_fn2) + create_file(test_fn1) + stat1 = os.stat(test_fn1) + self.assertTrue(self.pathmodule.samestat(stat1, os.stat(test_fn1))) - self._create_file(test_fn2) - self.assertFalse(self.pathmodule.samefile(test_fn1, test_fn2)) - finally: - os.remove(test_fn1) - os.remove(test_fn2) + create_file(test_fn2) + stat2 = os.stat(test_fn2) + self.assertFalse(self.pathmodule.samestat(stat1, stat2)) - def test_samestat(self): - try: - test_fn = support.TESTFN + "1" - self._create_file(test_fn) - test_fns = [test_fn]*2 - stats = map(os.stat, test_fns) - self.assertTrue(self.pathmodule.samestat(*stats)) - finally: - os.remove(test_fn) + self.assertRaises(TypeError, self.pathmodule.samestat) + + def _test_samestat_on_link_func(self, func): + test_fn1 = support.TESTFN + "1" + test_fn2 = support.TESTFN + "2" + self.addCleanup(support.unlink, test_fn1) + self.addCleanup(support.unlink, test_fn2) + + create_file(test_fn1) + func(test_fn1, test_fn2) + self.assertTrue(self.pathmodule.samestat(os.stat(test_fn1), + os.stat(test_fn2))) + os.remove(test_fn2) + + create_file(test_fn2) + self.assertFalse(self.pathmodule.samestat(os.stat(test_fn1), + os.stat(test_fn2))) @support.skip_unless_symlink def test_samestat_on_symlink(self): self._test_samestat_on_link_func(os.symlink) + @unittest.skipIf(android_not_root, "hard links not allowed, non root user") def test_samestat_on_link(self): self._test_samestat_on_link_func(os.link) - def _test_samestat_on_link_func(self, func): - try: - test_fn1 = support.TESTFN + "1" - test_fn2 = support.TESTFN + "2" - self._create_file(test_fn1) - test_fns = (test_fn1, test_fn2) - func(*test_fns) - stats = map(os.stat, test_fns) - self.assertTrue(self.pathmodule.samestat(*stats)) - os.remove(test_fn2) - - self._create_file(test_fn2) - stats = map(os.stat, test_fns) - self.assertFalse(self.pathmodule.samestat(*stats)) - - self.assertRaises(TypeError, self.pathmodule.samestat) - finally: - os.remove(test_fn1) - os.remove(test_fn2) - def test_sameopenfile(self): - fname = support.TESTFN + "1" - with open(fname, "wb") as a, open(fname, "wb") as b: - self.assertTrue(self.pathmodule.sameopenfile( - a.fileno(), b.fileno())) + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + create_file(filename) + + with open(filename, "rb", 0) as fp1: + fd1 = fp1.fileno() + with open(filename, "rb", 0) as fp2: + fd2 = fp2.fileno() + self.assertTrue(self.pathmodule.sameopenfile(fd1, fd2)) + class TestGenericTest(GenericTest, unittest.TestCase): # Issue 16852: GenericTest can't inherit from unittest.TestCase @@ -390,10 +391,13 @@ class CommonTest(GenericTest): warnings.simplefilter("ignore", DeprecationWarning) self.assertIn(b"foo", self.pathmodule.abspath(b"foo")) + # avoid UnicodeDecodeError on Windows + undecodable_path = b'' if sys.platform == 'win32' else b'f\xf2\xf2' + # Abspath returns bytes when the arg is bytes with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - for path in (b'', b'foo', b'f\xf2\xf2', b'/foo', b'C:\\'): + for path in (b'', b'foo', undecodable_path, b'/foo', b'C:\\'): self.assertIsInstance(self.pathmodule.abspath(path), bytes) def test_realpath(self): @@ -452,16 +456,15 @@ class CommonTest(GenericTest): with self.assertRaisesRegex(TypeError, errmsg): self.pathmodule.join('str', b'bytes') # regression, see #15377 - errmsg = r'join\(\) argument must be str or bytes, not %r' - with self.assertRaisesRegex(TypeError, errmsg % 'int'): + with self.assertRaisesRegex(TypeError, 'int'): self.pathmodule.join(42, 'str') - with self.assertRaisesRegex(TypeError, errmsg % 'int'): + with self.assertRaisesRegex(TypeError, 'int'): self.pathmodule.join('str', 42) - with self.assertRaisesRegex(TypeError, errmsg % 'int'): + with self.assertRaisesRegex(TypeError, 'int'): self.pathmodule.join(42) - with self.assertRaisesRegex(TypeError, errmsg % 'list'): + with self.assertRaisesRegex(TypeError, 'list'): self.pathmodule.join([]) - with self.assertRaisesRegex(TypeError, errmsg % 'bytearray'): + with self.assertRaisesRegex(TypeError, 'bytearray'): self.pathmodule.join(bytearray(b'foo'), bytearray(b'bar')) def test_relpath_errors(self): @@ -473,14 +476,59 @@ class CommonTest(GenericTest): self.pathmodule.relpath(b'bytes', 'str') with self.assertRaisesRegex(TypeError, errmsg): self.pathmodule.relpath('str', b'bytes') - errmsg = r'relpath\(\) argument must be str or bytes, not %r' - with self.assertRaisesRegex(TypeError, errmsg % 'int'): + with self.assertRaisesRegex(TypeError, 'int'): self.pathmodule.relpath(42, 'str') - with self.assertRaisesRegex(TypeError, errmsg % 'int'): + with self.assertRaisesRegex(TypeError, 'int'): self.pathmodule.relpath('str', 42) - with self.assertRaisesRegex(TypeError, errmsg % 'bytearray'): + with self.assertRaisesRegex(TypeError, 'bytearray'): self.pathmodule.relpath(bytearray(b'foo'), bytearray(b'bar')) +class PathLikeTests(unittest.TestCase): + + class PathLike: + def __init__(self, path=''): + self.path = path + def __fspath__(self): + if isinstance(self.path, BaseException): + raise self.path + else: + return self.path + + def setUp(self): + self.file_name = support.TESTFN.lower() + self.file_path = self.PathLike(support.TESTFN) + self.addCleanup(support.unlink, self.file_name) + create_file(self.file_name, b"test_genericpath.PathLikeTests") + + def assertPathEqual(self, func): + self.assertEqual(func(self.file_path), func(self.file_name)) + + def test_path_exists(self): + self.assertPathEqual(os.path.exists) + + def test_path_isfile(self): + self.assertPathEqual(os.path.isfile) + + def test_path_isdir(self): + self.assertPathEqual(os.path.isdir) + + def test_path_commonprefix(self): + self.assertEqual(os.path.commonprefix([self.file_path, self.file_name]), + self.file_name) + + def test_path_getsize(self): + self.assertPathEqual(os.path.getsize) + + def test_path_getmtime(self): + self.assertPathEqual(os.path.getatime) + + def test_path_getctime(self): + self.assertPathEqual(os.path.getctime) + + def test_path_samefile(self): + self.assertTrue(os.path.samefile(self.file_path, self.file_name)) + + if __name__=="__main__": unittest.main() diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_getargs2.py index 984aac7d91..8a194aa03d 100644 --- a/Lib/test/test_getargs2.py +++ b/Lib/test/test_getargs2.py @@ -5,10 +5,6 @@ from test import support # Skip this test if the _testcapi module isn't available. support.import_module('_testcapi') from _testcapi import getargs_keywords, getargs_keyword_only -try: - from _testcapi import getargs_L, getargs_K -except ImportError: - getargs_L = None # PY_LONG_LONG not available # > How about the following counterproposal. This also changes some of # > the other format codes to be a little more regular. @@ -309,7 +305,6 @@ class Signed_TestCase(unittest.TestCase): self.assertRaises(OverflowError, getargs_n, VERY_LARGE) -@unittest.skipIf(getargs_L is None, 'PY_LONG_LONG is not available') class LongLong_TestCase(unittest.TestCase): def test_L(self): from _testcapi import getargs_L @@ -365,7 +360,8 @@ class Float_TestCase(unittest.TestCase): self.assertEqual(getargs_f(FloatSubclass(7.5)), 7.5) self.assertEqual(getargs_f(FloatSubclass2(7.5)), 7.5) self.assertRaises(TypeError, getargs_f, BadFloat()) - self.assertEqual(getargs_f(BadFloat2()), 4.25) + with self.assertWarns(DeprecationWarning): + self.assertEqual(getargs_f(BadFloat2()), 4.25) self.assertEqual(getargs_f(BadFloat3(7.5)), 7.5) for x in (FLT_MIN, -FLT_MIN, FLT_MAX, -FLT_MAX, INF, -INF): @@ -390,7 +386,8 @@ class Float_TestCase(unittest.TestCase): self.assertEqual(getargs_d(FloatSubclass(7.5)), 7.5) self.assertEqual(getargs_d(FloatSubclass2(7.5)), 7.5) self.assertRaises(TypeError, getargs_d, BadFloat()) - self.assertEqual(getargs_d(BadFloat2()), 4.25) + with self.assertWarns(DeprecationWarning): + self.assertEqual(getargs_d(BadFloat2()), 4.25) self.assertEqual(getargs_d(BadFloat3(7.5)), 7.5) for x in (DBL_MIN, -DBL_MIN, DBL_MAX, -DBL_MAX, INF, -INF): @@ -474,7 +471,7 @@ class Tuple_TestCase(unittest.TestCase): ret = get_args(*TupleSubclass([1, 2])) self.assertEqual(ret, (1, 2)) - self.assertIsInstance(ret, tuple) + self.assertIs(type(ret), tuple) ret = get_args() self.assertIn(ret, ((), None)) @@ -512,7 +509,7 @@ class Keywords_TestCase(unittest.TestCase): ret = get_kwargs(**DictSubclass({'a': 1, 'b': 2})) self.assertEqual(ret, {'a': 1, 'b': 2}) - self.assertIsInstance(ret, dict) + self.assertIs(type(ret), dict) ret = get_kwargs() self.assertIn(ret, ({}, None)) @@ -628,20 +625,20 @@ class KeywordOnly_TestCase(unittest.TestCase): ) # required arg missing with self.assertRaisesRegex(TypeError, - "Required argument 'required' \(pos 1\) not found"): + r"Required argument 'required' \(pos 1\) not found"): getargs_keyword_only(optional=2) with self.assertRaisesRegex(TypeError, - "Required argument 'required' \(pos 1\) not found"): + r"Required argument 'required' \(pos 1\) not found"): getargs_keyword_only(keyword_only=3) def test_too_many_args(self): with self.assertRaisesRegex(TypeError, - "Function takes at most 2 positional arguments \(3 given\)"): + r"Function takes at most 2 positional arguments \(3 given\)"): getargs_keyword_only(1, 2, 3) with self.assertRaisesRegex(TypeError, - "function takes at most 3 arguments \(4 given\)"): + r"function takes at most 3 arguments \(4 given\)"): getargs_keyword_only(1, 2, 3, keyword_only=5) def test_invalid_keyword(self): @@ -656,6 +653,39 @@ class KeywordOnly_TestCase(unittest.TestCase): getargs_keyword_only(1, 2, **{'\uDC80': 10}) +class PositionalOnlyAndKeywords_TestCase(unittest.TestCase): + from _testcapi import getargs_positional_only_and_keywords as getargs + + def test_positional_args(self): + # using all possible positional args + self.assertEqual(self.getargs(1, 2, 3), (1, 2, 3)) + + def test_mixed_args(self): + # positional and keyword args + self.assertEqual(self.getargs(1, 2, keyword=3), (1, 2, 3)) + + def test_optional_args(self): + # missing optional args + self.assertEqual(self.getargs(1, 2), (1, 2, -1)) + self.assertEqual(self.getargs(1, keyword=3), (1, -1, 3)) + + def test_required_args(self): + self.assertEqual(self.getargs(1), (1, -1, -1)) + # required positional arg missing + with self.assertRaisesRegex(TypeError, + r"Function takes at least 1 positional arguments \(0 given\)"): + self.getargs() + + with self.assertRaisesRegex(TypeError, + r"Function takes at least 1 positional arguments \(0 given\)"): + self.getargs(keyword=3) + + def test_empty_keyword(self): + with self.assertRaisesRegex(TypeError, + "'' is an invalid keyword argument for this function"): + self.getargs(1, 2, **{'': 666}) + + class Bytes_TestCase(unittest.TestCase): def test_c(self): from _testcapi import getargs_c @@ -822,10 +852,10 @@ class String_TestCase(unittest.TestCase): self.assertEqual(getargs_es_hash('abc\xe9', 'latin1', buf), b'abc\xe9') self.assertEqual(buf, bytearray(b'abc\xe9\x00')) buf = bytearray(b'x'*4) - self.assertRaises(TypeError, getargs_es_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_es_hash, 'abc\xe9', 'latin1', buf) self.assertEqual(buf, bytearray(b'x'*4)) buf = bytearray() - self.assertRaises(TypeError, getargs_es_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_es_hash, 'abc\xe9', 'latin1', buf) def test_et_hash(self): from _testcapi import getargs_et_hash @@ -848,10 +878,10 @@ class String_TestCase(unittest.TestCase): self.assertEqual(getargs_et_hash('abc\xe9', 'latin1', buf), b'abc\xe9') self.assertEqual(buf, bytearray(b'abc\xe9\x00')) buf = bytearray(b'x'*4) - self.assertRaises(TypeError, getargs_et_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) self.assertEqual(buf, bytearray(b'x'*4)) buf = bytearray() - self.assertRaises(TypeError, getargs_et_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) def test_u(self): from _testcapi import getargs_u diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index f57e54689c..a852443e68 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -1,6 +1,5 @@ import os import base64 -import shutil import gettext import unittest @@ -527,6 +526,12 @@ class GettextCacheTestCase(GettextBaseTest): self.assertEqual(t.__class__, DummyGNUTranslations) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'c2py', 'ENOENT'} + support.check__all__(self, gettext, blacklist=blacklist) + + def test_main(): support.run_unittest(__name__) diff --git a/Lib/test/test_global.py b/Lib/test/test_global.py index 37ec67255a..9eeccf1250 100644 --- a/Lib/test/test_global.py +++ b/Lib/test/test_global.py @@ -24,7 +24,7 @@ def wrong1(): global a global b """ - check_syntax_error(self, prog_text_1) + check_syntax_error(self, prog_text_1, lineno=4, offset=4) def test2(self): prog_text_2 = """\ @@ -32,7 +32,7 @@ def wrong2(): print(x) global x """ - check_syntax_error(self, prog_text_2) + check_syntax_error(self, prog_text_2, lineno=3, offset=4) def test3(self): prog_text_3 = """\ @@ -41,7 +41,7 @@ def wrong3(): x = 2 global x """ - check_syntax_error(self, prog_text_3) + check_syntax_error(self, prog_text_3, lineno=4, offset=4) def test4(self): prog_text_4 = """\ diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 154e3b608c..65e26bfd38 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -8,6 +8,95 @@ import sys # testing import * from sys import * +# different import patterns to check that __annotations__ does not interfere +# with import machinery +import test.ann_module as ann_module +import typing +from collections import ChainMap +from test import ann_module2 +import test + +# These are shared with test_tokenize and other test modules. +# +# Note: since several test cases filter out floats by looking for "e" and ".", +# don't add hexadecimal literals that contain "e" or "E". +VALID_UNDERSCORE_LITERALS = [ + '0_0_0', + '4_2', + '1_0000_0000', + '0b1001_0100', + '0xffff_ffff', + '0o5_7_7', + '1_00_00.5', + '1_00_00.5e5', + '1_00_00e5_1', + '1e1_0', + '.1_4', + '.1_4e1', + '0b_0', + '0x_f', + '0o_5', + '1_00_00j', + '1_00_00.5j', + '1_00_00e5_1j', + '.1_4j', + '(1_2.5+3_3j)', + '(.5_6j)', +] +INVALID_UNDERSCORE_LITERALS = [ + # Trailing underscores: + '0_', + '42_', + '1.4j_', + '0x_', + '0b1_', + '0xf_', + '0o5_', + '0 if 1_Else 1', + # Underscores in the base selector: + '0_b0', + '0_xf', + '0_o5', + # Old-style octal, still disallowed: + '0_7', + '09_99', + # Multiple consecutive underscores: + '4_______2', + '0.1__4', + '0.1__4j', + '0b1001__0100', + '0xffff__ffff', + '0x___', + '0o5__77', + '1e1__0', + '1e1__0j', + # Underscore right before a dot: + '1_.4', + '1_.4j', + # Underscore right after a dot: + '1._4', + '1._4j', + '._5', + '._5j', + # Underscore right after a sign: + '1.0e+_1', + '1.0e+_1j', + # Underscore right before j: + '1.4_j', + '1.4e5_j', + # Underscore right before e: + '1_e1', + '1.4_e1', + '1.4_e1j', + # Underscore right after e: + '1e_1', + '1.4e_1', + '1.4e_1j', + # Complex cases with parens: + '(1+1.5_j_)', + '(1+1.5_j)', +] + class TokenTests(unittest.TestCase): @@ -87,6 +176,14 @@ class TokenTests(unittest.TestCase): self.assertEqual(1 if 0else 0, 0) self.assertRaises(SyntaxError, eval, "0 if 1Else 0") + def test_underscore_literals(self): + for lit in VALID_UNDERSCORE_LITERALS: + self.assertEqual(eval(lit), eval(lit.replace('_', ''))) + for lit in INVALID_UNDERSCORE_LITERALS: + self.assertRaises(SyntaxError, eval, lit) + # Sanity check: no literal begins with an underscore + self.assertRaises(NameError, eval, "_0") + def test_string_literals(self): x = ''; y = ""; self.assertTrue(len(x) == 0 and x == y) x = '\''; y = "'"; self.assertTrue(len(x) == 1 and x == y and ord(x) == 39) @@ -139,6 +236,19 @@ the \'lazy\' dog.\n\ compile(s, "<test>", "exec") self.assertIn("unexpected EOF", str(cm.exception)) +var_annot_global: int # a global annotated is necessary for test_var_annot + +# custom namespace for testing __annotations__ + +class CNS: + def __init__(self): + self._dct = {} + def __setitem__(self, item, value): + self._dct[item.lower()] = value + def __getitem__(self, item): + return self._dct[item] + + class GrammarTests(unittest.TestCase): # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE @@ -154,6 +264,160 @@ class GrammarTests(unittest.TestCase): # testlist ENDMARKER x = eval('1, 0 or 1') + def test_var_annot_basics(self): + # all these should be allowed + var1: int = 5 + var2: [int, str] + my_lst = [42] + def one(): + return 1 + int.new_attr: int + [list][0]: type + my_lst[one()-1]: int = 5 + self.assertEqual(my_lst, [5]) + + def test_var_annot_syntax_errors(self): + # parser pass + check_syntax_error(self, "def f: int") + check_syntax_error(self, "x: int: str") + check_syntax_error(self, "def f():\n" + " nonlocal x: int\n") + # AST pass + check_syntax_error(self, "[x, 0]: int\n") + check_syntax_error(self, "f(): int\n") + check_syntax_error(self, "(x,): int") + check_syntax_error(self, "def f():\n" + " (x, y): int = (1, 2)\n") + # symtable pass + check_syntax_error(self, "def f():\n" + " x: int\n" + " global x\n") + check_syntax_error(self, "def f():\n" + " global x\n" + " x: int\n") + + def test_var_annot_basic_semantics(self): + # execution order + with self.assertRaises(ZeroDivisionError): + no_name[does_not_exist]: no_name_again = 1/0 + with self.assertRaises(NameError): + no_name[does_not_exist]: 1/0 = 0 + global var_annot_global + + # function semantics + def f(): + st: str = "Hello" + a.b: int = (1, 2) + return st + self.assertEqual(f.__annotations__, {}) + def f_OK(): + x: 1/0 + f_OK() + def fbad(): + x: int + print(x) + with self.assertRaises(UnboundLocalError): + fbad() + def f2bad(): + (no_such_global): int + print(no_such_global) + try: + f2bad() + except Exception as e: + self.assertIs(type(e), NameError) + + # class semantics + class C: + __foo: int + s: str = "attr" + z = 2 + def __init__(self, x): + self.x: int = x + self.assertEqual(C.__annotations__, {'_C__foo': int, 's': str}) + with self.assertRaises(NameError): + class CBad: + no_such_name_defined.attr: int = 0 + with self.assertRaises(NameError): + class Cbad2(C): + x: int + x.y: list = [] + + def test_var_annot_metaclass_semantics(self): + class CMeta(type): + @classmethod + def __prepare__(metacls, name, bases, **kwds): + return {'__annotations__': CNS()} + class CC(metaclass=CMeta): + XX: 'ANNOT' + self.assertEqual(CC.__annotations__['xx'], 'ANNOT') + + def test_var_annot_module_semantics(self): + with self.assertRaises(AttributeError): + print(test.__annotations__) + self.assertEqual(ann_module.__annotations__, + {1: 2, 'x': int, 'y': str, 'f': typing.Tuple[int, int]}) + self.assertEqual(ann_module.M.__annotations__, + {'123': 123, 'o': type}) + self.assertEqual(ann_module2.__annotations__, {}) + + def test_var_annot_in_module(self): + # check that functions fail the same way when executed + # outside of module where they were defined + from test.ann_module3 import f_bad_ann, g_bad_ann, D_bad_ann + with self.assertRaises(NameError): + f_bad_ann() + with self.assertRaises(NameError): + g_bad_ann() + with self.assertRaises(NameError): + D_bad_ann(5) + + def test_var_annot_simple_exec(self): + gns = {}; lns= {} + exec("'docstring'\n" + "__annotations__[1] = 2\n" + "x: int = 5\n", gns, lns) + self.assertEqual(lns["__annotations__"], {1: 2, 'x': int}) + with self.assertRaises(KeyError): + gns['__annotations__'] + + def test_var_annot_custom_maps(self): + # tests with custom locals() and __annotations__ + ns = {'__annotations__': CNS()} + exec('X: int; Z: str = "Z"; (w): complex = 1j', ns) + self.assertEqual(ns['__annotations__']['x'], int) + self.assertEqual(ns['__annotations__']['z'], str) + with self.assertRaises(KeyError): + ns['__annotations__']['w'] + nonloc_ns = {} + class CNS2: + def __init__(self): + self._dct = {} + def __setitem__(self, item, value): + nonlocal nonloc_ns + self._dct[item] = value + nonloc_ns[item] = value + def __getitem__(self, item): + return self._dct[item] + exec('x: int = 1', {}, CNS2()) + self.assertEqual(nonloc_ns['__annotations__']['x'], int) + + def test_var_annot_refleak(self): + # complex case: custom locals plus custom __annotations__ + # this was causing refleak + cns = CNS() + nonloc_ns = {'__annotations__': cns} + class CNS2: + def __init__(self): + self._dct = {'__annotations__': cns} + def __setitem__(self, item, value): + nonlocal nonloc_ns + self._dct[item] = value + nonloc_ns[item] = value + def __getitem__(self, item): + return self._dct[item] + exec('X: str', {}, CNS2()) + self.assertEqual(nonloc_ns['__annotations__']['x'], str) + def test_funcdef(self): ### [decorators] 'def' NAME parameters ['->' test] ':' suite ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE @@ -295,6 +559,10 @@ class GrammarTests(unittest.TestCase): pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200) pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100) + self.assertRaises(SyntaxError, eval, "def f(*): pass") + self.assertRaises(SyntaxError, eval, "def f(*,): pass") + self.assertRaises(SyntaxError, eval, "def f(*, **kwds): pass") + # keyword arguments after *arglist def f(*args, **kwargs): return args, kwargs @@ -341,7 +609,7 @@ class GrammarTests(unittest.TestCase): def f(x) -> list: pass self.assertEqual(f.__annotations__, {'return': list}) - # test MAKE_CLOSURE with a variety of oparg's + # test closures with a variety of opargs closure = 1 def f(): return closure def f(x=1): return closure @@ -352,6 +620,23 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "f(*g(1=2))") check_syntax_error(self, "f(**g(1=2))") + # Check trailing commas are permitted in funcdef argument list + def f(a,): pass + def f(*args,): pass + def f(**kwds,): pass + def f(a, *args,): pass + def f(a, **kwds,): pass + def f(*args, b,): pass + def f(*, b,): pass + def f(*args, **kwds,): pass + def f(a, *args, b,): pass + def f(a, *, b,): pass + def f(a, *args, **kwds,): pass + def f(*args, b, **kwds,): pass + def f(*, b, **kwds,): pass + def f(a, *args, b, **kwds,): pass + def f(a, *, b, **kwds,): pass + def test_lambdef(self): ### lambdef: 'lambda' [varargslist] ':' test l1 = lambda : 0 @@ -370,6 +655,23 @@ class GrammarTests(unittest.TestCase): self.assertEqual(l6(1,2), 1+2+20) self.assertEqual(l6(1,2,k=10), 1+2+10) + # check that trailing commas are permitted + l10 = lambda a,: 0 + l11 = lambda *args,: 0 + l12 = lambda **kwds,: 0 + l13 = lambda a, *args,: 0 + l14 = lambda a, **kwds,: 0 + l15 = lambda *args, b,: 0 + l16 = lambda *, b,: 0 + l17 = lambda *args, **kwds,: 0 + l18 = lambda a, *args, b,: 0 + l19 = lambda a, *, b,: 0 + l20 = lambda a, *args, **kwds,: 0 + l21 = lambda *args, b, **kwds,: 0 + l22 = lambda *, b, **kwds,: 0 + l23 = lambda a, *args, b, **kwds,: 0 + l24 = lambda a, *, b, **kwds,: 0 + ### stmt: simple_stmt | compound_stmt # Tested below @@ -1040,18 +1342,6 @@ class GrammarTests(unittest.TestCase): self.assertEqual(m.other, 42) def test_async_await(self): - async = 1 - await = 2 - self.assertEqual(async, 1) - - def async(): - nonlocal await - await = 10 - async() - self.assertEqual(await, 10) - - self.assertFalse(bool(async.__code__.co_flags & inspect.CO_COROUTINE)) - async def test(): def sum(): pass diff --git a/Lib/test/test_grp.py b/Lib/test/test_grp.py index 272b08615d..69095a3fb9 100644 --- a/Lib/test/test_grp.py +++ b/Lib/test/test_grp.py @@ -92,5 +92,15 @@ class GroupDatabaseTestCase(unittest.TestCase): self.assertRaises(KeyError, grp.getgrgid, fakegid) + def test_noninteger_gid(self): + entries = grp.getgrall() + if not entries: + self.skipTest('no groups') + # Choose an existent gid. + gid = entries[0][2] + self.assertWarns(DeprecationWarning, grp.getgrgid, float(gid)) + self.assertWarns(DeprecationWarning, grp.getgrgid, str(gid)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index 3c51673a92..b457bd3f44 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -5,6 +5,7 @@ import unittest from test import support from test.support import bigmemtest, _4G import os +import pathlib import io import struct import array @@ -67,6 +68,18 @@ class TestGzip(BaseTest): # Test multiple close() calls. f.close() + def test_write_read_with_pathlike_file(self): + filename = pathlib.Path(self.filename) + with gzip.GzipFile(filename, 'w') as f: + f.write(data1 * 50) + self.assertIsInstance(f.name, str) + with gzip.GzipFile(filename, 'a') as f: + f.write(data1) + with gzip.GzipFile(filename) as f: + d = f.read() + self.assertEqual(d, data1 * 51) + self.assertIsInstance(f.name, str) + # The following test_write_xy methods test that write accepts # the corresponding bytes-like object type as input # and that the data written equals bytes(xy) in all cases. @@ -434,12 +447,12 @@ class TestGzip(BaseTest): def test_decompress_limited(self): """Decompressed data buffering should be limited""" - bomb = gzip.compress(bytes(int(2e6)), compresslevel=9) + bomb = gzip.compress(b'\0' * int(2e6), compresslevel=9) self.assertLess(len(bomb), io.DEFAULT_BUFFER_SIZE) bomb = io.BytesIO(bomb) decomp = gzip.GzipFile(fileobj=bomb) - self.assertEqual(bytes(1), decomp.read(1)) + self.assertEqual(decomp.read(1), b'\0') max_decomp = 1 + io.DEFAULT_BUFFER_SIZE self.assertLessEqual(decomp._buffer.raw.tell(), max_decomp, "Excessive amount of data was decompressed") @@ -521,6 +534,15 @@ class TestOpen(BaseTest): file_data = gzip.decompress(f.read()) self.assertEqual(file_data, uncompressed) + def test_pathlike_file(self): + filename = pathlib.Path(self.filename) + with gzip.open(filename, "wb") as f: + f.write(data1 * 50) + with gzip.open(filename, "ab") as f: + f.write(data1) + with gzip.open(filename) as f: + self.assertEqual(f.read(), data1 * 51) + def test_implicit_binary_modes(self): # Test implicit binary modes (no "b" or "t" in mode string). uncompressed = data1 * 50 diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index c9b113e6ef..f748b46190 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -7,6 +7,7 @@ # import array +from binascii import unhexlify import hashlib import itertools import os @@ -19,6 +20,7 @@ import unittest import warnings from test import support from test.support import _4G, bigmemtest, import_fresh_module +from http.client import HTTPException # Were we compiled --with-pydebug or with #define Py_DEBUG? COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount') @@ -26,6 +28,21 @@ COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount') c_hashlib = import_fresh_module('hashlib', fresh=['_hashlib']) py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) +try: + import _blake2 +except ImportError: + _blake2 = None + +requires_blake2 = unittest.skipUnless(_blake2, 'requires _blake2') + +try: + import _sha3 +except ImportError: + _sha3 = None + +requires_sha3 = unittest.skipUnless(_sha3, 'requires _sha3') + + def hexstr(s): assert isinstance(s, bytes), repr(s) h = "0123456789abcdef" @@ -35,10 +52,33 @@ def hexstr(s): return r +URL = "http://www.pythontest.net/hashlib/{}.txt" + +def read_vectors(hash_name): + url = URL.format(hash_name) + try: + testdata = support.open_urlresource(url) + except (OSError, HTTPException): + raise unittest.SkipTest("Could not retrieve {}".format(url)) + with testdata: + for line in testdata: + line = line.strip() + if line.startswith('#') or not line: + continue + parts = line.split(',') + parts[0] = bytes.fromhex(parts[0]) + yield parts + + class HashLibTestCase(unittest.TestCase): supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1', 'sha224', 'SHA224', 'sha256', 'SHA256', - 'sha384', 'SHA384', 'sha512', 'SHA512') + 'sha384', 'SHA384', 'sha512', 'SHA512', + 'blake2b', 'blake2s', + 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', + 'shake_128', 'shake_256') + + shakes = {'shake_128', 'shake_256'} # Issue #14693: fallback modules are always compiled under POSIX _warn_on_extension_import = os.name == 'posix' or COMPILED_WITH_PYDEBUG @@ -56,6 +96,11 @@ class HashLibTestCase(unittest.TestCase): algorithms = set() for algorithm in self.supported_hash_names: algorithms.add(algorithm.lower()) + + _blake2 = self._conditional_import_module('_blake2') + if _blake2: + algorithms.update({'blake2b', 'blake2s'}) + self.constructors_to_test = {} for algorithm in algorithms: self.constructors_to_test[algorithm] = set() @@ -64,10 +109,10 @@ class HashLibTestCase(unittest.TestCase): # of hashlib.new given the algorithm name. for algorithm, constructors in self.constructors_to_test.items(): constructors.add(getattr(hashlib, algorithm)) - def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm): + def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs): if data is None: - return hashlib.new(_alg) - return hashlib.new(_alg, data) + return hashlib.new(_alg, **kwargs) + return hashlib.new(_alg, data, **kwargs) constructors.add(_test_algorithm_via_hashlib_new) _hashlib = self._conditional_import_module('_hashlib') @@ -99,6 +144,18 @@ class HashLibTestCase(unittest.TestCase): if _sha512: add_builtin_constructor('sha384') add_builtin_constructor('sha512') + if _blake2: + add_builtin_constructor('blake2s') + add_builtin_constructor('blake2b') + + _sha3 = self._conditional_import_module('_sha3') + if _sha3: + add_builtin_constructor('sha3_224') + add_builtin_constructor('sha3_256') + add_builtin_constructor('sha3_384') + add_builtin_constructor('sha3_512') + add_builtin_constructor('shake_128') + add_builtin_constructor('shake_256') super(HashLibTestCase, self).__init__(*args, **kwargs) @@ -111,7 +168,10 @@ class HashLibTestCase(unittest.TestCase): a = array.array("b", range(10)) for cons in self.hash_constructors: c = cons(a) - c.hexdigest() + if c.name in self.shakes: + c.hexdigest(16) + else: + c.hexdigest() def test_algorithms_guaranteed(self): self.assertEqual(hashlib.algorithms_guaranteed, @@ -155,14 +215,21 @@ class HashLibTestCase(unittest.TestCase): def test_hexdigest(self): for cons in self.hash_constructors: h = cons() - self.assertIsInstance(h.digest(), bytes) - self.assertEqual(hexstr(h.digest()), h.hexdigest()) + if h.name in self.shakes: + self.assertIsInstance(h.digest(16), bytes) + self.assertEqual(hexstr(h.digest(16)), h.hexdigest(16)) + else: + self.assertIsInstance(h.digest(), bytes) + self.assertEqual(hexstr(h.digest()), h.hexdigest()) def test_name_attribute(self): for cons in self.hash_constructors: h = cons() self.assertIsInstance(h.name, str) - self.assertIn(h.name, self.supported_hash_names) + if h.name in self.supported_hash_names: + self.assertIn(h.name, self.supported_hash_names) + else: + self.assertNotIn(h.name, self.supported_hash_names) self.assertEqual(h.name, hashlib.new(h.name).name) def test_large_update(self): @@ -177,40 +244,46 @@ class HashLibTestCase(unittest.TestCase): m1.update(bees) m1.update(cees) m1.update(dees) + if m1.name in self.shakes: + args = (16,) + else: + args = () m2 = cons() m2.update(aas + bees + cees + dees) - self.assertEqual(m1.digest(), m2.digest()) + self.assertEqual(m1.digest(*args), m2.digest(*args)) m3 = cons(aas + bees + cees + dees) - self.assertEqual(m1.digest(), m3.digest()) + self.assertEqual(m1.digest(*args), m3.digest(*args)) # verify copy() doesn't touch original m4 = cons(aas + bees + cees) - m4_digest = m4.digest() + m4_digest = m4.digest(*args) m4_copy = m4.copy() m4_copy.update(dees) - self.assertEqual(m1.digest(), m4_copy.digest()) - self.assertEqual(m4.digest(), m4_digest) + self.assertEqual(m1.digest(*args), m4_copy.digest(*args)) + self.assertEqual(m4.digest(*args), m4_digest) - def check(self, name, data, hexdigest): + def check(self, name, data, hexdigest, shake=False, **kwargs): + length = len(hexdigest)//2 hexdigest = hexdigest.lower() constructors = self.constructors_to_test[name] # 2 is for hashlib.name(...) and hashlib.new(name, ...) self.assertGreaterEqual(len(constructors), 2) for hash_object_constructor in constructors: - m = hash_object_constructor(data) - computed = m.hexdigest() + m = hash_object_constructor(data, **kwargs) + computed = m.hexdigest() if not shake else m.hexdigest(length) self.assertEqual( computed, hexdigest, "Hash algorithm %s constructed using %s returned hexdigest" " %r for %d byte input data that should have hashed to %r." % (name, hash_object_constructor, computed, len(data), hexdigest)) - computed = m.digest() + computed = m.digest() if not shake else m.digest(length) digest = bytes.fromhex(hexdigest) self.assertEqual(computed, digest) - self.assertEqual(len(digest), m.digest_size) + if not shake: + self.assertEqual(len(digest), m.digest_size) def check_no_unicode(self, algorithm_name): # Unicode objects are not allowed as input. @@ -226,13 +299,35 @@ class HashLibTestCase(unittest.TestCase): self.check_no_unicode('sha384') self.check_no_unicode('sha512') - def check_blocksize_name(self, name, block_size=0, digest_size=0): + @requires_blake2 + def test_no_unicode_blake2(self): + self.check_no_unicode('blake2b') + self.check_no_unicode('blake2s') + + @requires_sha3 + def test_no_unicode_sha3(self): + self.check_no_unicode('sha3_224') + self.check_no_unicode('sha3_256') + self.check_no_unicode('sha3_384') + self.check_no_unicode('sha3_512') + self.check_no_unicode('shake_128') + self.check_no_unicode('shake_256') + + def check_blocksize_name(self, name, block_size=0, digest_size=0, + digest_length=None): constructors = self.constructors_to_test[name] for hash_object_constructor in constructors: m = hash_object_constructor() self.assertEqual(m.block_size, block_size) self.assertEqual(m.digest_size, digest_size) - self.assertEqual(len(m.digest()), digest_size) + if digest_length: + self.assertEqual(len(m.digest(digest_length)), + digest_length) + self.assertEqual(len(m.hexdigest(digest_length)), + 2*digest_length) + else: + self.assertEqual(len(m.digest()), digest_size) + self.assertEqual(len(m.hexdigest()), 2*digest_size) self.assertEqual(m.name, name) # split for sha3_512 / _sha3.sha3 object self.assertIn(name.split("_")[0], repr(m)) @@ -245,6 +340,38 @@ class HashLibTestCase(unittest.TestCase): self.check_blocksize_name('sha384', 128, 48) self.check_blocksize_name('sha512', 128, 64) + @requires_sha3 + def test_blocksize_name_sha3(self): + self.check_blocksize_name('sha3_224', 144, 28) + self.check_blocksize_name('sha3_256', 136, 32) + self.check_blocksize_name('sha3_384', 104, 48) + self.check_blocksize_name('sha3_512', 72, 64) + self.check_blocksize_name('shake_128', 168, 0, 32) + self.check_blocksize_name('shake_256', 136, 0, 64) + + def check_sha3(self, name, capacity, rate, suffix): + constructors = self.constructors_to_test[name] + for hash_object_constructor in constructors: + m = hash_object_constructor() + self.assertEqual(capacity + rate, 1600) + self.assertEqual(m._capacity_bits, capacity) + self.assertEqual(m._rate_bits, rate) + self.assertEqual(m._suffix, suffix) + + @requires_sha3 + def test_extra_sha3(self): + self.check_sha3('sha3_224', 448, 1152, b'\x06') + self.check_sha3('sha3_256', 512, 1088, b'\x06') + self.check_sha3('sha3_384', 768, 832, b'\x06') + self.check_sha3('sha3_512', 1024, 576, b'\x06') + self.check_sha3('shake_128', 256, 1344, b'\x1f') + self.check_sha3('shake_256', 512, 1088, b'\x1f') + + @requires_blake2 + def test_blocksize_name_blake2(self): + self.check_blocksize_name('blake2b', 128, 64) + self.check_blocksize_name('blake2s', 64, 32) + def test_case_md5_0(self): self.check('md5', b'', 'd41d8cd98f00b204e9800998ecf8427e') @@ -373,6 +500,221 @@ class HashLibTestCase(unittest.TestCase): "e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973eb"+ "de0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b") + def check_blake2(self, constructor, salt_size, person_size, key_size, + digest_size, max_offset): + self.assertEqual(constructor.SALT_SIZE, salt_size) + for i in range(salt_size + 1): + constructor(salt=b'a' * i) + salt = b'a' * (salt_size + 1) + self.assertRaises(ValueError, constructor, salt=salt) + + self.assertEqual(constructor.PERSON_SIZE, person_size) + for i in range(person_size+1): + constructor(person=b'a' * i) + person = b'a' * (person_size + 1) + self.assertRaises(ValueError, constructor, person=person) + + self.assertEqual(constructor.MAX_DIGEST_SIZE, digest_size) + for i in range(1, digest_size + 1): + constructor(digest_size=i) + self.assertRaises(ValueError, constructor, digest_size=-1) + self.assertRaises(ValueError, constructor, digest_size=0) + self.assertRaises(ValueError, constructor, digest_size=digest_size+1) + + self.assertEqual(constructor.MAX_KEY_SIZE, key_size) + for i in range(key_size+1): + constructor(key=b'a' * i) + key = b'a' * (key_size + 1) + self.assertRaises(ValueError, constructor, key=key) + self.assertEqual(constructor().hexdigest(), + constructor(key=b'').hexdigest()) + + for i in range(0, 256): + constructor(fanout=i) + self.assertRaises(ValueError, constructor, fanout=-1) + self.assertRaises(ValueError, constructor, fanout=256) + + for i in range(1, 256): + constructor(depth=i) + self.assertRaises(ValueError, constructor, depth=-1) + self.assertRaises(ValueError, constructor, depth=0) + self.assertRaises(ValueError, constructor, depth=256) + + for i in range(0, 256): + constructor(node_depth=i) + self.assertRaises(ValueError, constructor, node_depth=-1) + self.assertRaises(ValueError, constructor, node_depth=256) + + for i in range(0, digest_size + 1): + constructor(inner_size=i) + self.assertRaises(ValueError, constructor, inner_size=-1) + self.assertRaises(ValueError, constructor, inner_size=digest_size+1) + + constructor(leaf_size=0) + constructor(leaf_size=(1<<32)-1) + self.assertRaises(OverflowError, constructor, leaf_size=-1) + self.assertRaises(OverflowError, constructor, leaf_size=1<<32) + + constructor(node_offset=0) + constructor(node_offset=max_offset) + self.assertRaises(OverflowError, constructor, node_offset=-1) + self.assertRaises(OverflowError, constructor, node_offset=max_offset+1) + + constructor( + string=b'', + key=b'', + salt=b'', + person=b'', + digest_size=17, + fanout=1, + depth=1, + leaf_size=256, + node_offset=512, + node_depth=1, + inner_size=7, + last_node=True + ) + + def blake2_rfc7693(self, constructor, md_len, in_len): + def selftest_seq(length, seed): + mask = (1<<32)-1 + a = (0xDEAD4BAD * seed) & mask + b = 1 + out = bytearray(length) + for i in range(length): + t = (a + b) & mask + a, b = b, t + out[i] = (t >> 24) & 0xFF + return out + outer = constructor(digest_size=32) + for outlen in md_len: + for inlen in in_len: + indata = selftest_seq(inlen, inlen) + key = selftest_seq(outlen, outlen) + unkeyed = constructor(indata, digest_size=outlen) + outer.update(unkeyed.digest()) + keyed = constructor(indata, key=key, digest_size=outlen) + outer.update(keyed.digest()) + return outer.hexdigest() + + @requires_blake2 + def test_blake2b(self): + self.check_blake2(hashlib.blake2b, 16, 16, 64, 64, (1<<64)-1) + b2b_md_len = [20, 32, 48, 64] + b2b_in_len = [0, 3, 128, 129, 255, 1024] + self.assertEqual( + self.blake2_rfc7693(hashlib.blake2b, b2b_md_len, b2b_in_len), + "c23a7800d98123bd10f506c61e29da5603d763b8bbad2e737f5e765a7bccd475") + + @requires_blake2 + def test_case_blake2b_0(self): + self.check('blake2b', b"", + "786a02f742015903c6c6fd852552d272912f4740e15847618a86e217f71f5419"+ + "d25e1031afee585313896444934eb04b903a685b1448b755d56f701afe9be2ce") + + @requires_blake2 + def test_case_blake2b_1(self): + self.check('blake2b', b"abc", + "ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d1"+ + "7d87c5392aab792dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923") + + @requires_blake2 + def test_blake2b_vectors(self): + for msg, key, md in read_vectors('blake2b'): + key = bytes.fromhex(key) + self.check('blake2b', msg, md, key=key) + + @requires_blake2 + def test_blake2s(self): + self.check_blake2(hashlib.blake2s, 8, 8, 32, 32, (1<<48)-1) + b2s_md_len = [16, 20, 28, 32] + b2s_in_len = [0, 3, 64, 65, 255, 1024] + self.assertEqual( + self.blake2_rfc7693(hashlib.blake2s, b2s_md_len, b2s_in_len), + "6a411f08ce25adcdfb02aba641451cec53c598b24f4fc787fbdc88797f4c1dfe") + + @requires_blake2 + def test_case_blake2s_0(self): + self.check('blake2s', b"", + "69217a3079908094e11121d042354a7c1f55b6482ca1a51e1b250dfd1ed0eef9") + + @requires_blake2 + def test_case_blake2s_1(self): + self.check('blake2s', b"abc", + "508c5e8c327c14e2e1a72ba34eeb452f37458b209ed63a294d999b4c86675982") + + @requires_blake2 + def test_blake2s_vectors(self): + for msg, key, md in read_vectors('blake2s'): + key = bytes.fromhex(key) + self.check('blake2s', msg, md, key=key) + + @requires_sha3 + def test_case_sha3_224_0(self): + self.check('sha3_224', b"", + "6b4e03423667dbb73b6e15454f0eb1abd4597f9a1b078e3f5b5a6bc7") + + @requires_sha3 + def test_case_sha3_224_vector(self): + for msg, md in read_vectors('sha3_224'): + self.check('sha3_224', msg, md) + + @requires_sha3 + def test_case_sha3_256_0(self): + self.check('sha3_256', b"", + "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a") + + @requires_sha3 + def test_case_sha3_256_vector(self): + for msg, md in read_vectors('sha3_256'): + self.check('sha3_256', msg, md) + + @requires_sha3 + def test_case_sha3_384_0(self): + self.check('sha3_384', b"", + "0c63a75b845e4f7d01107d852e4c2485c51a50aaaa94fc61995e71bbee983a2a"+ + "c3713831264adb47fb6bd1e058d5f004") + + @requires_sha3 + def test_case_sha3_384_vector(self): + for msg, md in read_vectors('sha3_384'): + self.check('sha3_384', msg, md) + + @requires_sha3 + def test_case_sha3_512_0(self): + self.check('sha3_512', b"", + "a69f73cca23a9ac5c8b567dc185a756e97c982164fe25859e0d1dcc1475c80a6"+ + "15b2123af1f5f94c11e3e9402c3ac558f500199d95b6d3e301758586281dcd26") + + @requires_sha3 + def test_case_sha3_512_vector(self): + for msg, md in read_vectors('sha3_512'): + self.check('sha3_512', msg, md) + + @requires_sha3 + def test_case_shake_128_0(self): + self.check('shake_128', b"", + "7f9c2ba4e88f827d616045507605853ed73b8093f6efbc88eb1a6eacfa66ef26", + True) + self.check('shake_128', b"", "7f9c", True) + + @requires_sha3 + def test_case_shake128_vector(self): + for msg, md in read_vectors('shake_128'): + self.check('shake_128', msg, md, True) + + @requires_sha3 + def test_case_shake_256_0(self): + self.check('shake_256', b"", + "46b9dd2b0ba88d13233b3feb743eeb243fcd52ea62b81b82b50c27646ed5762f", + True) + self.check('shake_256', b"", "46b9", True) + + @requires_sha3 + def test_case_shake256_vector(self): + for msg, md in read_vectors('shake_256'): + self.check('shake_256', msg, md, True) + def test_gil(self): # Check things work fine with an input larger than the size required # for multithreaded operation (which is hardwired to 2048). @@ -447,6 +789,12 @@ class KDFTests(unittest.TestCase): (b'pass\0word', b'sa\0lt', 4096, 16), ] + scrypt_test_vectors = [ + (b'', b'', 16, 1, 1, unhexlify('77d6576238657b203b19ca42c18a0497f16b4844e3074ae8dfdffa3fede21442fcd0069ded0948f8326a753a0fc81f17e8d3e0fb2e0d3628cf35e20c38d18906')), + (b'password', b'NaCl', 1024, 8, 16, unhexlify('fdbabe1c9d3472007856e7190d01e9fe7c6ad7cbc8237830e77376634b3731622eaf30d92e22a3886ff109279d9830dac727afb94a83ee6d8360cbdfa2cc0640')), + (b'pleaseletmein', b'SodiumChloride', 16384, 8, 1, unhexlify('7023bdcb3afd7348461c06cd81fd38ebfda8fbba904f8e3ea9b543f6545da1f2d5432955613f0fcf62d49705242a9af9e61e85dc0d651e40dfcf017b45575887')), + ] + pbkdf2_results = { "sha1": [ # official test vectors from RFC 6070 @@ -526,5 +874,45 @@ class KDFTests(unittest.TestCase): self._test_pbkdf2_hmac(c_hashlib.pbkdf2_hmac) + @unittest.skipUnless(hasattr(c_hashlib, 'scrypt'), + ' test requires OpenSSL > 1.1') + def test_scrypt(self): + for password, salt, n, r, p, expected in self.scrypt_test_vectors: + result = hashlib.scrypt(password, salt=salt, n=n, r=r, p=p) + self.assertEqual(result, expected) + + # this values should work + hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1) + # password and salt must be bytes-like + with self.assertRaises(TypeError): + hashlib.scrypt('password', salt=b'salt', n=2, r=8, p=1) + with self.assertRaises(TypeError): + hashlib.scrypt(b'password', salt='salt', n=2, r=8, p=1) + # require keyword args + with self.assertRaises(TypeError): + hashlib.scrypt(b'password') + with self.assertRaises(TypeError): + hashlib.scrypt(b'password', b'salt') + with self.assertRaises(TypeError): + hashlib.scrypt(b'password', 2, 8, 1, salt=b'salt') + for n in [-1, 0, 1, None]: + with self.assertRaises((ValueError, OverflowError, TypeError)): + hashlib.scrypt(b'password', salt=b'salt', n=n, r=8, p=1) + for r in [-1, 0, None]: + with self.assertRaises((ValueError, OverflowError, TypeError)): + hashlib.scrypt(b'password', salt=b'salt', n=2, r=r, p=1) + for p in [-1, 0, None]: + with self.assertRaises((ValueError, OverflowError, TypeError)): + hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=p) + for maxmem in [-1, None]: + with self.assertRaises((ValueError, OverflowError, TypeError)): + hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1, + maxmem=maxmem) + for dklen in [-1, None]: + with self.assertRaises((ValueError, OverflowError, TypeError)): + hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1, + dklen=dklen) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py index b7e8259f8d..2f8c648d84 100644 --- a/Lib/test/test_heapq.py +++ b/Lib/test/test_heapq.py @@ -1,6 +1,5 @@ """Unittests for heapq.""" -import sys import random import unittest diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 98826b5a60..067e13f107 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -3,7 +3,6 @@ import hmac import hashlib import unittest import warnings -from test import support def ignore_warning(func): diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 11420b2c84..326e34290f 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -3,7 +3,6 @@ import html.parser import pprint import unittest -from test import support class EventCollector(html.parser.HTMLParser): @@ -702,7 +701,7 @@ class AttributesTestCase(TestCaseBase): def test_attr_funky_names2(self): self._run_check( - "<a $><b $=%><c \=/>", + r"<a $><b $=%><c \=/>", [("starttag", "a", [("$", None)]), ("starttag", "b", [("$", "%")]), ("starttag", "c", [("\\", "/")])]) diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index 49c01ae489..6fee4df10a 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -1051,7 +1051,7 @@ class CookieTests(unittest.TestCase): url = "http://foo.bar.com/" interact_2965(c, url, "spam=eggs; Version=1; Port") h = interact_2965(c, url) - self.assertRegex(h, "\$Port([^=]|$)", + self.assertRegex(h, r"\$Port([^=]|$)", "port with no value not returned with no value") c = CookieJar(pol) @@ -1396,9 +1396,9 @@ class LWPCookieTests(unittest.TestCase): self.assertRegex(cookie, r'^\$Version="?1"?;') self.assertRegex(cookie, r'Part_Number="?Rocket_Launcher_0001"?;' - '\s*\$Path="\/acme"') + r'\s*\$Path="\/acme"') self.assertRegex(cookie, r'Customer="?WILE_E_COYOTE"?;' - '\s*\$Path="\/acme"') + r'\s*\$Path="\/acme"') # # 7. User Agent -> Server diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 61ed6bbbd7..68f6946a3a 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -344,6 +344,134 @@ class HeaderTests(TestCase): conn.putheader(name, value) +class TransferEncodingTest(TestCase): + expected_body = b"It's just a flesh wound" + + def test_endheaders_chunked(self): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.putrequest('POST', '/') + conn.endheaders(self._make_body(), encode_chunked=True) + + _, _, body = self._parse_request(conn.sock.data) + body = self._parse_chunked(body) + self.assertEqual(body, self.expected_body) + + def test_explicit_headers(self): + # explicit chunked + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + # this shouldn't actually be automatically chunk-encoded because the + # calling code has explicitly stated that it's taking care of it + conn.request( + 'POST', '/', self._make_body(), {'Transfer-Encoding': 'chunked'}) + + _, headers, body = self._parse_request(conn.sock.data) + self.assertNotIn('content-length', [k.lower() for k in headers.keys()]) + self.assertEqual(headers['Transfer-Encoding'], 'chunked') + self.assertEqual(body, self.expected_body) + + # explicit chunked, string body + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.request( + 'POST', '/', self.expected_body.decode('latin-1'), + {'Transfer-Encoding': 'chunked'}) + + _, headers, body = self._parse_request(conn.sock.data) + self.assertNotIn('content-length', [k.lower() for k in headers.keys()]) + self.assertEqual(headers['Transfer-Encoding'], 'chunked') + self.assertEqual(body, self.expected_body) + + # User-specified TE, but request() does the chunk encoding + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.request('POST', '/', + headers={'Transfer-Encoding': 'gzip, chunked'}, + encode_chunked=True, + body=self._make_body()) + _, headers, body = self._parse_request(conn.sock.data) + self.assertNotIn('content-length', [k.lower() for k in headers]) + self.assertEqual(headers['Transfer-Encoding'], 'gzip, chunked') + self.assertEqual(self._parse_chunked(body), self.expected_body) + + def test_request(self): + for empty_lines in (False, True,): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.request( + 'POST', '/', self._make_body(empty_lines=empty_lines)) + + _, headers, body = self._parse_request(conn.sock.data) + body = self._parse_chunked(body) + self.assertEqual(body, self.expected_body) + self.assertEqual(headers['Transfer-Encoding'], 'chunked') + + # Content-Length and Transfer-Encoding SHOULD not be sent in the + # same request + self.assertNotIn('content-length', [k.lower() for k in headers]) + + def test_empty_body(self): + # Zero-length iterable should be treated like any other iterable + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.request('POST', '/', ()) + _, headers, body = self._parse_request(conn.sock.data) + self.assertEqual(headers['Transfer-Encoding'], 'chunked') + self.assertNotIn('content-length', [k.lower() for k in headers]) + self.assertEqual(body, b"0\r\n\r\n") + + def _make_body(self, empty_lines=False): + lines = self.expected_body.split(b' ') + for idx, line in enumerate(lines): + # for testing handling empty lines + if empty_lines and idx % 2: + yield b'' + if idx < len(lines) - 1: + yield line + b' ' + else: + yield line + + def _parse_request(self, data): + lines = data.split(b'\r\n') + request = lines[0] + headers = {} + n = 1 + while n < len(lines) and len(lines[n]) > 0: + key, val = lines[n].split(b':') + key = key.decode('latin-1').strip() + headers[key] = val.decode('latin-1').strip() + n += 1 + + return request, headers, b'\r\n'.join(lines[n + 1:]) + + def _parse_chunked(self, data): + body = [] + trailers = {} + n = 0 + lines = data.split(b'\r\n') + # parse body + while True: + size, chunk = lines[n:n+2] + size = int(size, 16) + + if size == 0: + n += 1 + break + + self.assertEqual(size, len(chunk)) + body.append(chunk) + + n += 2 + # we /should/ hit the end chunk, but check against the size of + # lines so we're not stuck in an infinite loop should we get + # malformed data + if n > len(lines): + break + + return b''.join(body) + + class BasicTest(TestCase): def test_status_lines(self): # Test HTTP status lines @@ -564,7 +692,9 @@ class BasicTest(TestCase): def test_send_file(self): expected = (b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' - b'Accept-Encoding: identity\r\nContent-Length:') + b'Accept-Encoding: identity\r\n' + b'Transfer-Encoding: chunked\r\n' + b'\r\n') with open(__file__, 'rb') as body: conn = client.HTTPConnection('example.com') @@ -594,11 +724,11 @@ class BasicTest(TestCase): yield None yield 'data_two' - class UpdatingFile(): + class UpdatingFile(io.TextIOBase): mode = 'r' d = data() def read(self, blocksize=-1): - return self.d.__next__() + return next(self.d) expected = b'data' @@ -970,6 +1100,7 @@ class BasicTest(TestCase): thread = threading.Thread(target=run_server) thread.start() + self.addCleanup(thread.join, float(1)) conn = client.HTTPConnection(*serv.getsockname()) conn.request("CONNECT", "dummy:1234") response = conn.getresponse() @@ -983,7 +1114,7 @@ class BasicTest(TestCase): finally: response.close() conn.close() - thread.join() + thread.join() self.assertEqual(result, b"proxied data\n") class ExtendedReadTest(TestCase): @@ -1511,14 +1642,16 @@ class HTTPSTest(TestCase): with self.assertRaises(ssl.CertificateError): h.request('GET', '/') # Same with explicit check_hostname=True - h = client.HTTPSConnection('localhost', server.port, context=context, - check_hostname=True) + with support.check_warnings(('', DeprecationWarning)): + h = client.HTTPSConnection('localhost', server.port, + context=context, check_hostname=True) with self.assertRaises(ssl.CertificateError): h.request('GET', '/') # With check_hostname=False, the mismatching is ignored context.check_hostname = False - h = client.HTTPSConnection('localhost', server.port, context=context, - check_hostname=False) + with support.check_warnings(('', DeprecationWarning)): + h = client.HTTPSConnection('localhost', server.port, + context=context, check_hostname=False) h.request('GET', '/nonexistent') resp = h.getresponse() resp.close() @@ -1535,8 +1668,9 @@ class HTTPSTest(TestCase): h.close() # Passing check_hostname to HTTPSConnection should override the # context's setting. - h = client.HTTPSConnection('localhost', server.port, context=context, - check_hostname=True) + with support.check_warnings(('', DeprecationWarning)): + h = client.HTTPSConnection('localhost', server.port, + context=context, check_hostname=True) with self.assertRaises(ssl.CertificateError): h.request('GET', '/') @@ -1575,6 +1709,26 @@ class RequestBodyTest(TestCase): message = client.parse_headers(f) return message, f + def test_list_body(self): + # Note that no content-length is automatically calculated for + # an iterable. The request will fall back to send chunked + # transfer encoding. + cases = ( + ([b'foo', b'bar'], b'3\r\nfoo\r\n3\r\nbar\r\n0\r\n\r\n'), + ((b'foo', b'bar'), b'3\r\nfoo\r\n3\r\nbar\r\n0\r\n\r\n'), + ) + for body, expected in cases: + with self.subTest(body): + self.conn = client.HTTPConnection('example.com') + self.conn.sock = self.sock = FakeSocket('') + + self.conn.request('PUT', '/url', body) + msg, f = self.get_headers_and_fp() + self.assertNotIn('Content-Type', msg) + self.assertNotIn('Content-Length', msg) + self.assertEqual(msg.get('Transfer-Encoding'), 'chunked') + self.assertEqual(expected, f.read()) + def test_manual_content_length(self): # Set an incorrect content-length so that we can verify that # it will not be over-ridden by the library. @@ -1608,7 +1762,7 @@ class RequestBodyTest(TestCase): self.assertEqual("5", message.get("content-length")) self.assertEqual(b'body\xc1', f.read()) - def test_file_body(self): + def test_text_file_body(self): self.addCleanup(support.unlink, support.TESTFN) with open(support.TESTFN, "w") as f: f.write("body") @@ -1617,8 +1771,11 @@ class RequestBodyTest(TestCase): message, f = self.get_headers_and_fp() self.assertEqual("text/plain", message.get_content_type()) self.assertIsNone(message.get_charset()) - self.assertEqual("4", message.get("content-length")) - self.assertEqual(b'body', f.read()) + # No content-length will be determined for files; the body + # will be sent using chunked transfer encoding instead. + self.assertIsNone(message.get("content-length")) + self.assertEqual("chunked", message.get("transfer-encoding")) + self.assertEqual(b'4\r\nbody\r\n0\r\n\r\n', f.read()) def test_binary_file_body(self): self.addCleanup(support.unlink, support.TESTFN) @@ -1629,8 +1786,9 @@ class RequestBodyTest(TestCase): message, f = self.get_headers_and_fp() self.assertEqual("text/plain", message.get_content_type()) self.assertIsNone(message.get_charset()) - self.assertEqual("5", message.get("content-length")) - self.assertEqual(b'body\xc1', f.read()) + self.assertEqual("chunked", message.get("Transfer-Encoding")) + self.assertNotIn("Content-Length", message) + self.assertEqual(b'5\r\nbody\xc1\r\n0\r\n\r\n', f.read()) class HTTPResponseTest(TestCase): @@ -1741,13 +1899,5 @@ class TunnelTests(TestCase): self.assertIn('header: {}'.format(expected_header), lines) -@support.reap_threads -def test_main(verbose=None): - support.run_unittest(HeaderTests, OfflineTest, BasicTest, TimeoutTest, - PersistenceTest, - HTTPSTest, RequestBodyTest, SourceAddressTest, - HTTPResponseTest, ExtendedReadTest, - ExtendedReadTestChunked, TunnelTests) - if __name__ == '__main__': - test_main() + unittest.main(verbosity=2) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 72e6e08880..4e931446b9 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -18,6 +18,7 @@ import urllib.parse import html import http.client import tempfile +import time from io import BytesIO import unittest @@ -189,7 +190,7 @@ class BaseHTTPServerTestCase(BaseTestCase): res = self.con.getresponse() self.assertEqual(res.status, HTTPStatus.NOT_IMPLEMENTED) - def test_head_keep_alive(self): + def test_header_keep_alive(self): self.con._http_vsn_str = 'HTTP/1.1' self.con.putrequest('GET', '/') self.con.putheader('Connection', 'keep-alive') @@ -369,6 +370,8 @@ class SimpleHTTPServerTestCase(BaseTestCase): return body @support.requires_mac_ver(10, 5) + @unittest.skipIf(sys.platform == 'win32', + 'undecodable name cannot be decoded on win32') @unittest.skipUnless(support.TESTFN_UNDECODABLE, 'need support.TESTFN_UNDECODABLE') def test_undecodable_filename(self): @@ -388,7 +391,7 @@ class SimpleHTTPServerTestCase(BaseTestCase): quotedname = urllib.parse.quote(filename, errors='surrogatepass') self.assertIn(('href="%s"' % quotedname) .encode(enc, 'surrogateescape'), body) - self.assertIn(('>%s<' % html.escape(filename)) + self.assertIn(('>%s<' % html.escape(filename, quote=False)) .encode(enc, 'surrogateescape'), body) response = self.request(self.base_url + '/' + quotedname) self.check_status_and_reason(response, HTTPStatus.OK, @@ -466,6 +469,27 @@ class SimpleHTTPServerTestCase(BaseTestCase): self.assertEqual(response.getheader("Location"), self.tempdir_name + "/?hi=1") + def test_html_escape_filename(self): + filename = '<test&>.txt' + fullpath = os.path.join(self.tempdir, filename) + + try: + open(fullpath, 'w').close() + except OSError: + raise unittest.SkipTest('Can not create file %s on current file ' + 'system' % filename) + + try: + response = self.request(self.base_url + '/') + body = self.check_status_and_reason(response, HTTPStatus.OK) + enc = response.headers.get_content_charset() + finally: + os.unlink(fullpath) # avoid affecting test_undecodable_filename + + self.assertIsNotNone(enc) + html_text = '>%s<' % html.escape(filename, quote=False) + self.assertIn(html_text.encode(enc), body) + cgi_file1 = """\ #!%s @@ -916,7 +940,7 @@ class BaseHTTPRequestHandlerTestCase(unittest.TestCase): # Issue #6791: same for headers result = self.send_typical_request( b'GET / HTTP/1.1\r\nX-Foo: bar' + b'r' * 65537 + b'\r\n\r\n') - self.assertEqual(result[0], b'HTTP/1.1 400 Line too long\r\n') + self.assertEqual(result[0], b'HTTP/1.1 431 Line too long\r\n') self.assertFalse(self.handler.get_called) self.assertEqual(self.handler.requestline, 'GET / HTTP/1.1') @@ -927,6 +951,13 @@ class BaseHTTPRequestHandlerTestCase(unittest.TestCase): self.assertFalse(self.handler.get_called) self.assertEqual(self.handler.requestline, 'GET / HTTP/1.1') + def test_html_escape_on_error(self): + result = self.send_typical_request( + b'<script>alert("hello")</script> / HTTP/1.1') + result = b''.join(result) + text = '<script>alert("hello")</script>' + self.assertIn(html.escape(text, quote=False).encode('ascii'), result) + def test_close_connection(self): # handle_one_request() should be repeatedly called until # it sets close_connection @@ -942,6 +973,19 @@ class BaseHTTPRequestHandlerTestCase(unittest.TestCase): self.handler.handle() self.assertRaises(StopIteration, next, close_values) + def test_date_time_string(self): + now = time.time() + # this is the old code that formats the timestamp + year, month, day, hh, mm, ss, wd, y, z = time.gmtime(now) + expected = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( + self.handler.weekdayname[wd], + day, + self.handler.monthname[month], + year, hh, mm, ss + ) + self.assertEqual(self.handler.date_time_string(timestamp=now), expected) + + class SimpleHTTPRequestHandlerTestCase(unittest.TestCase): """ Test url parsing """ def setUp(self): diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index 141e89e493..da05da50f3 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -1,16 +1,23 @@ import unittest -from test import support from test.support import import_module -# Skip test if _thread or _tkinter wasn't built or idlelib was deleted. +# Skip test if _thread or _tkinter wasn't built, if idlelib is missing, +# or if tcl/tk is not the 8.5+ needed for ttk widgets. import_module('threading') # imported by PyShell, imports _thread tk = import_module('tkinter') # imports _tkinter -idletest = import_module('idlelib.idle_test') +if tk.TkVersion < 8.5: + raise unittest.SkipTest("IDLE requires tk 8.5 or later.") +idlelib = import_module('idlelib') -# Without test_main present, regrtest.runtest_inner (line1219) calls -# unittest.TestLoader().loadTestsFromModule(this_module) which calls -# load_tests() if it finds it. (Unittest.main does the same.) -load_tests = idletest.load_tests +# Before test imports, tell IDLE to avoid changing the environment. +idlelib.testing = True + +# unittest.main and test.libregrtest.runtest.runtest_inner +# call load_tests, when present, to discover tests to run. +from idlelib.idle_test import load_tests if __name__ == '__main__': - unittest.main(verbosity=2, exit=False) + tk.NoDefaultRoot() + unittest.main(exit=False) + tk._support_default_root = 1 + tk._default_root = None diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index a29b40ac23..fac6b57081 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -79,9 +79,9 @@ if ssl: def get_request(self): newsocket, fromaddr = self.socket.accept() - connstream = ssl.wrap_socket(newsocket, - server_side=True, - certfile=CERTFILE) + context = ssl.SSLContext() + context.load_cert_chain(CERTFILE) + connstream = context.wrap_socket(newsocket, server_side=True) return connstream, fromaddr IMAP4_SSL = imaplib.IMAP4_SSL @@ -573,6 +573,55 @@ class ThreadedNetworkedTests(unittest.TestCase): client.shutdown() @reap_threads + def test_bracket_flags(self): + + # This violates RFC 3501, which disallows ']' characters in tag names, + # but imaplib has allowed producing such tags forever, other programs + # also produce them (eg: OtherInbox's Organizer app as of 20140716), + # and Gmail, for example, accepts them and produces them. So we + # support them. See issue #21815. + + class BracketFlagHandler(SimpleIMAPHandler): + + def handle(self): + self.flags = ['Answered', 'Flagged', 'Deleted', 'Seen', 'Draft'] + super().handle() + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + + def cmd_SELECT(self, tag, args): + flag_msg = ' \\'.join(self.flags) + self._send_line(('* FLAGS (%s)' % flag_msg).encode('ascii')) + self._send_line(b'* 2 EXISTS') + self._send_line(b'* 0 RECENT') + msg = ('* OK [PERMANENTFLAGS %s \\*)] Flags permitted.' + % flag_msg) + self._send_line(msg.encode('ascii')) + self._send_tagged(tag, 'OK', '[READ-WRITE] SELECT completed.') + + def cmd_STORE(self, tag, args): + new_flags = args[2].strip('(').strip(')').split() + self.flags.extend(new_flags) + flags_msg = '(FLAGS (%s))' % ' \\'.join(self.flags) + msg = '* %s FETCH %s' % (args[0], flags_msg) + self._send_line(msg.encode('ascii')) + self._send_tagged(tag, 'OK', 'STORE completed.') + + with self.reaped_pair(BracketFlagHandler) as (server, client): + code, data = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, b'ZmFrZQ==\r\n') + client.select('test') + typ, [data] = client.store(b'1', "+FLAGS", "[test]") + self.assertIn(b'[test]', data) + client.select('test') + typ, [data] = client.response('PERMANENTFLAGS') + self.assertIn(b'[test]', data) + + @reap_threads def test_issue5949(self): class EOFHandler(socketserver.StreamRequestHandler): @@ -919,8 +968,10 @@ class RemoteIMAP_SSLTest(RemoteIMAPTest): def test_logincapa_with_client_certfile(self): with transient_internet(self.host): - _server = self.imap_class(self.host, self.port, certfile=CERTFILE) - self.check_logincapa(_server) + with support.check_warnings(('', DeprecationWarning)): + _server = self.imap_class(self.host, self.port, + certfile=CERTFILE) + self.check_logincapa(_server) def test_logincapa_with_client_ssl_context(self): with transient_internet(self.host): diff --git a/Lib/test/test_imghdr.py b/Lib/test/test_imghdr.py index b54daf8e2c..476ba95f17 100644 --- a/Lib/test/test_imghdr.py +++ b/Lib/test/test_imghdr.py @@ -1,6 +1,7 @@ import imghdr import io import os +import pathlib import unittest import warnings from test.support import findfile, TESTFN, unlink @@ -49,6 +50,12 @@ class TestImghdr(unittest.TestCase): self.assertEqual(imghdr.what(None, data), expected) self.assertEqual(imghdr.what(None, bytearray(data)), expected) + def test_pathlike_filename(self): + for filename, expected in TEST_FILES: + with self.subTest(filename=filename): + filename = findfile(filename, subdir='imghdrdata') + self.assertEqual(imghdr.what(pathlib.Path(filename)), expected) + def test_register_test(self): def test_jumbo(h, file): if h.startswith(b'eggs'): diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index ee9ee1ad8c..4ece365410 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -6,13 +6,12 @@ import importlib import importlib.util import os import os.path -import shutil import sys from test import support import unittest import warnings with warnings.catch_warnings(): - warnings.simplefilter('ignore', PendingDeprecationWarning) + warnings.simplefilter('ignore', DeprecationWarning) import imp diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 1e33274b87..760908efe6 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -69,6 +69,18 @@ class ImportTests(unittest.TestCase): def tearDown(self): unload(TESTFN) + def test_import_raises_ModuleNotFoundError(self): + with self.assertRaises(ModuleNotFoundError): + import something_that_should_not_exist_anywhere + + def test_from_import_missing_module_raises_ModuleNotFoundError(self): + with self.assertRaises(ModuleNotFoundError): + from something_that_should_not_exist_anywhere import blah + + def test_from_import_missing_attr_raises_ImportError(self): + with self.assertRaises(ImportError): + from importlib import something_that_should_not_exist_anywhere + def test_case_sensitivity(self): # Brief digression to test that import is case-sensitive: if we got # this far, we know for sure that "random" exists. diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py index c112ca7564..0dd9c8615f 100644 --- a/Lib/test/test_importlib/extension/test_case_sensitivity.py +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -1,5 +1,4 @@ from importlib import _bootstrap_external -import sys from test import support import unittest @@ -9,8 +8,6 @@ importlib = util.import_importlib('importlib') machinery = util.import_importlib('importlib.machinery') -# XXX find_spec tests - @unittest.skipIf(util.EXTENSIONS.filename is None, '_testcapi not available') @util.case_insensitive_tests class ExtensionModuleCaseSensitivityTest(util.CASEOKTestBase): diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index 71bf67febd..c9b4a3772c 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -6,7 +6,6 @@ machinery = util.import_importlib('importlib.machinery') import unittest import warnings -# XXX find_spec tests class FinderTests(abc.FinderTests): diff --git a/Lib/test/test_importlib/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py index 8f4b8bb161..a4b5a64aae 100644 --- a/Lib/test/test_importlib/extension/test_path_hook.py +++ b/Lib/test/test_importlib/extension/test_path_hook.py @@ -2,8 +2,6 @@ from .. import util machinery = util.import_importlib('importlib.machinery') -import collections -import sys import unittest diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index 603c7d7b19..29ecff1774 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -3,8 +3,6 @@ from .. import util machinery = util.import_importlib('importlib.machinery') - -import sys from test.support import captured_stdout import types import unittest diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index c7d3a2a204..7f64548748 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -5,6 +5,7 @@ of using the typical __path__/__name__ test). """ import unittest +import warnings from .. import util @@ -33,31 +34,50 @@ class Using__package__: """ - def test_using___package__(self): - # [__package__] + def import_module(self, globals_): with self.mock_modules('pkg.__init__', 'pkg.fake') as importer: with util.import_state(meta_path=[importer]): self.__import__('pkg.fake') module = self.__import__('', - globals={'__package__': 'pkg.fake'}, - fromlist=['attr'], level=2) + globals=globals_, + fromlist=['attr'], level=2) + return module + + def test_using___package__(self): + # [__package__] + module = self.import_module({'__package__': 'pkg.fake'}) self.assertEqual(module.__name__, 'pkg') - def test_using___name__(self, package_as_None=False): + def test_using___name__(self): # [__name__] - globals_ = {'__name__': 'pkg.fake', '__path__': []} - if package_as_None: - globals_['__package__'] = None - with self.mock_modules('pkg.__init__', 'pkg.fake') as importer: - with util.import_state(meta_path=[importer]): - self.__import__('pkg.fake') - module = self.__import__('', globals= globals_, - fromlist=['attr'], level=2) - self.assertEqual(module.__name__, 'pkg') + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + module = self.import_module({'__name__': 'pkg.fake', + '__path__': []}) + self.assertEqual(module.__name__, 'pkg') + + def test_warn_when_using___name__(self): + with self.assertWarns(ImportWarning): + self.import_module({'__name__': 'pkg.fake', '__path__': []}) def test_None_as___package__(self): # [None] - self.test_using___name__(package_as_None=True) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + module = self.import_module({ + '__name__': 'pkg.fake', '__path__': [], '__package__': None }) + self.assertEqual(module.__name__, 'pkg') + + def test_spec_fallback(self): + # If __package__ isn't defined, fall back on __spec__.parent. + module = self.import_module({'__spec__': FakeSpec('pkg.fake')}) + self.assertEqual(module.__name__, 'pkg') + + def test_warn_when_package_and_spec_disagree(self): + # Raise an ImportWarning if __package__ != __spec__.parent. + with self.assertWarns(ImportWarning): + self.import_module({'__package__': 'pkg.fake', + '__spec__': FakeSpec('pkg.fakefake')}) def test_bad__package__(self): globals = {'__package__': '<not real>'} @@ -70,6 +90,11 @@ class Using__package__: self.__import__('', globals, {}, ['relimport'], 1) +class FakeSpec: + def __init__(self, parent): + self.parent = parent + + class Using__package__PEP302(Using__package__): mock_modules = util.mock_modules diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py index 7069d9e58d..a7bf2749cf 100644 --- a/Lib/test/test_importlib/import_/test_api.py +++ b/Lib/test/test_importlib/import_/test_api.py @@ -43,6 +43,10 @@ class APITest: """Test API-specific details for __import__ (e.g. raising the right exception when passing in an int for the module name).""" + def test_raises_ModuleNotFoundError(self): + with self.assertRaises(ModuleNotFoundError): + util.import_importlib('some module that does not exist') + def test_name_requires_rparition(self): # Raise TypeError if a non-string is passed in for the module name. with self.assertRaises(TypeError): diff --git a/Lib/test/test_importlib/import_/test_fromlist.py b/Lib/test/test_importlib/import_/test_fromlist.py index 80454655ad..14640032b4 100644 --- a/Lib/test/test_importlib/import_/test_fromlist.py +++ b/Lib/test/test_importlib/import_/test_fromlist.py @@ -73,16 +73,16 @@ class HandlingFromlist: self.assertTrue(hasattr(module, 'module')) self.assertEqual(module.module.__name__, 'pkg.module') - def test_module_from_package_triggers_ImportError(self): - # If a submodule causes an ImportError because it tries to import - # a module which doesn't exist, that should let the ImportError - # propagate. + def test_module_from_package_triggers_ModuleNotFoundError(self): + # If a submodule causes an ModuleNotFoundError because it tries + # to import a module which doesn't exist, that should let the + # ModuleNotFoundError propagate. def module_code(): import i_do_not_exist with util.mock_modules('pkg.__init__', 'pkg.mod', module_code={'pkg.mod': module_code}) as importer: with util.import_state(meta_path=[importer]): - with self.assertRaises(ImportError) as exc: + with self.assertRaises(ModuleNotFoundError) as exc: self.__import__('pkg', fromlist=['mod']) self.assertEqual('i_do_not_exist', exc.exception.name) diff --git a/Lib/test/test_importlib/import_/test_packages.py b/Lib/test/test_importlib/import_/test_packages.py index 3755b84a1a..24396044a5 100644 --- a/Lib/test/test_importlib/import_/test_packages.py +++ b/Lib/test/test_importlib/import_/test_packages.py @@ -1,7 +1,6 @@ from .. import util import sys import unittest -import importlib from test import support diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py index b32a876f89..7aa26b0eee 100644 --- a/Lib/test/test_importlib/import_/test_path.py +++ b/Lib/test/test_importlib/import_/test_path.py @@ -16,11 +16,14 @@ class FinderTests: """Tests for PathFinder.""" + find = None + check_found = None + def test_failure(self): # Test None returned upon not finding a suitable loader. module = '<test module>' with util.import_state(): - self.assertIsNone(self.machinery.PathFinder.find_module(module)) + self.assertIsNone(self.find(module)) def test_sys_path(self): # Test that sys.path is used when 'path' is None. @@ -30,8 +33,8 @@ class FinderTests: importer = util.mock_spec(module) with util.import_state(path_importer_cache={path: importer}, path=[path]): - loader = self.machinery.PathFinder.find_module(module) - self.assertIs(loader, importer) + found = self.find(module) + self.check_found(found, importer) def test_path(self): # Test that 'path' is used when set. @@ -40,8 +43,8 @@ class FinderTests: path = '<test path>' importer = util.mock_spec(module) with util.import_state(path_importer_cache={path: importer}): - loader = self.machinery.PathFinder.find_module(module, [path]) - self.assertIs(loader, importer) + found = self.find(module, [path]) + self.check_found(found, importer) def test_empty_list(self): # An empty list should not count as asking for sys.path. @@ -50,7 +53,7 @@ class FinderTests: importer = util.mock_spec(module) with util.import_state(path_importer_cache={path: importer}, path=[path]): - self.assertIsNone(self.machinery.PathFinder.find_module('module', [])) + self.assertIsNone(self.find('module', [])) def test_path_hooks(self): # Test that sys.path_hooks is used. @@ -60,8 +63,8 @@ class FinderTests: importer = util.mock_spec(module) hook = util.mock_path_hook(path, importer=importer) with util.import_state(path_hooks=[hook]): - loader = self.machinery.PathFinder.find_module(module, [path]) - self.assertIs(loader, importer) + found = self.find(module, [path]) + self.check_found(found, importer) self.assertIn(path, sys.path_importer_cache) self.assertIs(sys.path_importer_cache[path], importer) @@ -73,7 +76,7 @@ class FinderTests: path=[path_entry]): with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') - self.assertIsNone(self.machinery.PathFinder.find_module('os')) + self.assertIsNone(self.find('os')) self.assertIsNone(sys.path_importer_cache[path_entry]) self.assertEqual(len(w), 1) self.assertTrue(issubclass(w[-1].category, ImportWarning)) @@ -85,8 +88,8 @@ class FinderTests: importer = util.mock_spec(module) hook = util.mock_path_hook(os.getcwd(), importer=importer) with util.import_state(path=[path], path_hooks=[hook]): - loader = self.machinery.PathFinder.find_module(module) - self.assertIs(loader, importer) + found = self.find(module) + self.check_found(found, importer) self.assertIn(os.getcwd(), sys.path_importer_cache) def test_None_on_sys_path(self): @@ -182,16 +185,33 @@ class FinderTests: self.assertIsNone(self.machinery.PathFinder.find_spec('whatever')) +class FindModuleTests(FinderTests): + def find(self, *args, **kwargs): + return self.machinery.PathFinder.find_module(*args, **kwargs) + def check_found(self, found, importer): + self.assertIs(found, importer) + + +(Frozen_FindModuleTests, + Source_FindModuleTests +) = util.test_both(FindModuleTests, importlib=importlib, machinery=machinery) -(Frozen_FinderTests, - Source_FinderTests - ) = util.test_both(FinderTests, importlib=importlib, machinery=machinery) +class FindSpecTests(FinderTests): + def find(self, *args, **kwargs): + return self.machinery.PathFinder.find_spec(*args, **kwargs) + def check_found(self, found, importer): + self.assertIs(found.loader, importer) + + +(Frozen_FindSpecTests, + Source_FindSpecTests + ) = util.test_both(FindSpecTests, importlib=importlib, machinery=machinery) class PathEntryFinderTests: - def test_finder_with_failing_find_module(self): + def test_finder_with_failing_find_spec(self): # PathEntryFinder with find_module() defined should work. # Issue #20763. class Finder: @@ -209,6 +229,24 @@ class PathEntryFinderTests: path_hooks=[Finder]): self.machinery.PathFinder.find_spec('importlib') + def test_finder_with_failing_find_module(self): + # PathEntryFinder with find_module() defined should work. + # Issue #20763. + class Finder: + path_location = 'test_finder_with_find_module' + def __init__(self, path): + if path != self.path_location: + raise ImportError + + @staticmethod + def find_module(fullname): + return None + + + with util.import_state(path=[Finder.path_location]+sys.path[:], + path_hooks=[Finder]): + self.machinery.PathFinder.find_module('importlib') + (Frozen_PEFTests, Source_PEFTests diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index 3bb819f906..8a95a32109 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -1,7 +1,8 @@ """Test relative imports (PEP 328).""" from .. import util -import sys import unittest +import warnings + class RelativeImports: @@ -65,9 +66,11 @@ class RelativeImports: uncache_names.append(name[:-len('.__init__')]) with util.mock_spec(*create) as importer: with util.import_state(meta_path=[importer]): - for global_ in globals_: - with util.uncache(*uncache_names): - callback(global_) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + for global_ in globals_: + with util.uncache(*uncache_names): + callback(global_) def test_module_from_module(self): @@ -204,11 +207,18 @@ class RelativeImports: def test_relative_import_no_globals(self): # No globals for a relative import is an error. - with self.assertRaises(KeyError): - self.__import__('sys', level=1) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with self.assertRaises(KeyError): + self.__import__('sys', level=1) + + def test_relative_import_no_package(self): + with self.assertRaises(ImportError): + self.__import__('a', {'__package__': '', '__spec__': None}, + level=1) def test_relative_import_no_package_exists_absolute(self): - with self.assertRaises(SystemError): + with self.assertRaises(ImportError): self.__import__('sys', {'__package__': '', '__spec__': None}, level=1) diff --git a/Lib/test/test_importlib/regrtest.py b/Lib/test/test_importlib/regrtest.py deleted file mode 100644 index a5be11fd4e..0000000000 --- a/Lib/test/test_importlib/regrtest.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Run Python's standard test suite using importlib.__import__. - -Tests known to fail because of assumptions that importlib (properly) -invalidates are automatically skipped if the entire test suite is run. -Otherwise all command-line options valid for test.regrtest are also valid for -this script. - -""" -import importlib -import sys -from test import regrtest - -if __name__ == '__main__': - __builtins__.__import__ = importlib.__import__ - sys.path_importer_cache.clear() - - regrtest.main(quiet=True, verbose2=True) diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py index 34b86cddd1..12ce0cb934 100644 --- a/Lib/test/test_importlib/source/test_case_sensitivity.py +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -5,7 +5,6 @@ importlib = util.import_importlib('importlib') machinery = util.import_importlib('importlib.machinery') import os -import sys from test import support as test_support import unittest diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py index 73f4c62070..a151149f31 100644 --- a/Lib/test/test_importlib/source/test_file_loader.py +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -217,7 +217,7 @@ class SimpleTest(abc.LoaderTests): # PEP 302 with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) - mod = loader.load_module('_temp') # XXX + mod = loader.load_module('_temp') # Sanity checks. self.assertEqual(mod.__cached__, compiled) self.assertEqual(mod.x, 5) @@ -245,12 +245,7 @@ class SimpleTest(abc.LoaderTests): class BadBytecodeTest: def import_(self, file, module_name): - loader = self.loader(module_name, file) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - # XXX Change to use exec_module(). - module = loader.load_module(module_name) - self.assertIn(module_name, sys.modules) + raise NotImplementedError def manipulate_bytecode(self, name, mapping, manipulator, *, del_source=False): diff --git a/Lib/test/test_importlib/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py index e6a2415bda..795d436c3b 100644 --- a/Lib/test/test_importlib/source/test_path_hook.py +++ b/Lib/test/test_importlib/source/test_path_hook.py @@ -16,10 +16,19 @@ class PathHookTest: def test_success(self): with util.create_modules('dummy') as mapping: self.assertTrue(hasattr(self.path_hook()(mapping['.root']), - 'find_module')) + 'find_spec')) + + def test_success_legacy(self): + with util.create_modules('dummy') as mapping: + self.assertTrue(hasattr(self.path_hook()(mapping['.root']), + 'find_module')) def test_empty_string(self): # The empty string represents the cwd. + self.assertTrue(hasattr(self.path_hook()(''), 'find_spec')) + + def test_empty_string_legacy(self): + # The empty string represents the cwd. self.assertTrue(hasattr(self.path_hook()(''), 'find_module')) diff --git a/Lib/test/test_importlib/source/test_source_encoding.py b/Lib/test/test_importlib/source/test_source_encoding.py index 1e0771b19d..980855fe1a 100644 --- a/Lib/test/test_importlib/source/test_source_encoding.py +++ b/Lib/test/test_importlib/source/test_source_encoding.py @@ -5,7 +5,6 @@ machinery = util.import_importlib('importlib.machinery') import codecs import importlib.util import re -import sys import types # Because sys.path gets essentially blanked, need to have unicodedata already # imported for the parser to use. diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index d4bf9153e9..c86248047f 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -207,6 +207,10 @@ class LoaderDefaultsTests(ABCTestHarness): SPLIT = make_abc_subclasses(Loader) + def test_create_module(self): + spec = 'a spec' + self.assertIsNone(self.ins.create_module(spec)) + def test_load_module(self): with self.assertRaises(ImportError): self.ins.load_module('something') @@ -519,6 +523,12 @@ class InspectLoaderLoadModuleTests: support.unload(self.module_name) self.addCleanup(support.unload, self.module_name) + def load(self, loader): + spec = self.util.spec_from_loader(self.module_name, loader) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + return self.init._bootstrap._load_unlocked(spec) + def mock_get_code(self): return mock.patch.object(self.InspectLoaderSubclass, 'get_code') @@ -528,9 +538,7 @@ class InspectLoaderLoadModuleTests: mocked_get_code.side_effect = ImportError with self.assertRaises(ImportError): loader = self.InspectLoaderSubclass() - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - loader.load_module(self.module_name) + self.load(loader) def test_get_code_None(self): # If get_code() returns None, raise ImportError. @@ -538,7 +546,7 @@ class InspectLoaderLoadModuleTests: mocked_get_code.return_value = None with self.assertRaises(ImportError): loader = self.InspectLoaderSubclass() - loader.load_module(self.module_name) + self.load(loader) def test_module_returned(self): # The loaded module should be returned. @@ -546,14 +554,16 @@ class InspectLoaderLoadModuleTests: with self.mock_get_code() as mocked_get_code: mocked_get_code.return_value = code loader = self.InspectLoaderSubclass() - module = loader.load_module(self.module_name) + module = self.load(loader) self.assertEqual(module, sys.modules[self.module_name]) (Frozen_ILLoadModuleTests, Source_ILLoadModuleTests ) = test_util.test_both(InspectLoaderLoadModuleTests, - InspectLoaderSubclass=SPLIT_IL) + InspectLoaderSubclass=SPLIT_IL, + init=init, + util=util) ##### ExecutionLoader concrete methods ######################################### diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index 6bc3c564a5..b0a94aaff5 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -99,9 +99,7 @@ class ImportModuleTests: class FindLoaderTests: - class FakeMetaFinder: - @staticmethod - def find_module(name, path=None): return name, path + FakeMetaFinder = None def test_sys_modules(self): # If a module with __loader__ is in sys.modules, then return it. @@ -171,9 +169,30 @@ class FindLoaderTests: self.assertIsNone(self.init.find_loader('nevergoingtofindthismodule')) -(Frozen_FindLoaderTests, - Source_FindLoaderTests - ) = test_util.test_both(FindLoaderTests, init=init) +class FindLoaderPEP451Tests(FindLoaderTests): + + class FakeMetaFinder: + @staticmethod + def find_spec(name, path=None, target=None): + return machinery['Source'].ModuleSpec(name, (name, path)) + + +(Frozen_FindLoaderPEP451Tests, + Source_FindLoaderPEP451Tests + ) = test_util.test_both(FindLoaderPEP451Tests, init=init) + + +class FindLoaderPEP302Tests(FindLoaderTests): + + class FakeMetaFinder: + @staticmethod + def find_module(name, path=None): + return name, path + + +(Frozen_FindLoaderPEP302Tests, + Source_FindLoaderPEP302Tests + ) = test_util.test_both(FindLoaderPEP302Tests, init=init) class ReloadTests: diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py index cc383c286d..ffd8dc6cb0 100644 --- a/Lib/test/test_importlib/test_lazy.py +++ b/Lib/test/test_importlib/test_lazy.py @@ -66,6 +66,8 @@ class LazyLoaderTests(unittest.TestCase): spec = util.spec_from_loader(TestingImporter.module_name, util.LazyLoader(loader)) module = spec.loader.create_module(spec) + if module is None: + module = types.ModuleType(TestingImporter.module_name) module.__spec__ = spec module.__loader__ = spec.loader spec.loader.exec_module(module) diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py index df0af12d38..dbce9c2dfd 100644 --- a/Lib/test/test_importlib/test_locks.py +++ b/Lib/test/test_importlib/test_locks.py @@ -3,7 +3,6 @@ from . import util as test_util init = test_util.import_importlib('importlib') import sys -import time import unittest import weakref @@ -58,7 +57,7 @@ if threading is not None: def setUp(self): try: self.old_switchinterval = sys.getswitchinterval() - sys.setswitchinterval(0.000001) + support.setswitchinterval(0.000001) except AttributeError: self.old_switchinterval = None diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 6639612631..e37d8a18f4 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -1,13 +1,10 @@ import contextlib -import importlib.abc -import importlib.machinery +import importlib import os import sys -import types import unittest from test.test_importlib import util -from test.support import run_unittest # needed tests: # @@ -71,6 +68,7 @@ class NamespacePackageTest(unittest.TestCase): # TODO: will we ever want to pass exc_info to __exit__? self.ctx.__exit__(None, None, None) + class SingleNamespacePackage(NamespacePackageTest): paths = ['portion1'] @@ -87,7 +85,7 @@ class SingleNamespacePackage(NamespacePackageTest): self.assertEqual(repr(foo), "<module 'foo' (namespace)>") -class DynamicPatheNamespacePackage(NamespacePackageTest): +class DynamicPathNamespacePackage(NamespacePackageTest): paths = ['portion1'] def test_dynamic_path(self): @@ -289,5 +287,35 @@ class ModuleAndNamespacePackageInSameDir(NamespacePackageTest): self.assertEqual(a_test.attr, 'in module') +class ReloadTests(NamespacePackageTest): + paths = ['portion1'] + + def test_simple_package(self): + import foo.one + foo = importlib.reload(foo) + self.assertEqual(foo.one.attr, 'portion1 foo one') + + def test_cant_import_other(self): + import foo + with self.assertRaises(ImportError): + import foo.two + foo = importlib.reload(foo) + with self.assertRaises(ImportError): + import foo.two + + def test_dynamic_path(self): + import foo.one + with self.assertRaises(ImportError): + import foo.two + + # Now modify sys.path and reload. + sys.path.append(os.path.join(self.root, 'portion2')) + foo = importlib.reload(foo) + + # And make sure foo.two is now importable + import foo.two + self.assertEqual(foo.two.attr, 'portion2 foo two') + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_importlib/test_spec.py b/Lib/test/test_importlib/test_spec.py index 8b333e8c2f..5a16a03de6 100644 --- a/Lib/test/test_importlib/test_spec.py +++ b/Lib/test/test_importlib/test_spec.py @@ -5,6 +5,7 @@ machinery = test_util.import_importlib('importlib.machinery') util = test_util.import_importlib('importlib.util') import os.path +import pathlib from test.support import CleanImport import unittest import sys @@ -659,6 +660,11 @@ class FactoryTests: self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) + def test_spec_from_file_location_path_like_arg(self): + spec = self.util.spec_from_file_location(self.name, + pathlib.PurePath(self.path)) + self.assertEqual(spec.origin, self.path) + def test_spec_from_file_location_default_without_location(self): spec = self.util.spec_from_file_location(self.name) diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 41ca3332d5..d615375b34 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -5,6 +5,7 @@ machinery = util.import_importlib('importlib.machinery') importlib_util = util.import_importlib('importlib.util') import os +import pathlib import string import sys from test import support @@ -46,14 +47,8 @@ class ModuleFromSpecTests: def exec_module(self, module): pass spec = self.machinery.ModuleSpec('test', Loader()) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') + with self.assertRaises(ImportError): module = self.util.module_from_spec(spec) - self.assertEqual(1, len(w)) - self.assertTrue(issubclass(w[0].category, DeprecationWarning)) - self.assertIn('create_module', str(w[0].message)) - self.assertIsInstance(module, types.ModuleType) - self.assertEqual(module.__name__, spec.name) def test_create_module_returns_None(self): class Loader(self.abc.Loader): @@ -677,6 +672,15 @@ class PEP3147Tests: '\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag)) @unittest.skipUnless(sys.implementation.cache_tag is not None, + 'requires sys.implementation.cache_tag not be None') + def test_source_from_cache_path_like_arg(self): + path = pathlib.PurePath('foo', 'bar', 'baz', 'qux.py') + expect = os.path.join('foo', 'bar', 'baz', '__pycache__', + 'qux.{}.pyc'.format(self.tag)) + self.assertEqual(self.util.cache_from_source(path, optimization=''), + expect) + + @unittest.skipUnless(sys.implementation.cache_tag is not None, 'requires sys.implementation.cache_tag to not be ' 'None') def test_source_from_cache(self): @@ -738,6 +742,15 @@ class PEP3147Tests: with self.assertRaises(ValueError): self.util.source_from_cache(path) + @unittest.skipUnless(sys.implementation.cache_tag is not None, + 'requires sys.implementation.cache_tag to not be ' + 'None') + def test_source_from_cache_path_like_arg(self): + path = pathlib.PurePath('foo', 'bar', 'baz', '__pycache__', + 'qux.{}.pyc'.format(self.tag)) + expect = os.path.join('foo', 'bar', 'baz', 'qux.py') + self.assertEqual(self.util.source_from_cache(path), expect) + (Frozen_PEP3147Tests, Source_PEP3147Tests diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index c893bcf565..005b685cc0 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -40,7 +40,7 @@ def setup_module(machinery, name, path=None): else: root = machinery.WindowsRegistryFinder.REGISTRY_KEY key = root.format(fullname=name, - sys_version=sys.version[:3]) + sys_version='%d.%d' % sys.version_info[:2]) try: with temp_module(name, "a = 1") as location: subkey = CreateKey(HKEY_CURRENT_USER, key) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index d33de9e5bc..88eaabe676 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -30,6 +30,7 @@ from test.support import MISSING_C_DOCSTRINGS, cpython_only from test.support.script_helper import assert_python_ok, assert_python_failure from test import inspect_fodder as mod from test import inspect_fodder2 as mod2 +from test import support from test.test_import import _ready_to_import @@ -38,7 +39,7 @@ from test.test_import import _ready_to_import # ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode, # isbuiltin, isroutine, isgenerator, isgeneratorfunction, getmembers, # getdoc, getfile, getmodule, getsourcefile, getcomments, getsource, -# getclasstree, getargspec, getargvalues, formatargspec, formatargvalues, +# getclasstree, getargvalues, formatargspec, formatargvalues, # currentframe, stack, trace, isdatadescriptor # NOTE: There are some additional tests relating to interaction with @@ -64,7 +65,8 @@ class IsTestBase(unittest.TestCase): inspect.isframe, inspect.isfunction, inspect.ismethod, inspect.ismodule, inspect.istraceback, inspect.isgenerator, inspect.isgeneratorfunction, - inspect.iscoroutine, inspect.iscoroutinefunction]) + inspect.iscoroutine, inspect.iscoroutinefunction, + inspect.isasyncgen, inspect.isasyncgenfunction]) def istest(self, predicate, exp): obj = eval(exp) @@ -72,6 +74,7 @@ class IsTestBase(unittest.TestCase): for other in self.predicates - set([predicate]): if (predicate == inspect.isgeneratorfunction or \ + predicate == inspect.isasyncgenfunction or \ predicate == inspect.iscoroutinefunction) and \ other == inspect.isfunction: continue @@ -81,6 +84,10 @@ def generator_function_example(self): for i in range(2): yield i +async def async_generator_function_example(self): + async for i in range(2): + yield i + async def coroutine_function_example(self): return 'spam' @@ -121,6 +128,10 @@ class TestPredicates(IsTestBase): self.istest(inspect.isdatadescriptor, 'collections.defaultdict.default_factory') self.istest(inspect.isgenerator, '(x for x in range(2))') self.istest(inspect.isgeneratorfunction, 'generator_function_example') + self.istest(inspect.isasyncgen, + 'async_generator_function_example(1)') + self.istest(inspect.isasyncgenfunction, + 'async_generator_function_example') with warnings.catch_warnings(): warnings.simplefilter("ignore") @@ -405,7 +416,7 @@ class TestRetrievingSourceCode(GetSourceBase): self.assertEqual(normcase(inspect.getsourcefile(mod.spam)), modfile) self.assertEqual(normcase(inspect.getsourcefile(git.abuse)), modfile) fn = "_non_existing_filename_used_for_sourcefile_test.py" - co = compile("None", fn, "exec") + co = compile("x=1", fn, "exec") self.assertEqual(inspect.getsourcefile(co), None) linecache.cache[co.co_filename] = (1, None, "None", co.co_filename) try: @@ -2907,6 +2918,10 @@ class TestParameterObject(unittest.TestCase): 'is not a valid parameter name'): inspect.Parameter('$', kind=inspect.Parameter.VAR_KEYWORD) + with self.assertRaisesRegex(ValueError, + 'is not a valid parameter name'): + inspect.Parameter('.a', kind=inspect.Parameter.VAR_KEYWORD) + with self.assertRaisesRegex(ValueError, 'cannot have default values'): inspect.Parameter('a', default=42, kind=inspect.Parameter.VAR_KEYWORD) @@ -2990,6 +3005,17 @@ class TestParameterObject(unittest.TestCase): with self.assertRaisesRegex(TypeError, 'name must be a str'): inspect.Parameter(None, kind=inspect.Parameter.POSITIONAL_ONLY) + @cpython_only + def test_signature_parameter_implicit(self): + with self.assertRaisesRegex(ValueError, + 'implicit arguments must be passed in as'): + inspect.Parameter('.0', kind=inspect.Parameter.POSITIONAL_ONLY) + + param = inspect.Parameter( + '.0', kind=inspect.Parameter.POSITIONAL_OR_KEYWORD) + self.assertEqual(param.kind, inspect.Parameter.POSITIONAL_ONLY) + self.assertEqual(param.name, 'implicit0') + def test_signature_parameter_immutability(self): p = inspect.Parameter('spam', kind=inspect.Parameter.KEYWORD_ONLY) @@ -3238,6 +3264,17 @@ class TestSignatureBind(unittest.TestCase): ba = sig.bind(args=1) self.assertEqual(ba.arguments, {'kwargs': {'args': 1}}) + @cpython_only + def test_signature_bind_implicit_arg(self): + # Issue #19611: getcallargs should work with set comprehensions + def make_set(): + return {z * z for z in range(5)} + setcomp_code = make_set.__code__.co_consts[1] + setcomp_func = types.FunctionType(setcomp_code, {}) + + iterator = iter(range(5)) + self.assertEqual(self.call(setcomp_func, iterator), {0, 1, 4, 9, 16}) + class TestBoundArguments(unittest.TestCase): def test_signature_bound_arguments_unhashable(self): @@ -3548,14 +3585,14 @@ class TestMain(unittest.TestCase): def test_details(self): module = importlib.import_module('unittest') - rc, out, err = assert_python_ok('-m', 'inspect', + args = support.optim_args_from_interpreter_flags() + rc, out, err = assert_python_ok(*args, '-m', 'inspect', 'unittest', '--details') output = out.decode() # Just a quick sanity check on the output self.assertIn(module.__name__, output) self.assertIn(module.__file__, output) - if not sys.flags.optimize: - self.assertIn(module.__cached__, output) + self.assertIn(module.__cached__, output) self.assertEqual(err, b'') diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index b66c5d6709..14bbd6192a 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -2,6 +2,8 @@ import sys import unittest from test import support +from test.test_grammar import (VALID_UNDERSCORE_LITERALS, + INVALID_UNDERSCORE_LITERALS) L = [ ('0', 0), @@ -212,6 +214,25 @@ class IntTestCases(unittest.TestCase): self.assertEqual(int('2br45qc', 35), 4294967297) self.assertEqual(int('1z141z5', 36), 4294967297) + def test_underscores(self): + for lit in VALID_UNDERSCORE_LITERALS: + if any(ch in lit for ch in '.eEjJ'): + continue + self.assertEqual(int(lit, 0), eval(lit)) + self.assertEqual(int(lit, 0), int(lit.replace('_', ''), 0)) + for lit in INVALID_UNDERSCORE_LITERALS: + if any(ch in lit for ch in '.eEjJ'): + continue + self.assertRaises(ValueError, int, lit, 0) + # Additional test cases with bases != 0, only for the constructor: + self.assertEqual(int("1_00", 3), 9) + self.assertEqual(int("0_100"), 100) # not valid as a literal! + self.assertEqual(int(b"1_00"), 100) # byte underscore + self.assertRaises(ValueError, int, "_100") + self.assertRaises(ValueError, int, "+_100") + self.assertRaises(ValueError, int, "1__00") + self.assertRaises(ValueError, int, "100_") + @support.cpython_only def test_small_ints(self): # Bug #3236: Return small longs from PyLong_FromString @@ -430,21 +451,24 @@ class IntTestCases(unittest.TestCase): with self.assertWarns(DeprecationWarning): n = int(bad_int) self.assertEqual(n, 1) + self.assertIs(type(n), int) bad_int = BadInt2() with self.assertWarns(DeprecationWarning): n = int(bad_int) self.assertEqual(n, 1) + self.assertIs(type(n), int) bad_int = TruncReturnsBadInt() with self.assertWarns(DeprecationWarning): n = int(bad_int) self.assertEqual(n, 1) + self.assertIs(type(n), int) good_int = TruncReturnsIntSubclass() n = int(good_int) self.assertEqual(n, 1) - self.assertIs(type(n), bool) + self.assertIs(type(n), int) n = IntSubclass(good_int) self.assertEqual(n, 1) self.assertIs(type(n), IntSubclass) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 5111882a02..aaa64eadff 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -350,7 +350,10 @@ class IOTest(unittest.TestCase): def large_file_ops(self, f): assert f.readable() assert f.writable() - self.assertEqual(f.seek(self.LARGE), self.LARGE) + try: + self.assertEqual(f.seek(self.LARGE), self.LARGE) + except (OverflowError, ValueError): + self.skipTest("no largefile support") self.assertEqual(f.tell(), self.LARGE) self.assertEqual(f.write(b"xxx"), 3) self.assertEqual(f.tell(), self.LARGE + 3) @@ -496,7 +499,11 @@ class IOTest(unittest.TestCase): def test_open_handles_NUL_chars(self): fn_with_NUL = 'foo\0bar' self.assertRaises(ValueError, self.open, fn_with_NUL, 'w') - self.assertRaises(ValueError, self.open, bytes(fn_with_NUL, 'ascii'), 'w') + + bytes_fn = bytes(fn_with_NUL, 'ascii') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + self.assertRaises(ValueError, self.open, bytes_fn, 'w') def test_raw_file_io(self): with self.open(support.TESTFN, "wb", buffering=0) as f: @@ -856,6 +863,32 @@ class IOTest(unittest.TestCase): self.assertEqual(getattr(stream, method)(buffer), 5) self.assertEqual(bytes(buffer), b"12345") + def test_fspath_support(self): + class PathLike: + def __init__(self, path): + self.path = path + + def __fspath__(self): + return self.path + + def check_path_succeeds(path): + with self.open(path, "w") as f: + f.write("egg\n") + + with self.open(path, "r") as f: + self.assertEqual(f.read(), "egg\n") + + check_path_succeeds(PathLike(support.TESTFN)) + check_path_succeeds(PathLike(support.TESTFN.encode('utf-8'))) + + bad_path = PathLike(TypeError) + with self.assertRaises(TypeError): + self.open(bad_path, 'w') + + # ensure that refcounting is correct with some error conditions + with self.assertRaisesRegex(ValueError, 'read/write/append mode'): + self.open(PathLike(support.TESTFN), 'rwxa') + class CIOTest(IOTest): @@ -1782,7 +1815,7 @@ class BufferedRWPairTest(unittest.TestCase): with self.subTest(method): pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) - data = byteslike(5) + data = byteslike(b'\0' * 5) self.assertEqual(getattr(pair, method)(data), 5) self.assertEqual(bytes(data), b"abcde") @@ -3246,8 +3279,7 @@ class CTextIOWrapperTest(TextIOWrapperTest): class PyTextIOWrapperTest(TextIOWrapperTest): io = pyio - #shutdown_error = "LookupError: unknown encoding: ascii" - shutdown_error = "TypeError: 'NoneType' object is not iterable" + shutdown_error = "LookupError: unknown encoding: ascii" class IncrementalNewlineDecoderTest(unittest.TestCase): diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 91ae8d85ed..0e39516dc0 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -119,7 +119,7 @@ class CommonTestMixin_v4(CommonTestMixin): def test_bad_packed_length(self): def assertBadLength(length): - addr = bytes(length) + addr = b'\0' * length msg = "%r (len %d != 4) is not permitted as an IPv4 address" with self.assertAddressError(re.escape(msg % (addr, length))): self.factory(addr) @@ -139,11 +139,11 @@ class CommonTestMixin_v6(CommonTestMixin): self.assertInstancesEqual(3232235521, "::c0a8:1") def test_packed(self): - addr = bytes(12) + bytes.fromhex("00000000") + addr = b'\0'*12 + bytes.fromhex("00000000") self.assertInstancesEqual(addr, "::") - addr = bytes(12) + bytes.fromhex("c0a80001") + addr = b'\0'*12 + bytes.fromhex("c0a80001") self.assertInstancesEqual(addr, "::c0a8:1") - addr = bytes.fromhex("c0a80001") + bytes(12) + addr = bytes.fromhex("c0a80001") + b'\0'*12 self.assertInstancesEqual(addr, "c0a8:1::") def test_negative_ints_rejected(self): @@ -158,7 +158,7 @@ class CommonTestMixin_v6(CommonTestMixin): def test_bad_packed_length(self): def assertBadLength(length): - addr = bytes(length) + addr = b'\0' * length msg = "%r (len %d != 16) is not permitted as an IPv6 address" with self.assertAddressError(re.escape(msg % (addr, length))): self.factory(addr) @@ -1176,6 +1176,7 @@ class IpaddrUnitTest(unittest.TestCase): self.assertEqual(str(self.ipv6_network[5]), '2001:658:22a:cafe::5') + self.assertRaises(IndexError, self.ipv6_network.__getitem__, 1 << 64) def testGetitem(self): # http://code.google.com/p/ipaddr-py/issues/detail?id=15 diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index a91670b4a1..542b28419e 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -54,6 +54,14 @@ class UnlimitedSequenceClass: def __getitem__(self, i): return i +class DefaultIterClass: + pass + +class NoIterClass: + def __getitem__(self, i): + return i + __iter__ = None + # Main test suite class TestCase(unittest.TestCase): @@ -995,6 +1003,10 @@ class TestCase(unittest.TestCase): def test_free_after_iterating(self): check_free_after_iterating(self, iter, SequenceClass, (0,)) + def test_error_iter(self): + for typ in (DefaultIterClass, NoIterClass): + self.assertRaises(TypeError, iter, typ()) + def test_main(): run_unittest(TestCase) diff --git a/Lib/test/test_iterlen.py b/Lib/test/test_iterlen.py index 152f5fc0cb..41c9752e55 100644 --- a/Lib/test/test_iterlen.py +++ b/Lib/test/test_iterlen.py @@ -42,7 +42,6 @@ enumerate(iter('abc')). """ import unittest -from test import support from itertools import repeat from collections import deque from operator import length_hint diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index e054303ded..ea1f57caad 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -4,7 +4,6 @@ from itertools import * import weakref from decimal import Decimal from fractions import Fraction -import sys import operator import random import copy @@ -644,24 +643,58 @@ class TestBasicOps(unittest.TestCase): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, cycle('abc')) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + # test with partial consumed input iterable + it = iter('abcde') + c = cycle(it) + _ = [next(c) for i in range(2)] # consume 2 of 5 inputs + p = pickle.dumps(c, proto) + d = pickle.loads(p) # rebuild the cycle object + self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + + # test with completely consumed input iterable + it = iter('abcde') + c = cycle(it) + _ = [next(c) for i in range(7)] # consume 7 of 5 inputs + p = pickle.dumps(c, proto) + d = pickle.loads(p) # rebuild the cycle object + self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + def test_cycle_setstate(self): + # Verify both modes for restoring state + + # Mode 0 is efficient. It uses an incompletely consumed input + # iterator to build a cycle object and then passes in state with + # a list of previously consumed values. There is no data + # overlap between the two. + c = cycle('defg') + c.__setstate__((list('abc'), 0)) + self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) + + # Mode 1 is inefficient. It starts with a cycle object built + # from an iterator over the remaining elements in a partial + # cycle and then passes in state with all of the previously + # seen values (this overlaps values included in the iterator). + c = cycle('defg') + c.__setstate__((list('abcdefg'), 1)) + self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) + + # The first argument to setstate needs to be a tuple + with self.assertRaises(TypeError): + cycle('defg').__setstate__([list('abcdefg'), 0]) + + # The first argument in the setstate tuple must be a list + with self.assertRaises(TypeError): + c = cycle('defg') + c.__setstate__((tuple('defg'), 0)) + take(20, c) + + # The second argument in the setstate tuple must be an int + with self.assertRaises(TypeError): + cycle('defg').__setstate__((list('abcdefg'), 'x')) + self.assertRaises(TypeError, cycle('').__setstate__, ()) - self.assertRaises(TypeError, cycle('').__setstate__, []) - self.assertRaises(TypeError, cycle('').__setstate__, 0) self.assertRaises(TypeError, cycle('').__setstate__, ([],)) - self.assertRaises(TypeError, cycle('').__setstate__, ((), 0)) - it = cycle('abc') - it.__setstate__((['de', 'fg'], 0)) - self.assertEqual(list(islice(it, 15)), - ['a', 'b', 'c', 'de', 'fg', - 'a', 'b', 'c', 'de', 'fg', - 'a', 'b', 'c', 'de', 'fg']) - it = cycle('abc') - it.__setstate__((['de', 'fg'], 1)) - self.assertEqual(list(islice(it, 15)), - ['a', 'b', 'c', 'de', 'fg', - 'de', 'fg', 'de', 'fg', 'de', - 'fg', 'de', 'fg', 'de', 'fg']) def test_groupby(self): # Check whether it accepts arguments correctly diff --git a/Lib/test/test_json/__init__.py b/Lib/test/test_json/__init__.py index 0807e6fb4f..bac370dadf 100644 --- a/Lib/test/test_json/__init__.py +++ b/Lib/test/test_json/__init__.py @@ -1,5 +1,4 @@ import os -import sys import json import doctest import unittest diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index fdafeb6d8f..7e568be409 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -72,10 +72,8 @@ class TestDecode: def test_invalid_input_type(self): msg = 'the JSON object must be str' - for value in [1, 3.14, b'bytes', b'\xff\x00', [], {}, None]: + for value in [1, 3.14, [], {}, None]: self.assertRaisesRegex(TypeError, msg, self.loads, value) - with self.assertRaisesRegex(TypeError, msg): - self.json.load(BytesIO(b'[1,2,3]')) def test_string_with_utf8_bom(self): # see #18958 diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index 95ff5b8d1e..7910521021 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -1,5 +1,4 @@ from test.test_json import PyTest, CTest -import re # 2007-10-05 JSONDOCS = [ diff --git a/Lib/test/test_json/test_unicode.py b/Lib/test/test_json/test_unicode.py index c7cc8a7e92..2e8bba2775 100644 --- a/Lib/test/test_json/test_unicode.py +++ b/Lib/test/test_json/test_unicode.py @@ -1,3 +1,4 @@ +import codecs from collections import OrderedDict from test.test_json import PyTest, CTest @@ -52,9 +53,31 @@ class TestUnicode: self.assertRaises(TypeError, self.dumps, [b"hi"]) def test_bytes_decode(self): - self.assertRaises(TypeError, self.loads, b'"hi"') - self.assertRaises(TypeError, self.loads, b'["hi"]') - + for encoding, bom in [ + ('utf-8', codecs.BOM_UTF8), + ('utf-16be', codecs.BOM_UTF16_BE), + ('utf-16le', codecs.BOM_UTF16_LE), + ('utf-32be', codecs.BOM_UTF32_BE), + ('utf-32le', codecs.BOM_UTF32_LE), + ]: + data = ["a\xb5\u20ac\U0001d120"] + encoded = self.dumps(data).encode(encoding) + self.assertEqual(self.loads(bom + encoded), data) + self.assertEqual(self.loads(encoded), data) + self.assertRaises(UnicodeDecodeError, self.loads, b'["\x80"]') + # RFC-7159 and ECMA-404 extend JSON to allow documents that + # consist of only a string, which can present a special case + # not covered by the encoding detection patterns specified in + # RFC-4627 for utf-16-le (XX 00 XX 00). + self.assertEqual(self.loads('"\u2600"'.encode('utf-16-le')), + '\u2600') + # Encoding detection for small (<4) bytes objects + # is implemented as a special case. RFC-7159 and ECMA-404 + # allow single codepoint JSON documents which are only two + # bytes in utf-16 encodings w/o BOM. + self.assertEqual(self.loads(b'5\x00'), 5) + self.assertEqual(self.loads(b'\x007'), 7) + self.assertEqual(self.loads(b'57'), 57) def test_object_pairs_hook_with_unicode(self): s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' diff --git a/Lib/test/test_kqueue.py b/Lib/test/test_kqueue.py index f822024aa0..9f49886878 100644 --- a/Lib/test/test_kqueue.py +++ b/Lib/test/test_kqueue.py @@ -5,7 +5,6 @@ import errno import os import select import socket -import sys import time import unittest diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 47e2eddfb1..375d9c4213 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -3,6 +3,8 @@ import linecache import unittest import os.path +import tempfile +import tokenize from test import support @@ -10,8 +12,6 @@ FILENAME = linecache.__file__ NONEXISTENT_FILENAME = FILENAME + '.missing' INVALID_NAME = '!@$)(!@#_1' EMPTY = '' -TESTS = 'inspect_fodder inspect_fodder2 mapping_tests' -TESTS = TESTS.split() TEST_PATH = os.path.dirname(__file__) MODULES = "linecache abc".split() MODULE_PATH = os.path.dirname(FILENAME) @@ -37,6 +37,65 @@ def f(): return 3''' # No ending newline +class TempFile: + + def setUp(self): + super().setUp() + with tempfile.NamedTemporaryFile(delete=False) as fp: + self.file_name = fp.name + fp.write(self.file_byte_string) + self.addCleanup(support.unlink, self.file_name) + + +class GetLineTestsGoodData(TempFile): + # file_list = ['list\n', 'of\n', 'good\n', 'strings\n'] + + def setUp(self): + self.file_byte_string = ''.join(self.file_list).encode('utf-8') + super().setUp() + + def test_getline(self): + with tokenize.open(self.file_name) as fp: + for index, line in enumerate(fp): + if not line.endswith('\n'): + line += '\n' + + cached_line = linecache.getline(self.file_name, index + 1) + self.assertEqual(line, cached_line) + + def test_getlines(self): + lines = linecache.getlines(self.file_name) + self.assertEqual(lines, self.file_list) + + +class GetLineTestsBadData(TempFile): + # file_byte_string = b'Bad data goes here' + + def test_getline(self): + self.assertRaises((SyntaxError, UnicodeDecodeError), + linecache.getline, self.file_name, 1) + + def test_getlines(self): + self.assertRaises((SyntaxError, UnicodeDecodeError), + linecache.getlines, self.file_name) + + +class EmptyFile(GetLineTestsGoodData, unittest.TestCase): + file_list = [] + + +class SingleEmptyLine(GetLineTestsGoodData, unittest.TestCase): + file_list = ['\n'] + + +class GoodUnicode(GetLineTestsGoodData, unittest.TestCase): + file_list = ['á\n', 'b\n', 'abcdef\n', 'ááááá\n'] + + +class BadUnicode(GetLineTestsBadData, unittest.TestCase): + file_byte_string = b'\x80abc' + + class LineCacheTests(unittest.TestCase): def test_getline(self): @@ -53,13 +112,6 @@ class LineCacheTests(unittest.TestCase): self.assertEqual(getline(EMPTY, 1), EMPTY) self.assertEqual(getline(INVALID_NAME, 1), EMPTY) - # Check whether lines correspond to those from file iteration - for entry in TESTS: - filename = os.path.join(TEST_PATH, entry) + '.py' - with open(filename) as file: - for index, line in enumerate(file): - self.assertEqual(line, getline(filename, index + 1)) - # Check module loading for entry in MODULES: filename = os.path.join(MODULE_PATH, entry) + '.py' @@ -80,12 +132,13 @@ class LineCacheTests(unittest.TestCase): def test_clearcache(self): cached = [] - for entry in TESTS: - filename = os.path.join(TEST_PATH, entry) + '.py' + for entry in MODULES: + filename = os.path.join(MODULE_PATH, entry) + '.py' cached.append(filename) linecache.getline(filename, 1) # Are all files cached? + self.assertNotEqual(cached, []) cached_empty = [fn for fn in cached if fn not in linecache.cache] self.assertEqual(cached_empty, []) diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 8f82ab5297..aee62dca1c 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -1,5 +1,5 @@ import sys -from test import support, list_tests +from test import list_tests import pickle import unittest diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index fae2c3dabb..99fab58cb9 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -1,4 +1,4 @@ -from test.support import verbose +from test.support import verbose, is_android import unittest import locale import sys @@ -353,7 +353,7 @@ class TestEnUSCollation(BaseLocalizedTest, TestCollation): enc = codecs.lookup(locale.getpreferredencoding(False) or 'ascii').name if enc not in ('utf-8', 'iso8859-1', 'cp1252'): raise unittest.SkipTest('encoding not suitable') - if enc != 'iso8859-1' and (sys.platform == 'darwin' or + if enc != 'iso8859-1' and (sys.platform == 'darwin' or is_android or sys.platform.startswith('freebsd')): raise unittest.SkipTest('wcscoll/wcsxfrm have known bugs') BaseLocalizedTest.setUp(self) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 1c850456b1..9dedc098ba 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -1,4 +1,4 @@ -# Copyright 2001-2016 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2017 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -16,7 +16,7 @@ """Test harness for the logging module. Run all tests. -Copyright (C) 2001-2016 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2017 Vinay Sajip. All Rights Reserved. """ import logging @@ -26,6 +26,7 @@ import logging.config import codecs import configparser import datetime +import pathlib import pickle import io import gc @@ -308,6 +309,10 @@ class BuiltinLevelsTest(BaseTest): self.assertEqual(logging.getLevelName('INFO'), logging.INFO) self.assertEqual(logging.getLevelName(logging.INFO), 'INFO') + def test_issue27935(self): + fatal = logging.getLevelName('FATAL') + self.assertEqual(fatal, logging.FATAL) + def test_regression_29220(self): """See issue #29220 for more information.""" logging.addLevelName(logging.INFO, '') @@ -583,6 +588,29 @@ class HandlerTest(BaseTest): self.assertFalse(h.shouldFlush(r)) h.close() + def test_path_objects(self): + """ + Test that Path objects are accepted as filename arguments to handlers. + + See Issue #27493. + """ + fd, fn = tempfile.mkstemp() + os.close(fd) + os.unlink(fn) + pfn = pathlib.Path(fn) + cases = ( + (logging.FileHandler, (pfn, 'w')), + (logging.handlers.RotatingFileHandler, (pfn, 'a')), + (logging.handlers.TimedRotatingFileHandler, (pfn, 'h')), + ) + if sys.platform in ('linux', 'darwin'): + cases += ((logging.handlers.WatchedFileHandler, (pfn, 'w')),) + for cls, args in cases: + h = cls(*args) + self.assertTrue(os.path.exists(fn)) + h.close() + os.unlink(fn) + @unittest.skipIf(os.name == 'nt', 'WatchedFileHandler not appropriate for Windows.') @unittest.skipUnless(threading, 'Threading required for this test.') def test_race(self): @@ -966,7 +994,7 @@ class MemoryHandlerTest(BaseTest): def setUp(self): BaseTest.setUp(self) self.mem_hdlr = logging.handlers.MemoryHandler(10, logging.WARNING, - self.root_hdlr) + self.root_hdlr) self.mem_logger = logging.getLogger('mem') self.mem_logger.propagate = 0 self.mem_logger.addHandler(self.mem_hdlr) @@ -1003,6 +1031,36 @@ class MemoryHandlerTest(BaseTest): self.mem_logger.debug(self.next_message()) self.assert_log_lines(lines) + def test_flush_on_close(self): + """ + Test that the flush-on-close configuration works as expected. + """ + self.mem_logger.debug(self.next_message()) + self.assert_log_lines([]) + self.mem_logger.info(self.next_message()) + self.assert_log_lines([]) + self.mem_logger.removeHandler(self.mem_hdlr) + # Default behaviour is to flush on close. Check that it happens. + self.mem_hdlr.close() + lines = [ + ('DEBUG', '1'), + ('INFO', '2'), + ] + self.assert_log_lines(lines) + # Now configure for flushing not to be done on close. + self.mem_hdlr = logging.handlers.MemoryHandler(10, logging.WARNING, + self.root_hdlr, + False) + self.mem_logger.addHandler(self.mem_hdlr) + self.mem_logger.debug(self.next_message()) + self.assert_log_lines(lines) # no change + self.mem_logger.info(self.next_message()) + self.assert_log_lines(lines) # no change + self.mem_logger.removeHandler(self.mem_hdlr) + self.mem_hdlr.close() + # assert that no new lines have been added + self.assert_log_lines(lines) # no change + class ExceptionFormatter(logging.Formatter): """A special exception formatter.""" @@ -1390,9 +1448,17 @@ class SocketHandlerTest(BaseTest): """Set up a TCP server to receive log messages, and a SocketHandler pointing to that server's address and port.""" BaseTest.setUp(self) - self.server = server = self.server_class(self.address, - self.handle_socket, 0.01) - server.start() + # Issue #29177: deal with errors that happen during setup + self.server = self.sock_hdlr = self.server_exception = None + try: + self.server = server = self.server_class(self.address, + self.handle_socket, 0.01) + server.start() + # Uncomment next line to test error recovery in setUp() + # raise OSError('dummy error raised') + except OSError as e: + self.server_exception = e + return server.ready.wait() hcls = logging.handlers.SocketHandler if isinstance(server.server_address, tuple): @@ -1407,9 +1473,11 @@ class SocketHandlerTest(BaseTest): def tearDown(self): """Shutdown the TCP server.""" try: - self.server.stop(2.0) - self.root_logger.removeHandler(self.sock_hdlr) - self.sock_hdlr.close() + if self.server: + self.server.stop(2.0) + if self.sock_hdlr: + self.root_logger.removeHandler(self.sock_hdlr) + self.sock_hdlr.close() finally: BaseTest.tearDown(self) @@ -1430,6 +1498,8 @@ class SocketHandlerTest(BaseTest): def test_output(self): # The log message sent to the SocketHandler is properly received. + if self.server_exception: + self.skipTest(self.server_exception) logger = logging.getLogger("tcp") logger.error("spam") self.handled.acquire() @@ -1438,6 +1508,8 @@ class SocketHandlerTest(BaseTest): self.assertEqual(self.log_output, "spam\neggs\n") def test_noserver(self): + if self.server_exception: + self.skipTest(self.server_exception) # Avoid timing-related failures due to SocketHandler's own hard-wired # one-second timeout on socket.create_connection() (issue #16264). self.sock_hdlr.retryStart = 2.5 @@ -1478,7 +1550,7 @@ class UnixSocketHandlerTest(SocketHandlerTest): def tearDown(self): SocketHandlerTest.tearDown(self) - os.remove(self.address) + support.unlink(self.address) @unittest.skipUnless(threading, 'Threading required for this test.') class DatagramHandlerTest(BaseTest): @@ -1493,9 +1565,17 @@ class DatagramHandlerTest(BaseTest): """Set up a UDP server to receive log messages, and a DatagramHandler pointing to that server's address and port.""" BaseTest.setUp(self) - self.server = server = self.server_class(self.address, - self.handle_datagram, 0.01) - server.start() + # Issue #29177: deal with errors that happen during setup + self.server = self.sock_hdlr = self.server_exception = None + try: + self.server = server = self.server_class(self.address, + self.handle_datagram, 0.01) + server.start() + # Uncomment next line to test error recovery in setUp() + # raise OSError('dummy error raised') + except OSError as e: + self.server_exception = e + return server.ready.wait() hcls = logging.handlers.DatagramHandler if isinstance(server.server_address, tuple): @@ -1510,9 +1590,11 @@ class DatagramHandlerTest(BaseTest): def tearDown(self): """Shutdown the UDP server.""" try: - self.server.stop(2.0) - self.root_logger.removeHandler(self.sock_hdlr) - self.sock_hdlr.close() + if self.server: + self.server.stop(2.0) + if self.sock_hdlr: + self.root_logger.removeHandler(self.sock_hdlr) + self.sock_hdlr.close() finally: BaseTest.tearDown(self) @@ -1526,6 +1608,8 @@ class DatagramHandlerTest(BaseTest): def test_output(self): # The log message sent to the DatagramHandler is properly received. + if self.server_exception: + self.skipTest(self.server_exception) logger = logging.getLogger("udp") logger.error("spam") self.handled.wait() @@ -1550,7 +1634,7 @@ class UnixDatagramHandlerTest(DatagramHandlerTest): def tearDown(self): DatagramHandlerTest.tearDown(self) - os.remove(self.address) + support.unlink(self.address) @unittest.skipUnless(threading, 'Threading required for this test.') class SysLogHandlerTest(BaseTest): @@ -1565,9 +1649,17 @@ class SysLogHandlerTest(BaseTest): """Set up a UDP server to receive log messages, and a SysLogHandler pointing to that server's address and port.""" BaseTest.setUp(self) - self.server = server = self.server_class(self.address, - self.handle_datagram, 0.01) - server.start() + # Issue #29177: deal with errors that happen during setup + self.server = self.sl_hdlr = self.server_exception = None + try: + self.server = server = self.server_class(self.address, + self.handle_datagram, 0.01) + server.start() + # Uncomment next line to test error recovery in setUp() + # raise OSError('dummy error raised') + except OSError as e: + self.server_exception = e + return server.ready.wait() hcls = logging.handlers.SysLogHandler if isinstance(server.server_address, tuple): @@ -1580,11 +1672,13 @@ class SysLogHandlerTest(BaseTest): self.handled = threading.Event() def tearDown(self): - """Shutdown the UDP server.""" + """Shutdown the server.""" try: - self.server.stop(2.0) - self.root_logger.removeHandler(self.sl_hdlr) - self.sl_hdlr.close() + if self.server: + self.server.stop(2.0) + if self.sl_hdlr: + self.root_logger.removeHandler(self.sl_hdlr) + self.sl_hdlr.close() finally: BaseTest.tearDown(self) @@ -1593,6 +1687,8 @@ class SysLogHandlerTest(BaseTest): self.handled.set() def test_output(self): + if self.server_exception: + self.skipTest(self.server_exception) # The log message sent to the SysLogHandler is properly received. logger = logging.getLogger("slh") logger.error("sp\xe4m") @@ -1625,7 +1721,7 @@ class UnixSysLogHandlerTest(SysLogHandlerTest): def tearDown(self): SysLogHandlerTest.tearDown(self) - os.remove(self.address) + support.unlink(self.address) @unittest.skipUnless(threading, 'Threading required for this test.') class HTTPHandlerTest(BaseTest): @@ -4249,6 +4345,17 @@ class NTEventLogHandlerTest(BaseTest): msg = 'Record not found in event log, went back %d records' % GO_BACK self.assertTrue(found, msg=msg) + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'logThreads', 'logMultiprocessing', + 'logProcesses', 'currentframe', + 'PercentStyle', 'StrFormatStyle', 'StringTemplateStyle', + 'Filterer', 'PlaceHolder', 'Manager', 'RootLogger', + 'root', 'threading'} + support.check__all__(self, logging, blacklist=blacklist) + + # Set the locale to the platform-dependent default. I have no idea # why the test does this, but in any case we save the current locale # first and restore it at the end. @@ -4266,6 +4373,7 @@ def test_main(): ExceptionTest, SysLogHandlerTest, HTTPHandlerTest, NTEventLogHandlerTest, TimedRotatingFileHandlerTest, UnixSocketHandlerTest, UnixDatagramHandlerTest, UnixSysLogHandlerTest, + MiscTestCase ] if hasattr(logging.handlers, 'QueueListener'): tests.append(QueueListenerTest) diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index 4b2d81c98a..fd15f04ace 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -621,6 +621,8 @@ class LongTest(unittest.TestCase): def test__format__(self): self.assertEqual(format(123456789, 'd'), '123456789') self.assertEqual(format(123456789, 'd'), '123456789') + self.assertEqual(format(123456789, ','), '123,456,789') + self.assertEqual(format(123456789, '_'), '123_456_789') # sign and aligning are interdependent self.assertEqual(format(1, "-"), '1') @@ -649,8 +651,25 @@ class LongTest(unittest.TestCase): self.assertEqual(format(int('be', 16), "X"), "BE") self.assertEqual(format(-int('be', 16), "x"), "-be") self.assertEqual(format(-int('be', 16), "X"), "-BE") + self.assertRaises(ValueError, format, 1234567890, ',x') + self.assertEqual(format(1234567890, '_x'), '4996_02d2') + self.assertEqual(format(1234567890, '_X'), '4996_02D2') # octal + self.assertEqual(format(3, "o"), "3") + self.assertEqual(format(-3, "o"), "-3") + self.assertEqual(format(1234, "o"), "2322") + self.assertEqual(format(-1234, "o"), "-2322") + self.assertEqual(format(1234, "-o"), "2322") + self.assertEqual(format(-1234, "-o"), "-2322") + self.assertEqual(format(1234, " o"), " 2322") + self.assertEqual(format(-1234, " o"), "-2322") + self.assertEqual(format(1234, "+o"), "+2322") + self.assertEqual(format(-1234, "+o"), "-2322") + self.assertRaises(ValueError, format, 1234567890, ',o') + self.assertEqual(format(1234567890, '_o'), '111_4540_1322') + + # binary self.assertEqual(format(3, "b"), "11") self.assertEqual(format(-3, "b"), "-11") self.assertEqual(format(1234, "b"), "10011010010") @@ -661,12 +680,21 @@ class LongTest(unittest.TestCase): self.assertEqual(format(-1234, " b"), "-10011010010") self.assertEqual(format(1234, "+b"), "+10011010010") self.assertEqual(format(-1234, "+b"), "-10011010010") + self.assertRaises(ValueError, format, 1234567890, ',b') + self.assertEqual(format(12345, '_b'), '11_0000_0011_1001') # make sure these are errors self.assertRaises(ValueError, format, 3, "1.3") # precision disallowed + self.assertRaises(ValueError, format, 3, "_c") # underscore, + self.assertRaises(ValueError, format, 3, ",c") # comma, and self.assertRaises(ValueError, format, 3, "+c") # sign not allowed # with 'c' + self.assertRaisesRegex(ValueError, 'Cannot specify both', format, 3, '_,') + self.assertRaisesRegex(ValueError, 'Cannot specify both', format, 3, ',_') + self.assertRaisesRegex(ValueError, 'Cannot specify both', format, 3, '_,d') + self.assertRaisesRegex(ValueError, 'Cannot specify both', format, 3, ',_d') + # ensure that only int and float type specifiers work for format_spec in ([chr(x) for x in range(ord('a'), ord('z')+1)] + [chr(x) for x in range(ord('A'), ord('Z')+1)]): @@ -689,6 +717,20 @@ class LongTest(unittest.TestCase): self.assertRaises(OverflowError, int, float('-inf')) self.assertRaises(ValueError, int, float('nan')) + def test_mod_division(self): + with self.assertRaises(ZeroDivisionError): + _ = 1 % 0 + + self.assertEqual(13 % 10, 3) + self.assertEqual(-13 % 10, 7) + self.assertEqual(13 % -10, -7) + self.assertEqual(-13 % -10, -3) + + self.assertEqual(12 % 4, 0) + self.assertEqual(-12 % 4, 0) + self.assertEqual(12 % -4, 0) + self.assertEqual(-12 % -4, 0) + def test_true_division(self): huge = 1 << 40000 mhuge = -huge @@ -723,6 +765,25 @@ class LongTest(unittest.TestCase): for zero in ["huge / 0", "mhuge / 0"]: self.assertRaises(ZeroDivisionError, eval, zero, namespace) + def test_floordiv(self): + with self.assertRaises(ZeroDivisionError): + _ = 1 // 0 + + self.assertEqual(2 // 3, 0) + self.assertEqual(2 // -3, -1) + self.assertEqual(-2 // 3, -1) + self.assertEqual(-2 // -3, 0) + + self.assertEqual(-11 // -3, 3) + self.assertEqual(-11 // 3, -4) + self.assertEqual(11 // -3, -4) + self.assertEqual(11 // 3, 3) + + self.assertEqual(-12 // -3, 4) + self.assertEqual(-12 // 3, -4) + self.assertEqual(12 // -3, -4) + self.assertEqual(12 // 3, 4) + def check_truediv(self, a, b, skip_small=True): """Verify that the result of a/b is correctly rounded, by comparing it with a pure Python implementation of correctly @@ -845,6 +906,21 @@ class LongTest(unittest.TestCase): self.check_truediv(-x, y) self.check_truediv(-x, -y) + def test_lshift_of_zero(self): + self.assertEqual(0 << 0, 0) + self.assertEqual(0 << 10, 0) + with self.assertRaises(ValueError): + 0 << -1 + + @support.cpython_only + def test_huge_lshift_of_zero(self): + # Shouldn't try to allocate memory for a huge shift. See issue #27870. + # Other implementations may have a different boundary for overflow, + # or not raise at all. + self.assertEqual(0 << sys.maxsize, 0) + with self.assertRaises(OverflowError): + 0 << (sys.maxsize + 1) + def test_small_ints(self): for i in range(-5, 257): self.assertIs(i, i + 0) diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index b69da065e1..d7a8576d51 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -1,6 +1,7 @@ import _compression from io import BytesIO, UnsupportedOperation, DEFAULT_BUFFER_SIZE import os +import pathlib import pickle import random import unittest @@ -526,6 +527,16 @@ class FileTestCase(unittest.TestCase): with LZMAFile(BytesIO(), "a") as f: pass + def test_init_with_PathLike_filename(self): + filename = pathlib.Path(TESTFN) + with TempFile(filename, COMPRESSED_XZ): + with LZMAFile(filename) as f: + self.assertEqual(f.read(), INPUT) + with LZMAFile(filename, "a") as f: + f.write(INPUT) + with LZMAFile(filename) as f: + self.assertEqual(f.read(), INPUT * 2) + def test_init_with_filename(self): with TempFile(TESTFN, COMPRESSED_XZ): with LZMAFile(TESTFN) as f: @@ -973,11 +984,11 @@ class FileTestCase(unittest.TestCase): def test_decompress_limited(self): """Decompressed data buffering should be limited""" - bomb = lzma.compress(bytes(int(2e6)), preset=6) + bomb = lzma.compress(b'\0' * int(2e6), preset=6) self.assertLess(len(bomb), _compression.BUFFER_SIZE) decomp = LZMAFile(BytesIO(bomb)) - self.assertEqual(bytes(1), decomp.read(1)) + self.assertEqual(decomp.read(1), b'\0') max_decomp = 1 + DEFAULT_BUFFER_SIZE self.assertLessEqual(decomp._buffer.raw.tell(), max_decomp, "Excessive amount of data was decompressed") @@ -1218,6 +1229,17 @@ class OpenTestCase(unittest.TestCase): with lzma.open(TESTFN, "rb") as f: self.assertEqual(f.read(), INPUT * 2) + def test_with_pathlike_filename(self): + filename = pathlib.Path(TESTFN) + with TempFile(filename): + with lzma.open(filename, "wb") as f: + f.write(INPUT) + with open(filename, "rb") as f: + file_data = lzma.decompress(f.read()) + self.assertEqual(file_data, INPUT) + with lzma.open(filename, "rb") as f: + self.assertEqual(f.read(), INPUT) + def test_bad_params(self): # Test invalid parameter combinations. with self.assertRaises(ValueError): diff --git a/Lib/test/test_macpath.py b/Lib/test/test_macpath.py index 80bec7a799..0698ff5f6b 100644 --- a/Lib/test/test_macpath.py +++ b/Lib/test/test_macpath.py @@ -1,5 +1,5 @@ import macpath -from test import support, test_genericpath +from test import test_genericpath import unittest diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 0991f74f84..2ba944335a 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -7,17 +7,12 @@ import email import email.message import re import io -import shutil import tempfile from test import support import unittest import textwrap import mailbox import glob -try: - import fcntl -except ImportError: - pass class TestBase: @@ -2142,9 +2137,9 @@ class MaildirTestCase(unittest.TestCase): if mbox: fp.write(FROM_) fp.write(DUMMY_MESSAGE) - if hasattr(os, "link"): + try: os.link(tmpname, newname) - else: + except (AttributeError, PermissionError): with open(newname, "w") as fp: fp.write(DUMMY_MESSAGE) self._msgfiles.append(newname) @@ -2273,12 +2268,18 @@ Gregory K. Johnson """) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {"linesep", "fcntl"} + support.check__all__(self, mailbox, blacklist=blacklist) + + def test_main(): tests = (TestMailboxSuperclass, TestMaildir, TestMbox, TestMMDF, TestMH, TestBabyl, TestMessage, TestMaildirMessage, TestMboxMessage, TestMHMessage, TestBabylMessage, TestMMDFMessage, TestMessageConversion, TestProxyFile, TestPartialFile, - MaildirTestCase, TestFakeMailBox) + MaildirTestCase, TestFakeMailBox, MiscTestCase) support.run_unittest(*tests) support.reap_children() diff --git a/Lib/test/test_mailcap.py b/Lib/test/test_mailcap.py index a85c69164d..c08423c670 100644 --- a/Lib/test/test_mailcap.py +++ b/Lib/test/test_mailcap.py @@ -1,6 +1,5 @@ import mailcap import os -import shutil import copy import test.support import unittest @@ -123,7 +122,7 @@ class HelperFunctionTest(unittest.TestCase): (["echo foo", "audio/*", "foo.txt"], "echo foo"), (["echo %s", "audio/*", "foo.txt"], "echo foo.txt"), (["echo %t", "audio/*", "foo.txt"], "echo audio/*"), - (["echo \%t", "audio/*", "foo.txt"], "echo %t"), + (["echo \\%t", "audio/*", "foo.txt"], "echo %t"), (["echo foo", "audio/*", "foo.txt", plist], "echo foo"), (["echo %{total}", "audio/*", "foo.txt", plist], "echo 3") ] diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index c7def9a599..b378ffea00 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -135,6 +135,13 @@ class ContainerTestCase(unittest.TestCase, HelperMixin): for constructor in (set, frozenset): self.helper(constructor(self.d.keys())) + @support.cpython_only + def test_empty_frozenset_singleton(self): + # marshal.loads() must reuse the empty frozenset singleton + obj = frozenset() + obj2 = marshal.loads(marshal.dumps(obj)) + self.assertIs(obj2, obj) + class BufferTestCase(unittest.TestCase, HelperMixin): diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index a379a6ad10..eaa41bca3f 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -7,14 +7,15 @@ import unittest import math import os import platform -import sys import struct +import sys import sysconfig eps = 1E-05 NAN = float('nan') INF = float('inf') NINF = float('-inf') +FLOAT_MAX = sys.float_info.max # detect evidence of double-rounding: fsum is not always correctly # rounded on machines that suffer from double rounding. @@ -30,6 +31,7 @@ test_dir = os.path.dirname(file) or os.curdir math_testcases = os.path.join(test_dir, 'math_testcases.txt') test_file = os.path.join(test_dir, 'cmath_testcases.txt') + def to_ulps(x): """Convert a non-NaN float x to an integer, in such a way that adjacent floats are converted to adjacent integers. Then @@ -37,25 +39,39 @@ def to_ulps(x): floats. The results from this function will only make sense on platforms - where C doubles are represented in IEEE 754 binary64 format. + where native doubles are represented in IEEE 754 binary64 format. + Note: 0.0 and -0.0 are converted to 0 and -1, respectively. """ n = struct.unpack('<q', struct.pack('<d', x))[0] if n < 0: n = ~(n+2**63) return n -def ulps_check(expected, got, ulps=20): - """Given non-NaN floats `expected` and `got`, - check that they're equal to within the given number of ulps. - Returns None on success and an error message on failure.""" +def ulp(x): + """Return the value of the least significant bit of a + float x, such that the first float bigger than x is x+ulp(x). + Then, given an expected result x and a tolerance of n ulps, + the result y should be such that abs(y-x) <= n * ulp(x). + The results from this function will only make sense on platforms + where native doubles are represented in IEEE 754 binary64 format. + """ + x = abs(float(x)) + if math.isnan(x) or math.isinf(x): + return x - ulps_error = to_ulps(got) - to_ulps(expected) - if abs(ulps_error) <= ulps: - return None - return "error = {} ulps; permitted error = {} ulps".format(ulps_error, - ulps) + # Find next float up from x. + n = struct.unpack('<q', struct.pack('<d', x))[0] + x_next = struct.unpack('<d', struct.pack('<q', n + 1))[0] + if math.isinf(x_next): + # Corner case: x was the largest finite float. Then it's + # not an exact power of two, so we can take the difference + # between x and the previous float. + x_prev = struct.unpack('<d', struct.pack('<q', n - 1))[0] + return x - x_prev + else: + return x_next - x # Here's a pure Python version of the math.factorial algorithm, for # documentation and comparison purposes. @@ -107,24 +123,23 @@ def py_factorial(n): outer *= inner return outer << (n - count_set_bits(n)) -def acc_check(expected, got, rel_err=2e-15, abs_err = 5e-323): - """Determine whether non-NaN floats a and b are equal to within a - (small) rounding error. The default values for rel_err and - abs_err are chosen to be suitable for platforms where a float is - represented by an IEEE 754 double. They allow an error of between - 9 and 19 ulps.""" - - # need to special case infinities, since inf - inf gives nan - if math.isinf(expected) and got == expected: - return None +def ulp_abs_check(expected, got, ulp_tol, abs_tol): + """Given finite floats `expected` and `got`, check that they're + approximately equal to within the given number of ulps or the + given absolute tolerance, whichever is bigger. - error = got - expected + Returns None on success and an error message on failure. + """ + ulp_error = abs(to_ulps(expected) - to_ulps(got)) + abs_error = abs(expected - got) - permitted_error = max(abs_err, rel_err * abs(expected)) - if abs(error) < permitted_error: + # Succeed if either abs_error <= abs_tol or ulp_error <= ulp_tol. + if abs_error <= abs_tol or ulp_error <= ulp_tol: return None - return "error = {}; permitted error = {}".format(error, - permitted_error) + else: + fmt = ("error = {:.3g} ({:d} ulps); " + "permitted error = {:.3g} or {:d} ulps") + return fmt.format(abs_error, ulp_error, abs_tol, ulp_tol) def parse_mtestfile(fname): """Parse a file with test values @@ -151,6 +166,7 @@ def parse_mtestfile(fname): yield (id, fn, float(arg), float(exp), flags) + def parse_testfile(fname): """Parse a file with test values @@ -172,8 +188,53 @@ def parse_testfile(fname): yield (id, fn, float(arg_real), float(arg_imag), float(exp_real), float(exp_imag), - flags - ) + flags) + + +def result_check(expected, got, ulp_tol=5, abs_tol=0.0): + # Common logic of MathTests.(ftest, test_testcases, test_mtestcases) + """Compare arguments expected and got, as floats, if either + is a float, using a tolerance expressed in multiples of + ulp(expected) or absolutely (if given and greater). + + As a convenience, when neither argument is a float, and for + non-finite floats, exact equality is demanded. Also, nan==nan + as far as this function is concerned. + + Returns None on success and an error message on failure. + """ + + # Check exactly equal (applies also to strings representing exceptions) + if got == expected: + return None + + failure = "not equal" + + # Turn mixed float and int comparison (e.g. floor()) to all-float + if isinstance(expected, float) and isinstance(got, int): + got = float(got) + elif isinstance(got, float) and isinstance(expected, int): + expected = float(expected) + + if isinstance(expected, float) and isinstance(got, float): + if math.isnan(expected) and math.isnan(got): + # Pass, since both nan + failure = None + elif math.isinf(expected) or math.isinf(got): + # We already know they're not equal, drop through to failure + pass + else: + # Both are finite floats (now). Are they close enough? + failure = ulp_abs_check(expected, got, ulp_tol, abs_tol) + + # arguments are not equal, and if numeric, are too far apart + if failure is not None: + fail_fmt = "expected {!r}, got {!r}" + fail_msg = fail_fmt.format(expected, got) + fail_msg += ' ({})'.format(failure) + return fail_msg + else: + return None # Class providing an __index__ method. class MyIndexable(object): @@ -185,18 +246,24 @@ class MyIndexable(object): class MathTests(unittest.TestCase): - def ftest(self, name, value, expected): - if abs(value-expected) > eps: - # Use %r instead of %f so the error message - # displays full precision. Otherwise discrepancies - # in the last few bits will lead to very confusing - # error messages - self.fail('%s returned %r, expected %r' % - (name, value, expected)) + def ftest(self, name, got, expected, ulp_tol=5, abs_tol=0.0): + """Compare arguments expected and got, as floats, if either + is a float, using a tolerance expressed in multiples of + ulp(expected) or absolutely, whichever is greater. + + As a convenience, when neither argument is a float, and for + non-finite floats, exact equality is demanded. Also, nan==nan + in this function. + """ + failure = result_check(expected, got, ulp_tol, abs_tol) + if failure is not None: + self.fail("{}: {}".format(name, failure)) def testConstants(self): - self.ftest('pi', math.pi, 3.1415926) - self.ftest('e', math.e, 2.7182818) + # Ref: Abramowitz & Stegun (Dover, 1965) + self.ftest('pi', math.pi, 3.141592653589793238462643) + self.ftest('e', math.e, 2.718281828459045235360287) + self.assertEqual(math.tau, 2*math.pi) def testAcos(self): self.assertRaises(TypeError, math.acos) @@ -205,6 +272,8 @@ class MathTests(unittest.TestCase): self.ftest('acos(1)', math.acos(1), 0) self.assertRaises(ValueError, math.acos, INF) self.assertRaises(ValueError, math.acos, NINF) + self.assertRaises(ValueError, math.acos, 1 + eps) + self.assertRaises(ValueError, math.acos, -1 - eps) self.assertTrue(math.isnan(math.acos(NAN))) def testAcosh(self): @@ -224,6 +293,8 @@ class MathTests(unittest.TestCase): self.ftest('asin(1)', math.asin(1), math.pi/2) self.assertRaises(ValueError, math.asin, INF) self.assertRaises(ValueError, math.asin, NINF) + self.assertRaises(ValueError, math.asin, 1 + eps) + self.assertRaises(ValueError, math.asin, -1 - eps) self.assertTrue(math.isnan(math.asin(NAN))) def testAsinh(self): @@ -378,9 +449,9 @@ class MathTests(unittest.TestCase): def testCos(self): self.assertRaises(TypeError, math.cos) - self.ftest('cos(-pi/2)', math.cos(-math.pi/2), 0) + self.ftest('cos(-pi/2)', math.cos(-math.pi/2), 0, abs_tol=ulp(1)) self.ftest('cos(0)', math.cos(0), 1) - self.ftest('cos(pi/2)', math.cos(math.pi/2), 0) + self.ftest('cos(pi/2)', math.cos(math.pi/2), 0, abs_tol=ulp(1)) self.ftest('cos(pi)', math.cos(math.pi), -1) try: self.assertTrue(math.isnan(math.cos(INF))) @@ -403,6 +474,7 @@ class MathTests(unittest.TestCase): self.ftest('degrees(pi)', math.degrees(math.pi), 180.0) self.ftest('degrees(pi/2)', math.degrees(math.pi/2), 90.0) self.ftest('degrees(-pi/4)', math.degrees(-math.pi/4), -45.0) + self.ftest('degrees(0)', math.degrees(0), 0) def testExp(self): self.assertRaises(TypeError, math.exp) @@ -412,6 +484,7 @@ class MathTests(unittest.TestCase): self.assertEqual(math.exp(INF), INF) self.assertEqual(math.exp(NINF), 0.) self.assertTrue(math.isnan(math.exp(NAN))) + self.assertRaises(OverflowError, math.exp, 1000000) def testFabs(self): self.assertRaises(TypeError, math.fabs) @@ -654,6 +727,7 @@ class MathTests(unittest.TestCase): self.assertEqual(math.hypot(INF, NAN), INF) self.assertEqual(math.hypot(NAN, NINF), INF) self.assertEqual(math.hypot(NINF, NAN), INF) + self.assertRaises(OverflowError, math.hypot, FLOAT_MAX, FLOAT_MAX) self.assertTrue(math.isnan(math.hypot(1.0, NAN))) self.assertTrue(math.isnan(math.hypot(NAN, -2.0))) @@ -707,8 +781,10 @@ class MathTests(unittest.TestCase): def testLog1p(self): self.assertRaises(TypeError, math.log1p) - n= 2**90 - self.assertAlmostEqual(math.log1p(n), math.log1p(float(n))) + for n in [2, 2**90, 2**300]: + self.assertAlmostEqual(math.log1p(n), math.log1p(float(n))) + self.assertRaises(ValueError, math.log1p, -1) + self.assertEqual(math.log1p(INF), INF) @requires_IEEE_754 def testLog2(self): @@ -922,6 +998,7 @@ class MathTests(unittest.TestCase): self.ftest('radians(180)', math.radians(180), math.pi) self.ftest('radians(90)', math.radians(90), math.pi/2) self.ftest('radians(-45)', math.radians(-45), -math.pi/4) + self.ftest('radians(0)', math.radians(0), 0) def testSin(self): self.assertRaises(TypeError, math.sin) @@ -951,6 +1028,7 @@ class MathTests(unittest.TestCase): self.ftest('sqrt(1)', math.sqrt(1), 1) self.ftest('sqrt(4)', math.sqrt(4), 2) self.assertEqual(math.sqrt(INF), INF) + self.assertRaises(ValueError, math.sqrt, -1) self.assertRaises(ValueError, math.sqrt, NINF) self.assertTrue(math.isnan(math.sqrt(NAN))) @@ -970,7 +1048,8 @@ class MathTests(unittest.TestCase): def testTanh(self): self.assertRaises(TypeError, math.tanh) self.ftest('tanh(0)', math.tanh(0), 0) - self.ftest('tanh(1)+tanh(-1)', math.tanh(1)+math.tanh(-1), 0) + self.ftest('tanh(1)+tanh(-1)', math.tanh(1)+math.tanh(-1), 0, + abs_tol=ulp(1)) self.ftest('tanh(inf)', math.tanh(INF), 1) self.ftest('tanh(-inf)', math.tanh(NINF), -1) self.assertTrue(math.isnan(math.tanh(NAN))) @@ -1020,7 +1099,8 @@ class MathTests(unittest.TestCase): def testIsnan(self): self.assertTrue(math.isnan(float("nan"))) - self.assertTrue(math.isnan(float("inf")* 0.)) + self.assertTrue(math.isnan(float("-nan"))) + self.assertTrue(math.isnan(float("inf") * 0.)) self.assertFalse(math.isnan(float("inf"))) self.assertFalse(math.isnan(0.)) self.assertFalse(math.isnan(1.)) @@ -1084,30 +1164,64 @@ class MathTests(unittest.TestCase): @requires_IEEE_754 def test_testfile(self): + # Some tests need to be skipped on ancient OS X versions. + # See issue #27953. + SKIP_ON_TIGER = {'tan0064'} + + osx_version = None + if sys.platform == 'darwin': + version_txt = platform.mac_ver()[0] + try: + osx_version = tuple(map(int, version_txt.split('.'))) + except ValueError: + pass + + fail_fmt = "{}: {}({!r}): {}" + + failures = [] for id, fn, ar, ai, er, ei, flags in parse_testfile(test_file): - # Skip if either the input or result is complex, or if - # flags is nonempty - if ai != 0. or ei != 0. or flags: + # Skip if either the input or result is complex + if ai != 0.0 or ei != 0.0: continue if fn in ['rect', 'polar']: # no real versions of rect, polar continue + # Skip certain tests on OS X 10.4. + if osx_version is not None and osx_version < (10, 5): + if id in SKIP_ON_TIGER: + continue + func = getattr(math, fn) + + if 'invalid' in flags or 'divide-by-zero' in flags: + er = 'ValueError' + elif 'overflow' in flags: + er = 'OverflowError' + try: result = func(ar) - except ValueError as exc: - message = (("Unexpected ValueError: %s\n " + - "in test %s:%s(%r)\n") % (exc.args[0], id, fn, ar)) - self.fail(message) + except ValueError: + result = 'ValueError' except OverflowError: - message = ("Unexpected OverflowError in " + - "test %s:%s(%r)\n" % (id, fn, ar)) - self.fail(message) - self.ftest("%s:%s(%r)" % (id, fn, ar), result, er) + result = 'OverflowError' + + # Default tolerances + ulp_tol, abs_tol = 5, 0.0 + + failure = result_check(er, result, ulp_tol, abs_tol) + if failure is None: + continue + + msg = fail_fmt.format(id, fn, ar, failure) + failures.append(msg) + + if failures: + self.fail('Failures in test_testfile:\n ' + + '\n '.join(failures)) @requires_IEEE_754 def test_mtestfile(self): - fail_fmt = "{}:{}({!r}): expected {!r}, got {!r}" + fail_fmt = "{}: {}({!r}): {}" failures = [] for id, fn, arg, expected, flags in parse_mtestfile(math_testcases): @@ -1125,41 +1239,48 @@ class MathTests(unittest.TestCase): except OverflowError: got = 'OverflowError' - accuracy_failure = None - if isinstance(got, float) and isinstance(expected, float): - if math.isnan(expected) and math.isnan(got): - continue - if not math.isnan(expected) and not math.isnan(got): - if fn == 'lgamma': - # we use a weaker accuracy test for lgamma; - # lgamma only achieves an absolute error of - # a few multiples of the machine accuracy, in - # general. - accuracy_failure = acc_check(expected, got, - rel_err = 5e-15, - abs_err = 5e-15) - elif fn == 'erfc': - # erfc has less-than-ideal accuracy for large - # arguments (x ~ 25 or so), mainly due to the - # error involved in computing exp(-x*x). - # - # XXX Would be better to weaken this test only - # for large x, instead of for all x. - accuracy_failure = ulps_check(expected, got, 2000) - - else: - accuracy_failure = ulps_check(expected, got, 20) - if accuracy_failure is None: - continue - - if isinstance(got, str) and isinstance(expected, str): - if got == expected: - continue + # Default tolerances + ulp_tol, abs_tol = 5, 0.0 + + # Exceptions to the defaults + if fn == 'gamma': + # Experimental results on one platform gave + # an accuracy of <= 10 ulps across the entire float + # domain. We weaken that to require 20 ulp accuracy. + ulp_tol = 20 + + elif fn == 'lgamma': + # we use a weaker accuracy test for lgamma; + # lgamma only achieves an absolute error of + # a few multiples of the machine accuracy, in + # general. + abs_tol = 1e-15 + + elif fn == 'erfc' and arg >= 0.0: + # erfc has less-than-ideal accuracy for large + # arguments (x ~ 25 or so), mainly due to the + # error involved in computing exp(-x*x). + # + # Observed between CPython and mpmath at 25 dp: + # x < 0 : err <= 2 ulp + # 0 <= x < 1 : err <= 10 ulp + # 1 <= x < 10 : err <= 100 ulp + # 10 <= x < 20 : err <= 300 ulp + # 20 <= x : < 600 ulp + # + if arg < 1.0: + ulp_tol = 10 + elif arg < 10.0: + ulp_tol = 100 + else: + ulp_tol = 1000 + + failure = result_check(expected, got, ulp_tol, abs_tol) + if failure is None: + continue - fail_msg = fail_fmt.format(id, fn, arg, expected, got) - if accuracy_failure is not None: - fail_msg += ' ({})'.format(accuracy_failure) - failures.append(fail_msg) + msg = fail_fmt.format(id, fn, arg, failure) + failures.append(msg) if failures: self.fail('Failures in test_mtestfile:\n ' + @@ -1272,7 +1393,8 @@ class IsCloseTests(unittest.TestCase): decimal_examples = [(Decimal('1.00000001'), Decimal('1.0')), (Decimal('1.00000001e-20'), Decimal('1.0e-20')), - (Decimal('1.00000001e-100'), Decimal('1.0e-100'))] + (Decimal('1.00000001e-100'), Decimal('1.0e-100')), + (Decimal('1.00000001e20'), Decimal('1.0e20'))] self.assertAllClose(decimal_examples, rel_tol=1e-8) self.assertAllNotClose(decimal_examples, rel_tol=1e-9) @@ -1280,8 +1402,10 @@ class IsCloseTests(unittest.TestCase): # test with Fraction values from fractions import Fraction - # could use some more examples here! - fraction_examples = [(Fraction(1, 100000000) + 1, Fraction(1))] + fraction_examples = [ + (Fraction(1, 100000000) + 1, Fraction(1)), + (Fraction(100000001), Fraction(100000000)), + (Fraction(10**8 + 1, 10**28), Fraction(1, 10**20))] self.assertAllClose(fraction_examples, rel_tol=1e-8) self.assertAllNotClose(fraction_examples, rel_tol=1e-9) diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 68565930a4..4e2c9089b5 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -101,5 +101,11 @@ class Win32MimeTypesTestCase(unittest.TestCase): eq(self.db.guess_type("image.jpg"), ("image/jpeg", None)) eq(self.db.guess_type("image.png"), ("image/png", None)) + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, mimetypes) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index f3f70cc251..56d85e75f8 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -713,6 +713,13 @@ class MmapTests(unittest.TestCase): gc_collect() self.assertIs(wr(), None) + def test_write_returning_the_number_of_bytes_written(self): + mm = mmap.mmap(-1, 16) + self.assertEqual(mm.write(b""), 0) + self.assertEqual(mm.write(b"x"), 1) + self.assertEqual(mm.write(b"yz"), 2) + self.assertEqual(mm.write(b"python"), 6) + @unittest.skipIf(os.name == 'nt', 'cannot resize anonymous mmaps on Windows') def test_resize_past_pos(self): m = mmap.mmap(-1, 8192) diff --git a/Lib/test/test_msilib.py b/Lib/test/test_msilib.py index 8ef334fdb3..f656f7234e 100644 --- a/Lib/test/test_msilib.py +++ b/Lib/test/test_msilib.py @@ -1,6 +1,5 @@ """ Test suite for the code in msilib """ import unittest -import os from test.support import import_module msilib = import_module('msilib') diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 8d7a213c16..01a1cd3c69 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -5,7 +5,7 @@ from test import support from test.support import TESTFN -import unittest, io, codecs, sys, os +import unittest, io, codecs, sys import _multibytecodec ALL_CJKENCODINGS = [ diff --git a/Lib/test/test_multiprocessing_fork.py b/Lib/test/test_multiprocessing_fork.py index 2bf4e75644..9f22500e8f 100644 --- a/Lib/test/test_multiprocessing_fork.py +++ b/Lib/test/test_multiprocessing_fork.py @@ -1,6 +1,12 @@ import unittest import test._test_multiprocessing +from test import support + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + + test._test_multiprocessing.install_tests_in_module_dict(globals(), 'fork') if __name__ == '__main__': diff --git a/Lib/test/test_multiprocessing_forkserver.py b/Lib/test/test_multiprocessing_forkserver.py index 193a04a5fc..407bb3f9b6 100644 --- a/Lib/test/test_multiprocessing_forkserver.py +++ b/Lib/test/test_multiprocessing_forkserver.py @@ -1,6 +1,11 @@ import unittest import test._test_multiprocessing +from test import support + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + test._test_multiprocessing.install_tests_in_module_dict(globals(), 'forkserver') if __name__ == '__main__': diff --git a/Lib/test/test_multiprocessing_main_handling.py b/Lib/test/test_multiprocessing_main_handling.py index 52273ea22d..32593dab86 100644 --- a/Lib/test/test_multiprocessing_main_handling.py +++ b/Lib/test/test_multiprocessing_main_handling.py @@ -6,7 +6,6 @@ support.import_module('_multiprocessing') import importlib import importlib.machinery -import zipimport import unittest import sys import os @@ -15,7 +14,10 @@ import py_compile from test.support.script_helper import ( make_pkg, make_script, make_zip_pkg, make_zip_script, - assert_python_ok, assert_python_failure, spawn_python, kill_python) + assert_python_ok) + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") # Look up which start methods are available to test import multiprocessing diff --git a/Lib/test/test_multiprocessing_spawn.py b/Lib/test/test_multiprocessing_spawn.py index 334ae9e8f7..6558952308 100644 --- a/Lib/test/test_multiprocessing_spawn.py +++ b/Lib/test/test_multiprocessing_spawn.py @@ -1,6 +1,11 @@ import unittest import test._test_multiprocessing +from test import support + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + test._test_multiprocessing.install_tests_in_module_dict(globals(), 'spawn') if __name__ == '__main__': diff --git a/Lib/test/test_nis.py b/Lib/test/test_nis.py index 387a4e7856..21074c6810 100644 --- a/Lib/test/test_nis.py +++ b/Lib/test/test_nis.py @@ -1,6 +1,5 @@ from test import support import unittest -import sys # Skip test if nis module does not exist. nis = support.import_module('nis') diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py index 66ef930a54..c46ded1186 100644 --- a/Lib/test/test_nntplib.py +++ b/Lib/test/test_nntplib.py @@ -1544,8 +1544,10 @@ class LocalServerTests(unittest.TestCase): elif cmd == b'STARTTLS\r\n': reader.close() client.sendall(b'382 Begin TLS negotiation now\r\n') - client = ssl.wrap_socket( - client, server_side=True, certfile=certfile) + context = ssl.SSLContext() + context.load_cert_chain(certfile) + client = context.wrap_socket( + client, server_side=True) cleanup.enter_context(client) reader = cleanup.enter_context(client.makefile('rb')) elif cmd == b'QUIT\r\n': diff --git a/Lib/test/test_normalization.py b/Lib/test/test_normalization.py index 30fa612645..5b590e1af3 100644 --- a/Lib/test/test_normalization.py +++ b/Lib/test/test_normalization.py @@ -3,7 +3,6 @@ import unittest from http.client import HTTPException import sys -import os from unicodedata import normalize, unidata_version TESTDATAFILE = "NormalizationTest.txt" diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 580f2030a3..90edb6d080 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -452,5 +452,88 @@ class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase): attributes = ['relpath', 'splitunc'] +class PathLikeTests(unittest.TestCase): + + path = ntpath + + class PathLike: + def __init__(self, path=''): + self.path = path + def __fspath__(self): + if isinstance(self.path, BaseException): + raise self.path + else: + return self.path + + def setUp(self): + self.file_name = support.TESTFN.lower() + self.file_path = self.PathLike(support.TESTFN) + self.addCleanup(support.unlink, self.file_name) + with open(self.file_name, 'xb', 0) as file: + file.write(b"test_ntpath.PathLikeTests") + + def assertPathEqual(self, func): + self.assertEqual(func(self.file_path), func(self.file_name)) + + def test_path_normcase(self): + self.assertPathEqual(self.path.normcase) + + def test_path_isabs(self): + self.assertPathEqual(self.path.isabs) + + def test_path_join(self): + self.assertEqual(self.path.join('a', self.PathLike('b'), 'c'), + self.path.join('a', 'b', 'c')) + + def test_path_split(self): + self.assertPathEqual(self.path.split) + + def test_path_splitext(self): + self.assertPathEqual(self.path.splitext) + + def test_path_splitdrive(self): + self.assertPathEqual(self.path.splitdrive) + + def test_path_basename(self): + self.assertPathEqual(self.path.basename) + + def test_path_dirname(self): + self.assertPathEqual(self.path.dirname) + + def test_path_islink(self): + self.assertPathEqual(self.path.islink) + + def test_path_lexists(self): + self.assertPathEqual(self.path.lexists) + + def test_path_ismount(self): + self.assertPathEqual(self.path.ismount) + + def test_path_expanduser(self): + self.assertPathEqual(self.path.expanduser) + + def test_path_expandvars(self): + self.assertPathEqual(self.path.expandvars) + + def test_path_normpath(self): + self.assertPathEqual(self.path.normpath) + + def test_path_abspath(self): + self.assertPathEqual(self.path.abspath) + + def test_path_realpath(self): + self.assertPathEqual(self.path.realpath) + + def test_path_relpath(self): + self.assertPathEqual(self.path.relpath) + + def test_path_commonpath(self): + common_path = self.path.commonpath([self.file_path, self.file_name]) + self.assertEqual(common_path, self.file_name) + + def test_path_isdir(self): + self.assertPathEqual(self.path.isdir) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index 6ef93d9500..6806c616cb 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -1,6 +1,7 @@ # Python test set -- part 2, opcodes import unittest +from test import ann_module class OpcodeTest(unittest.TestCase): @@ -20,6 +21,32 @@ class OpcodeTest(unittest.TestCase): if n != 90: self.fail('try inside for') + def test_setup_annotations_line(self): + # check that SETUP_ANNOTATIONS does not create spurious line numbers + try: + with open(ann_module.__file__) as f: + txt = f.read() + co = compile(txt, ann_module.__file__, 'exec') + self.assertEqual(co.co_firstlineno, 6) + except OSError: + pass + + def test_no_annotations_if_not_needed(self): + class C: pass + with self.assertRaises(AttributeError): + C.__annotations__ + + def test_use_existing_annotations(self): + ns = {'__annotations__': {1: 2}} + exec('x: int', ns) + self.assertEqual(ns['__annotations__'], {'x': int, 1: 2}) + + def test_do_not_recreate_annotations(self): + class C: + del __annotations__ + with self.assertRaises(NameError): + x: int + def test_raise_class_exceptions(self): class AClass(Exception): pass diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index b5ba97625f..6254091e78 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -120,63 +120,63 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.add) self.assertRaises(TypeError, operator.add, None, None) - self.assertTrue(operator.add(3, 4) == 7) + self.assertEqual(operator.add(3, 4), 7) def test_bitwise_and(self): operator = self.module self.assertRaises(TypeError, operator.and_) self.assertRaises(TypeError, operator.and_, None, None) - self.assertTrue(operator.and_(0xf, 0xa) == 0xa) + self.assertEqual(operator.and_(0xf, 0xa), 0xa) def test_concat(self): operator = self.module self.assertRaises(TypeError, operator.concat) self.assertRaises(TypeError, operator.concat, None, None) - self.assertTrue(operator.concat('py', 'thon') == 'python') - self.assertTrue(operator.concat([1, 2], [3, 4]) == [1, 2, 3, 4]) - self.assertTrue(operator.concat(Seq1([5, 6]), Seq1([7])) == [5, 6, 7]) - self.assertTrue(operator.concat(Seq2([5, 6]), Seq2([7])) == [5, 6, 7]) + self.assertEqual(operator.concat('py', 'thon'), 'python') + self.assertEqual(operator.concat([1, 2], [3, 4]), [1, 2, 3, 4]) + self.assertEqual(operator.concat(Seq1([5, 6]), Seq1([7])), [5, 6, 7]) + self.assertEqual(operator.concat(Seq2([5, 6]), Seq2([7])), [5, 6, 7]) self.assertRaises(TypeError, operator.concat, 13, 29) def test_countOf(self): operator = self.module self.assertRaises(TypeError, operator.countOf) self.assertRaises(TypeError, operator.countOf, None, None) - self.assertTrue(operator.countOf([1, 2, 1, 3, 1, 4], 3) == 1) - self.assertTrue(operator.countOf([1, 2, 1, 3, 1, 4], 5) == 0) + self.assertEqual(operator.countOf([1, 2, 1, 3, 1, 4], 3), 1) + self.assertEqual(operator.countOf([1, 2, 1, 3, 1, 4], 5), 0) def test_delitem(self): operator = self.module a = [4, 3, 2, 1] self.assertRaises(TypeError, operator.delitem, a) self.assertRaises(TypeError, operator.delitem, a, None) - self.assertTrue(operator.delitem(a, 1) is None) - self.assertTrue(a == [4, 2, 1]) + self.assertIsNone(operator.delitem(a, 1)) + self.assertEqual(a, [4, 2, 1]) def test_floordiv(self): operator = self.module self.assertRaises(TypeError, operator.floordiv, 5) self.assertRaises(TypeError, operator.floordiv, None, None) - self.assertTrue(operator.floordiv(5, 2) == 2) + self.assertEqual(operator.floordiv(5, 2), 2) def test_truediv(self): operator = self.module self.assertRaises(TypeError, operator.truediv, 5) self.assertRaises(TypeError, operator.truediv, None, None) - self.assertTrue(operator.truediv(5, 2) == 2.5) + self.assertEqual(operator.truediv(5, 2), 2.5) def test_getitem(self): operator = self.module a = range(10) self.assertRaises(TypeError, operator.getitem) self.assertRaises(TypeError, operator.getitem, a, None) - self.assertTrue(operator.getitem(a, 2) == 2) + self.assertEqual(operator.getitem(a, 2), 2) def test_indexOf(self): operator = self.module self.assertRaises(TypeError, operator.indexOf) self.assertRaises(TypeError, operator.indexOf, None, None) - self.assertTrue(operator.indexOf([4, 3, 2, 1], 3) == 1) + self.assertEqual(operator.indexOf([4, 3, 2, 1], 3), 1) self.assertRaises(ValueError, operator.indexOf, [4, 3, 2, 1], 0) def test_invert(self): @@ -189,21 +189,21 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.lshift) self.assertRaises(TypeError, operator.lshift, None, 42) - self.assertTrue(operator.lshift(5, 1) == 10) - self.assertTrue(operator.lshift(5, 0) == 5) + self.assertEqual(operator.lshift(5, 1), 10) + self.assertEqual(operator.lshift(5, 0), 5) self.assertRaises(ValueError, operator.lshift, 2, -1) def test_mod(self): operator = self.module self.assertRaises(TypeError, operator.mod) self.assertRaises(TypeError, operator.mod, None, 42) - self.assertTrue(operator.mod(5, 2) == 1) + self.assertEqual(operator.mod(5, 2), 1) def test_mul(self): operator = self.module self.assertRaises(TypeError, operator.mul) self.assertRaises(TypeError, operator.mul, None, None) - self.assertTrue(operator.mul(5, 2) == 10) + self.assertEqual(operator.mul(5, 2), 10) def test_matmul(self): operator = self.module @@ -227,7 +227,7 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.or_) self.assertRaises(TypeError, operator.or_, None, None) - self.assertTrue(operator.or_(0xa, 0x5) == 0xf) + self.assertEqual(operator.or_(0xa, 0x5), 0xf) def test_pos(self): operator = self.module @@ -250,8 +250,8 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.rshift) self.assertRaises(TypeError, operator.rshift, None, 42) - self.assertTrue(operator.rshift(5, 1) == 2) - self.assertTrue(operator.rshift(5, 0) == 5) + self.assertEqual(operator.rshift(5, 1), 2) + self.assertEqual(operator.rshift(5, 0), 5) self.assertRaises(ValueError, operator.rshift, 2, -1) def test_contains(self): @@ -266,15 +266,15 @@ class OperatorTestCase: a = list(range(3)) self.assertRaises(TypeError, operator.setitem, a) self.assertRaises(TypeError, operator.setitem, a, None, None) - self.assertTrue(operator.setitem(a, 0, 2) is None) - self.assertTrue(a == [2, 1, 2]) + self.assertIsNone(operator.setitem(a, 0, 2)) + self.assertEqual(a, [2, 1, 2]) self.assertRaises(IndexError, operator.setitem, a, 4, 2) def test_sub(self): operator = self.module self.assertRaises(TypeError, operator.sub) self.assertRaises(TypeError, operator.sub, None, None) - self.assertTrue(operator.sub(5, 2) == 3) + self.assertEqual(operator.sub(5, 2), 3) def test_truth(self): operator = self.module @@ -292,7 +292,7 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.xor) self.assertRaises(TypeError, operator.xor, None, None) - self.assertTrue(operator.xor(0xb, 0xc) == 0x7) + self.assertEqual(operator.xor(0xb, 0xc), 0x7) def test_is(self): operator = self.module diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index 7621c24305..91a0319a73 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -16,6 +16,7 @@ from io import StringIO from test import support +import optparse from optparse import make_option, Option, \ TitledHelpFormatter, OptionParser, OptionGroup, \ SUPPRESS_USAGE, OptionError, OptionConflictError, \ @@ -1650,6 +1651,12 @@ class TestParseNumber(BaseTest): "option -l: invalid integer value: '0x12x'") +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'check_builtin', 'AmbiguousOptionError', 'NO_DEFAULT'} + support.check__all__(self, optparse, blacklist=blacklist) + + def test_main(): support.run_unittest(__name__) diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 901d4b2ad2..93f812a530 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -1,3 +1,4 @@ +import builtins import contextlib import copy import gc @@ -51,6 +52,14 @@ class OrderedDictTests: self.assertEqual(list(d.items()), [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5), ('f', 6), ('g', 7)]) + def test_468(self): + OrderedDict = self.OrderedDict + items = [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5), ('f', 6), ('g', 7)] + shuffle(items) + argdict = OrderedDict(items) + d = OrderedDict(**argdict) + self.assertEqual(list(d.items()), items) + def test_update(self): OrderedDict = self.OrderedDict with self.assertRaises(TypeError): @@ -97,6 +106,19 @@ class OrderedDictTests: self.assertRaises(TypeError, OrderedDict().update, (), ()) self.assertRaises(TypeError, OrderedDict.update) + def test_init_calls(self): + calls = [] + class Spam: + def keys(self): + calls.append('keys') + return () + def items(self): + calls.append('items') + return () + + self.OrderedDict(Spam()) + self.assertEqual(calls, ['keys']) + def test_fromkeys(self): OrderedDict = self.OrderedDict od = OrderedDict.fromkeys('abc') @@ -298,9 +320,11 @@ class OrderedDictTests: # do not save instance dictionary if not needed pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] od = OrderedDict(pairs) + self.assertIsInstance(od.__dict__, dict) self.assertIsNone(od.__reduce__()[2]) od.x = 10 - self.assertIsNotNone(od.__reduce__()[2]) + self.assertEqual(od.__dict__['x'], 10) + self.assertEqual(od.__reduce__()[2], {'x': 10}) def test_pickle_recursive(self): OrderedDict = self.OrderedDict @@ -403,6 +427,14 @@ class OrderedDictTests: od = OrderedDict(**d) self.assertGreater(sys.getsizeof(od), sys.getsizeof(d)) + def test_views(self): + OrderedDict = self.OrderedDict + # See http://bugs.python.org/issue24286 + s = 'the quick brown fox jumped over a lazy dog yesterday before dawn'.split() + od = OrderedDict.fromkeys(s) + self.assertEqual(od.keys(), dict(od).keys()) + self.assertEqual(od.items(), dict(od).items()) + def test_override_update(self): OrderedDict = self.OrderedDict # Verify that subclasses can override update() without breaking __init__() @@ -611,6 +643,25 @@ class PurePythonOrderedDictTests(OrderedDictTests, unittest.TestCase): OrderedDict = py_coll.OrderedDict +class CPythonBuiltinDictTests(unittest.TestCase): + """Builtin dict preserves insertion order. + + Reuse some of tests in OrderedDict selectively. + """ + + module = builtins + OrderedDict = dict + +for method in ( + "test_init test_update test_abc test_clear test_delitem " + + "test_setitem test_detect_deletion_during_iteration " + + "test_popitem test_reinsert test_override_update " + + "test_highly_nested test_highly_nested_subclass " + + "test_delitem_hash_collision ").split(): + setattr(CPythonBuiltinDictTests, method, getattr(OrderedDictTests, method)) +del method + + @unittest.skipUnless(c_coll, 'requires the C version of the collections module') class CPythonOrderedDictTests(OrderedDictTests, unittest.TestCase): @@ -625,18 +676,20 @@ class CPythonOrderedDictTests(OrderedDictTests, unittest.TestCase): size = support.calcobjsize check = self.check_sizeof - basicsize = size('n2P' + '3PnPn2P') + calcsize('2nPn') - entrysize = calcsize('n2P') + calcsize('P') + basicsize = size('nQ2P' + '3PnPn2P') + calcsize('2nP2n') + + entrysize = calcsize('n2P') + p = calcsize('P') nodesize = calcsize('Pn2P') od = OrderedDict() - check(od, basicsize + 8*entrysize) + check(od, basicsize + 8*p + 8 + 5*entrysize) # 8byte indicies + 8*2//3 * entry table od.x = 1 - check(od, basicsize + 8*entrysize) + check(od, basicsize + 8*p + 8 + 5*entrysize) od.update([(i, i) for i in range(3)]) - check(od, basicsize + 8*entrysize + 3*nodesize) + check(od, basicsize + 8*p + 8 + 5*entrysize + 3*nodesize) od.update([(i, i) for i in range(3, 10)]) - check(od, basicsize + 16*entrysize + 10*nodesize) + check(od, basicsize + 16*p + 16 + 10*entrysize + 10*nodesize) check(od.keys(), size('P')) check(od.items(), size('P')) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 10383cfc1f..e9fdb0719f 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -15,7 +15,6 @@ import locale import mmap import os import pickle -import platform import re import shutil import signal @@ -57,7 +56,7 @@ except ImportError: try: import pwd all_users = [u.pw_uid for u in pwd.getpwall()] -except ImportError: +except (ImportError, AttributeError): all_users = [] try: from _testcapi import INT_MAX, PY_SSIZE_T_MAX @@ -65,6 +64,8 @@ except ImportError: INT_MAX = PY_SSIZE_T_MAX = sys.maxsize from test.support.script_helper import assert_python_ok +from test.support import unix_shell + root_in_posix = False if hasattr(os, 'geteuid'): @@ -82,6 +83,32 @@ else: # Issue #14110: Some tests fail on FreeBSD if the user is in the wheel group. HAVE_WHEEL_GROUP = sys.platform.startswith('freebsd') and os.getgid() == 0 + +@contextlib.contextmanager +def ignore_deprecation_warnings(msg_regex, quiet=False): + with support.check_warnings((msg_regex, DeprecationWarning), quiet=quiet): + yield + + +def requires_os_func(name): + return unittest.skipUnless(hasattr(os, name), 'requires os.%s' % name) + + +class _PathLike(os.PathLike): + + def __init__(self, path=""): + self.path = path + + def __str__(self): + return str(self.path) + + def __fspath__(self): + if isinstance(self.path, BaseException): + raise self.path + else: + return self.path + + def create_file(filename, content=b'content'): with open(filename, "xb", 0) as fp: fp.write(content) @@ -145,9 +172,8 @@ class FileTests(unittest.TestCase): "needs INT_MAX < PY_SSIZE_T_MAX") @support.bigmemtest(size=INT_MAX + 10, memuse=1, dry_run=False) def test_large_read(self, size): - with open(support.TESTFN, "wb") as fp: - fp.write(b'test') self.addCleanup(support.unlink, support.TESTFN) + create_file(support.TESTFN, b'test') # Issue #21932: Make sure that os.read() does not raise an # OverflowError for size larger than INT_MAX @@ -204,11 +230,12 @@ class FileTests(unittest.TestCase): def test_replace(self): TESTFN2 = support.TESTFN + ".2" - with open(support.TESTFN, 'w') as f: - f.write("1") - with open(TESTFN2, 'w') as f: - f.write("2") - self.addCleanup(os.unlink, TESTFN2) + self.addCleanup(support.unlink, support.TESTFN) + self.addCleanup(support.unlink, TESTFN2) + + create_file(support.TESTFN, b"1") + create_file(TESTFN2, b"2") + os.replace(support.TESTFN, TESTFN2) self.assertRaises(FileNotFoundError, os.stat, support.TESTFN) with open(TESTFN2, 'r') as f: @@ -309,9 +336,7 @@ class StatAttributeTests(unittest.TestCase): fname = self.fname.encode(sys.getfilesystemencoding()) except UnicodeEncodeError: self.skipTest("cannot encode %a for the filesystem" % self.fname) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - self.check_stat_attributes(fname) + self.check_stat_attributes(fname) def test_stat_result_pickle(self): result = os.stat(self.fname) @@ -462,19 +487,14 @@ class UtimeTests(unittest.TestCase): self.addCleanup(support.rmtree, self.dirname) os.mkdir(self.dirname) - with open(self.fname, 'wb') as fp: - fp.write(b"ABC") + create_file(self.fname) def restore_float_times(state): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - + with ignore_deprecation_warnings('stat_float_times'): os.stat_float_times(state) # ensure that st_atime and st_mtime are float - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - + with ignore_deprecation_warnings('stat_float_times'): old_float_times = os.stat_float_times(-1) self.addCleanup(restore_float_times, old_float_times) @@ -566,7 +586,7 @@ class UtimeTests(unittest.TestCase): "fd support for utime required for this test.") def test_utime_fd(self): def set_time(filename, ns): - with open(filename, 'wb') as fp: + with open(filename, 'wb', 0) as fp: # use a file descriptor to test futimens(timespec) # or futimes(timeval) os.utime(fp.fileno(), ns=ns) @@ -678,18 +698,20 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol): return os.environ # Bug 1110478 - @unittest.skipUnless(os.path.exists('/bin/sh'), 'requires /bin/sh') + @unittest.skipUnless(unix_shell and os.path.exists(unix_shell), + 'requires a shell') def test_update2(self): os.environ.clear() os.environ.update(HELLO="World") - with os.popen("/bin/sh -c 'echo $HELLO'") as popen: + with os.popen("%s -c 'echo $HELLO'" % unix_shell) as popen: value = popen.read().strip() self.assertEqual(value, "World") - @unittest.skipUnless(os.path.exists('/bin/sh'), 'requires /bin/sh') + @unittest.skipUnless(unix_shell and os.path.exists(unix_shell), + 'requires a shell') def test_os_popen_iter(self): - with os.popen( - "/bin/sh -c 'echo \"line1\nline2\nline3\"'") as popen: + with os.popen("%s -c 'echo \"line1\nline2\nline3\"'" + % unix_shell) as popen: it = iter(popen) self.assertEqual(next(it), "line1\n") self.assertEqual(next(it), "line2\n") @@ -820,6 +842,7 @@ class WalkTests(unittest.TestCase): def setUp(self): join = os.path.join + self.addCleanup(support.rmtree, support.TESTFN) # Build: # TESTFN/ @@ -842,11 +865,11 @@ class WalkTests(unittest.TestCase): self.sub1_path = join(self.walk_path, "SUB1") self.sub11_path = join(self.sub1_path, "SUB11") sub2_path = join(self.walk_path, "SUB2") - self.sub21_path = join(sub2_path, "SUB21") + sub21_path = join(sub2_path, "SUB21") tmp1_path = join(self.walk_path, "tmp1") tmp2_path = join(self.sub1_path, "tmp2") tmp3_path = join(sub2_path, "tmp3") - tmp5_path = join(self.sub21_path, "tmp3") + tmp5_path = join(sub21_path, "tmp3") self.link_path = join(sub2_path, "link") t2_path = join(support.TESTFN, "TEST2") tmp4_path = join(support.TESTFN, "TEST2", "tmp4") @@ -857,7 +880,7 @@ class WalkTests(unittest.TestCase): # Create stuff. os.makedirs(self.sub11_path) os.makedirs(sub2_path) - os.makedirs(self.sub21_path) + os.makedirs(sub21_path) os.makedirs(t2_path) for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: @@ -875,16 +898,15 @@ class WalkTests(unittest.TestCase): else: self.sub2_tree = (sub2_path, [], ["tmp3"]) - os.chmod(self.sub21_path, 0) + os.chmod(sub21_path, 0) try: - os.listdir(self.sub21_path) + os.listdir(sub21_path) except PermissionError: - pass + self.addCleanup(os.chmod, sub21_path, stat.S_IRWXU) else: - os.chmod(self.sub21_path, stat.S_IRWXU) + os.chmod(sub21_path, stat.S_IRWXU) os.unlink(tmp5_path) - os.rmdir(self.sub21_path) - self.sub21_path = None + os.rmdir(sub21_path) del self.sub2_tree[1][:1] def test_walk_topdown(self): @@ -904,10 +926,12 @@ class WalkTests(unittest.TestCase): self.assertEqual(all[2 + flipped], (self.sub11_path, [], [])) self.assertEqual(all[3 - 2 * flipped], self.sub2_tree) - def test_walk_prune(self): + def test_walk_prune(self, walk_path=None): + if walk_path is None: + walk_path = self.walk_path # Prune the search. all = [] - for root, dirs, files in self.walk(self.walk_path): + for root, dirs, files in self.walk(walk_path): all.append((root, dirs, files)) # Don't descend into SUB1. if 'SUB1' in dirs: @@ -916,17 +940,20 @@ class WalkTests(unittest.TestCase): self.assertEqual(len(all), 2) self.assertEqual(all[0], - (self.walk_path, ["SUB2"], ["tmp1"])) + (str(walk_path), ["SUB2"], ["tmp1"])) all[1][-1].sort() all[1][1].sort() self.assertEqual(all[1], self.sub2_tree) + def test_file_like_path(self): + self.test_walk_prune(_PathLike(self.walk_path)) + def test_walk_bottom_up(self): # Walk bottom-up. all = list(self.walk(self.walk_path, topdown=False)) - self.assertEqual(len(all), 4) + self.assertEqual(len(all), 4, all) # We can't know which order SUB1 and SUB2 will appear in. # Not flipped: SUB11, SUB1, SUB2, TESTFN # flipped: SUB2, SUB11, SUB1, TESTFN @@ -957,24 +984,6 @@ class WalkTests(unittest.TestCase): else: self.fail("Didn't follow symlink with followlinks=True") - def tearDown(self): - # Tear everything down. This is a decent use for bottom-up on - # Windows, which doesn't have a recursive delete command. The - # (not so) subtlety is that rmdir will fail unless the dir's - # kids are removed first, so bottom up is essential. - if self.sub21_path: - os.chmod(self.sub21_path, stat.S_IRWXU) - for root, dirs, files in os.walk(support.TESTFN, topdown=False): - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - dirname = os.path.join(root, name) - if not os.path.islink(dirname): - os.rmdir(dirname) - else: - os.remove(dirname) - os.rmdir(support.TESTFN) - def test_walk_bad_dir(self): # Walk top-down. errors = [] @@ -1062,29 +1071,11 @@ class FwalkTests(WalkTests): self.addCleanup(os.close, newfd) self.assertEqual(newfd, minfd) - def tearDown(self): - # cleanup - if self.sub21_path: - os.chmod(self.sub21_path, stat.S_IRWXU) - for root, dirs, files, rootfd in os.fwalk(support.TESTFN, topdown=False): - for name in files: - os.unlink(name, dir_fd=rootfd) - for name in dirs: - st = os.stat(name, dir_fd=rootfd, follow_symlinks=False) - if stat.S_ISDIR(st.st_mode): - os.rmdir(name, dir_fd=rootfd) - else: - os.unlink(name, dir_fd=rootfd) - os.rmdir(support.TESTFN) - class BytesWalkTests(WalkTests): """Tests for os.walk() with bytes.""" def setUp(self): super().setUp() self.stack = contextlib.ExitStack() - if os.name == 'nt': - self.stack.enter_context(warnings.catch_warnings()) - warnings.simplefilter("ignore", DeprecationWarning) def tearDown(self): self.stack.close() @@ -1261,8 +1252,7 @@ class RemoveDirsTests(unittest.TestCase): os.mkdir(dira) dirb = os.path.join(dira, 'dirb') os.mkdir(dirb) - with open(os.path.join(dira, 'file.txt'), 'w') as f: - f.write('text') + create_file(os.path.join(dira, 'file.txt')) os.removedirs(dirb) self.assertFalse(os.path.exists(dirb)) self.assertTrue(os.path.exists(dira)) @@ -1273,8 +1263,7 @@ class RemoveDirsTests(unittest.TestCase): os.mkdir(dira) dirb = os.path.join(dira, 'dirb') os.mkdir(dirb) - with open(os.path.join(dirb, 'file.txt'), 'w') as f: - f.write('text') + create_file(os.path.join(dirb, 'file.txt')) with self.assertRaises(OSError): os.removedirs(dirb) self.assertTrue(os.path.exists(dirb)) @@ -1284,7 +1273,7 @@ class RemoveDirsTests(unittest.TestCase): class DevNullTests(unittest.TestCase): def test_devnull(self): - with open(os.devnull, 'wb') as f: + with open(os.devnull, 'wb', 0) as f: f.write(b'hello') f.close() with open(os.devnull, 'rb') as f: @@ -1301,6 +1290,7 @@ class URandomTests(unittest.TestCase): def test_urandom_value(self): data1 = os.urandom(16) + self.assertIsInstance(data1, bytes) data2 = os.urandom(16) self.assertNotEqual(data1, data2) @@ -1321,6 +1311,49 @@ class URandomTests(unittest.TestCase): self.assertNotEqual(data1, data2) +@unittest.skipUnless(hasattr(os, 'getrandom'), 'need os.getrandom()') +class GetRandomTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + try: + os.getrandom(1) + except OSError as exc: + if exc.errno == errno.ENOSYS: + # Python compiled on a more recent Linux version + # than the current Linux kernel + raise unittest.SkipTest("getrandom() syscall fails with ENOSYS") + else: + raise + + def test_getrandom_type(self): + data = os.getrandom(16) + self.assertIsInstance(data, bytes) + self.assertEqual(len(data), 16) + + def test_getrandom0(self): + empty = os.getrandom(0) + self.assertEqual(empty, b'') + + def test_getrandom_random(self): + self.assertTrue(hasattr(os, 'GRND_RANDOM')) + + # Don't test os.getrandom(1, os.GRND_RANDOM) to not consume the rare + # resource /dev/random + + def test_getrandom_nonblock(self): + # The call must not fail. Check also that the flag exists + try: + os.getrandom(1, os.GRND_NONBLOCK) + except BlockingIOError: + # System urandom is not initialized yet + pass + + def test_getrandom_value(self): + data1 = os.getrandom(16) + data2 = os.getrandom(16) + self.assertNotEqual(data1, data2) + + # os.urandom() doesn't use a file descriptor when it is implemented with the # getentropy() function, the getrandom() function or the getrandom() syscall OS_URANDOM_DONT_USE_FD = ( @@ -1371,9 +1404,9 @@ class URandomFDTests(unittest.TestCase): def test_urandom_fd_reopened(self): # Issue #21207: urandom() should detect its fd to /dev/urandom # changed to something else, and reopen it. - with open(support.TESTFN, 'wb') as f: - f.write(b"x" * 256) - self.addCleanup(os.unlink, support.TESTFN) + self.addCleanup(support.unlink, support.TESTFN) + create_file(support.TESTFN, b"x" * 256) + code = """if 1: import os import sys @@ -1390,7 +1423,12 @@ class URandomFDTests(unittest.TestCase): break os.closerange(3, 256) with open({TESTFN!r}, 'rb') as f: - os.dup2(f.fileno(), fd) + new_fd = f.fileno() + # Issue #26935: posix allows new_fd and fd to be equal but + # some libc implementations have dup2 return an error in this + # case. + if new_fd != fd: + os.dup2(new_fd, fd) sys.stdout.buffer.write(os.urandom(4)) sys.stdout.buffer.write(os.urandom(4)) """.format(TESTFN=support.TESTFN) @@ -1435,6 +1473,7 @@ def _execvpe_mockup(defpath=None): os.execve = orig_execve os.defpath = orig_defpath + class ExecTests(unittest.TestCase): @unittest.skipIf(USING_LINUXTHREADS, "avoid triggering a linuxthreads bug: see issue #4970") @@ -1442,8 +1481,16 @@ class ExecTests(unittest.TestCase): self.assertRaises(OSError, os.execvpe, 'no such app-', ['no such app-'], None) + def test_execv_with_bad_arglist(self): + self.assertRaises(ValueError, os.execv, 'notepad', ()) + self.assertRaises(ValueError, os.execv, 'notepad', []) + self.assertRaises(ValueError, os.execv, 'notepad', ('',)) + self.assertRaises(ValueError, os.execv, 'notepad', ['']) + def test_execvpe_with_bad_arglist(self): self.assertRaises(ValueError, os.execvpe, 'notepad', [], None) + self.assertRaises(ValueError, os.execvpe, 'notepad', [], {}) + self.assertRaises(ValueError, os.execvpe, 'notepad', [''], {}) @unittest.skipUnless(hasattr(os, '_execvpe'), "No internal os._execvpe function to test.") @@ -1502,6 +1549,18 @@ class ExecTests(unittest.TestCase): @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") class Win32ErrorTests(unittest.TestCase): + def setUp(self): + try: + os.stat(support.TESTFN) + except FileNotFoundError: + exists = False + except OSError as exc: + exists = True + self.fail("file %s must not exist; os.stat failed with %s" + % (support.TESTFN, exc)) + else: + self.fail("file %s must not exist" % support.TESTFN) + def test_rename(self): self.assertRaises(OSError, os.rename, support.TESTFN, support.TESTFN+".bak") @@ -1512,12 +1571,10 @@ class Win32ErrorTests(unittest.TestCase): self.assertRaises(OSError, os.chdir, support.TESTFN) def test_mkdir(self): - f = open(support.TESTFN, "w") - try: + self.addCleanup(support.unlink, support.TESTFN) + + with open(support.TESTFN, "x") as f: self.assertRaises(OSError, os.mkdir, support.TESTFN) - finally: - f.close() - os.unlink(support.TESTFN) def test_utime(self): self.assertRaises(OSError, os.utime, support.TESTFN, None) @@ -1525,6 +1582,7 @@ class Win32ErrorTests(unittest.TestCase): def test_chmod(self): self.assertRaises(OSError, os.chmod, support.TESTFN, 0) + class TestInvalidFD(unittest.TestCase): singles = ["fchdir", "dup", "fdopen", "fdatasync", "fstat", "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] @@ -1636,12 +1694,9 @@ class LinkTests(unittest.TestCase): os.unlink(file) def _test_link(self, file1, file2): - with open(file1, "w") as f1: - f1.write("test") + create_file(file1) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - os.link(file1, file2) + os.link(file1, file2) with open(file1, "r") as f1, open(file2, "r") as f2: self.assertTrue(os.path.sameopenfile(f1.fileno(), f2.fileno())) @@ -1932,6 +1987,7 @@ class Win32ListdirTests(unittest.TestCase): self.assertEqual( sorted(os.listdir(support.TESTFN)), self.created_paths) + # bytes self.assertEqual( sorted(os.listdir(os.fsencode(support.TESTFN))), @@ -1945,6 +2001,7 @@ class Win32ListdirTests(unittest.TestCase): self.assertEqual( sorted(os.listdir(path)), self.created_paths) + # bytes path = b'\\\\?\\' + os.fsencode(os.path.abspath(support.TESTFN)) self.assertEqual( @@ -2024,51 +2081,43 @@ class Win32SymlinkTests(unittest.TestCase): self.assertNotEqual(os.lstat(link), os.stat(link)) bytes_link = os.fsencode(link) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - self.assertEqual(os.stat(bytes_link), os.stat(target)) - self.assertNotEqual(os.lstat(bytes_link), os.stat(bytes_link)) + self.assertEqual(os.stat(bytes_link), os.stat(target)) + self.assertNotEqual(os.lstat(bytes_link), os.stat(bytes_link)) def test_12084(self): level1 = os.path.abspath(support.TESTFN) level2 = os.path.join(level1, "level2") level3 = os.path.join(level2, "level3") - try: - os.mkdir(level1) - os.mkdir(level2) - os.mkdir(level3) + self.addCleanup(support.rmtree, level1) - file1 = os.path.abspath(os.path.join(level1, "file1")) + os.mkdir(level1) + os.mkdir(level2) + os.mkdir(level3) - with open(file1, "w") as f: - f.write("file1") + file1 = os.path.abspath(os.path.join(level1, "file1")) + create_file(file1) - orig_dir = os.getcwd() - try: - os.chdir(level2) - link = os.path.join(level2, "link") - os.symlink(os.path.relpath(file1), "link") - self.assertIn("link", os.listdir(os.getcwd())) - - # Check os.stat calls from the same dir as the link - self.assertEqual(os.stat(file1), os.stat("link")) - - # Check os.stat calls from a dir below the link - os.chdir(level1) - self.assertEqual(os.stat(file1), - os.stat(os.path.relpath(link))) - - # Check os.stat calls from a dir above the link - os.chdir(level3) - self.assertEqual(os.stat(file1), - os.stat(os.path.relpath(link))) - finally: - os.chdir(orig_dir) - except OSError as err: - self.fail(err) + orig_dir = os.getcwd() + try: + os.chdir(level2) + link = os.path.join(level2, "link") + os.symlink(os.path.relpath(file1), "link") + self.assertIn("link", os.listdir(os.getcwd())) + + # Check os.stat calls from the same dir as the link + self.assertEqual(os.stat(file1), os.stat("link")) + + # Check os.stat calls from a dir below the link + os.chdir(level1) + self.assertEqual(os.stat(file1), + os.stat(os.path.relpath(link))) + + # Check os.stat calls from a dir above the link + os.chdir(level3) + self.assertEqual(os.stat(file1), + os.stat(os.path.relpath(link))) finally: - os.remove(file1) - shutil.rmtree(level1) + os.chdir(orig_dir) @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") @@ -2106,7 +2155,7 @@ class Win32JunctionTests(unittest.TestCase): class NonLocalSymlinkTests(unittest.TestCase): def setUp(self): - """ + r""" Create this structure: base @@ -2177,11 +2226,137 @@ class PidTests(unittest.TestCase): def test_waitpid(self): args = [sys.executable, '-c', 'pass'] - pid = os.spawnv(os.P_NOWAIT, args[0], args) + # Add an implicit test for PyUnicode_FSConverter(). + pid = os.spawnv(os.P_NOWAIT, _PathLike(args[0]), args) status = os.waitpid(pid, 0) self.assertEqual(status, (pid, 0)) +class SpawnTests(unittest.TestCase): + def create_args(self, *, with_env=False, use_bytes=False): + self.exitcode = 17 + + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + if not with_env: + code = 'import sys; sys.exit(%s)' % self.exitcode + else: + self.env = dict(os.environ) + # create an unique key + self.key = str(uuid.uuid4()) + self.env[self.key] = self.key + # read the variable from os.environ to check that it exists + code = ('import sys, os; magic = os.environ[%r]; sys.exit(%s)' + % (self.key, self.exitcode)) + + with open(filename, "w") as fp: + fp.write(code) + + args = [sys.executable, filename] + if use_bytes: + args = [os.fsencode(a) for a in args] + self.env = {os.fsencode(k): os.fsencode(v) + for k, v in self.env.items()} + + return args + + @requires_os_func('spawnl') + def test_spawnl(self): + args = self.create_args() + exitcode = os.spawnl(os.P_WAIT, args[0], *args) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnle') + def test_spawnle(self): + args = self.create_args(with_env=True) + exitcode = os.spawnle(os.P_WAIT, args[0], *args, self.env) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnlp') + def test_spawnlp(self): + args = self.create_args() + exitcode = os.spawnlp(os.P_WAIT, args[0], *args) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnlpe') + def test_spawnlpe(self): + args = self.create_args(with_env=True) + exitcode = os.spawnlpe(os.P_WAIT, args[0], *args, self.env) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnv') + def test_spawnv(self): + args = self.create_args() + exitcode = os.spawnv(os.P_WAIT, args[0], args) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnve') + def test_spawnve(self): + args = self.create_args(with_env=True) + exitcode = os.spawnve(os.P_WAIT, args[0], args, self.env) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnvp') + def test_spawnvp(self): + args = self.create_args() + exitcode = os.spawnvp(os.P_WAIT, args[0], args) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnvpe') + def test_spawnvpe(self): + args = self.create_args(with_env=True) + exitcode = os.spawnvpe(os.P_WAIT, args[0], args, self.env) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnv') + def test_nowait(self): + args = self.create_args() + pid = os.spawnv(os.P_NOWAIT, args[0], args) + result = os.waitpid(pid, 0) + self.assertEqual(result[0], pid) + status = result[1] + if hasattr(os, 'WIFEXITED'): + self.assertTrue(os.WIFEXITED(status)) + self.assertEqual(os.WEXITSTATUS(status), self.exitcode) + else: + self.assertEqual(status, self.exitcode << 8) + + @requires_os_func('spawnve') + def test_spawnve_bytes(self): + # Test bytes handling in parse_arglist and parse_envlist (#28114) + args = self.create_args(with_env=True, use_bytes=True) + exitcode = os.spawnve(os.P_WAIT, args[0], args, self.env) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnl') + def test_spawnl_noargs(self): + args = self.create_args() + self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, args[0]) + self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, args[0], '') + + @requires_os_func('spawnle') + def test_spawnle_noargs(self): + args = self.create_args() + self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, args[0], {}) + self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, args[0], '', {}) + + @requires_os_func('spawnv') + def test_spawnv_noargs(self): + args = self.create_args() + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], ()) + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], []) + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], ('',)) + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], ['']) + + @requires_os_func('spawnve') + def test_spawnve_noargs(self): + args = self.create_args() + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], (), {}) + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], [], {}) + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], ('',), {}) + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], [''], {}) + # The introduction of this TestCase caused at least two different errors on # *nix buildbots. Temporarily skip this to let the buildbots move along. @unittest.skip("Skip due to platform/environment differences on *NIX buildbots") @@ -2204,8 +2379,8 @@ class ProgramPriorityTests(unittest.TestCase): try: new_prio = os.getpriority(os.PRIO_PROCESS, os.getpid()) if base >= 19 and new_prio <= 19: - raise unittest.SkipTest( - "unable to reliably test setpriority at current nice level of %s" % base) + raise unittest.SkipTest("unable to reliably test setpriority " + "at current nice level of %s" % base) else: self.assertEqual(new_prio, base + 1) finally: @@ -2315,8 +2490,7 @@ class TestSendfile(unittest.TestCase): @classmethod def setUpClass(cls): cls.key = support.threading_setup() - with open(support.TESTFN, "wb") as f: - f.write(cls.DATA) + create_file(support.TESTFN, cls.DATA) @classmethod def tearDownClass(cls): @@ -2466,10 +2640,11 @@ class TestSendfile(unittest.TestCase): def test_trailers(self): TESTFN2 = support.TESTFN + "2" file_data = b"abcdef" - with open(TESTFN2, 'wb') as f: - f.write(file_data) - with open(TESTFN2, 'rb')as f: - self.addCleanup(os.remove, TESTFN2) + + self.addCleanup(support.unlink, TESTFN2) + create_file(TESTFN2, file_data) + + with open(TESTFN2, 'rb') as f: os.sendfile(self.sockno, f.fileno(), 0, len(file_data), trailers=[b"1234"]) self.client.close() @@ -2492,35 +2667,37 @@ class TestSendfile(unittest.TestCase): def supports_extended_attributes(): if not hasattr(os, "setxattr"): return False + try: - with open(support.TESTFN, "wb") as fp: + with open(support.TESTFN, "xb", 0) as fp: try: os.setxattr(fp.fileno(), b"user.test", b"") except OSError: return False finally: support.unlink(support.TESTFN) - # Kernels < 2.6.39 don't respect setxattr flags. - kernel_version = platform.release() - m = re.match("2.6.(\d{1,2})", kernel_version) - return m is None or int(m.group(1)) >= 39 + + return True @unittest.skipUnless(supports_extended_attributes(), "no non-broken extended attribute support") +# Kernels < 2.6.39 don't respect setxattr flags. +@support.requires_linux_version(2, 6, 39) class ExtendedAttributeTests(unittest.TestCase): - def tearDown(self): - support.unlink(support.TESTFN) - def _check_xattrs_str(self, s, getxattr, setxattr, removexattr, listxattr, **kwargs): fn = support.TESTFN - open(fn, "wb").close() + self.addCleanup(support.unlink, fn) + create_file(fn) + with self.assertRaises(OSError) as cm: getxattr(fn, s("user.test"), **kwargs) self.assertEqual(cm.exception.errno, errno.ENODATA) + init_xattr = listxattr(fn) self.assertIsInstance(init_xattr, list) + setxattr(fn, s("user.test"), b"", **kwargs) xattr = set(init_xattr) xattr.add("user.test") @@ -2528,19 +2705,24 @@ class ExtendedAttributeTests(unittest.TestCase): self.assertEqual(getxattr(fn, b"user.test", **kwargs), b"") setxattr(fn, s("user.test"), b"hello", os.XATTR_REPLACE, **kwargs) self.assertEqual(getxattr(fn, b"user.test", **kwargs), b"hello") + with self.assertRaises(OSError) as cm: setxattr(fn, s("user.test"), b"bye", os.XATTR_CREATE, **kwargs) self.assertEqual(cm.exception.errno, errno.EEXIST) + with self.assertRaises(OSError) as cm: setxattr(fn, s("user.test2"), b"bye", os.XATTR_REPLACE, **kwargs) self.assertEqual(cm.exception.errno, errno.ENODATA) + setxattr(fn, s("user.test2"), b"foo", os.XATTR_CREATE, **kwargs) xattr.add("user.test2") self.assertEqual(set(listxattr(fn)), xattr) removexattr(fn, s("user.test"), **kwargs) + with self.assertRaises(OSError) as cm: getxattr(fn, s("user.test"), **kwargs) self.assertEqual(cm.exception.errno, errno.ENODATA) + xattr.remove("user.test") self.assertEqual(set(listxattr(fn)), xattr) self.assertEqual(getxattr(fn, s("user.test2"), **kwargs), b"foo") @@ -2553,11 +2735,11 @@ class ExtendedAttributeTests(unittest.TestCase): self.assertEqual(set(listxattr(fn)), set(init_xattr) | set(many)) def _check_xattrs(self, *args, **kwargs): - def make_bytes(s): - return bytes(s, "ascii") self._check_xattrs_str(str, *args, **kwargs) support.unlink(support.TESTFN) - self._check_xattrs_str(make_bytes, *args, **kwargs) + + self._check_xattrs_str(os.fsencode, *args, **kwargs) + support.unlink(support.TESTFN) def test_simple(self): self._check_xattrs(os.getxattr, os.setxattr, os.removexattr, @@ -2572,10 +2754,10 @@ class ExtendedAttributeTests(unittest.TestCase): with open(path, "rb") as fp: return os.getxattr(fp.fileno(), *args) def setxattr(path, *args): - with open(path, "wb") as fp: + with open(path, "wb", 0) as fp: os.setxattr(fp.fileno(), *args) def removexattr(path, *args): - with open(path, "wb") as fp: + with open(path, "wb", 0) as fp: os.removexattr(fp.fileno(), *args) def listxattr(path, *args): with open(path, "rb") as fp: @@ -2583,43 +2765,6 @@ class ExtendedAttributeTests(unittest.TestCase): self._check_xattrs(getxattr, setxattr, removexattr, listxattr) -@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") -class Win32DeprecatedBytesAPI(unittest.TestCase): - def test_deprecated(self): - import nt - filename = os.fsencode(support.TESTFN) - with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - for func, *args in ( - (nt._getfullpathname, filename), - (nt._isdir, filename), - (os.access, filename, os.R_OK), - (os.chdir, filename), - (os.chmod, filename, 0o777), - (os.getcwdb,), - (os.link, filename, filename), - (os.listdir, filename), - (os.lstat, filename), - (os.mkdir, filename), - (os.open, filename, os.O_RDONLY), - (os.rename, filename, filename), - (os.rmdir, filename), - (os.startfile, filename), - (os.stat, filename), - (os.unlink, filename), - (os.utime, filename), - ): - self.assertRaises(DeprecationWarning, func, *args) - - @support.skip_unless_symlink - def test_symlink(self): - filename = os.fsencode(support.TESTFN) - with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - self.assertRaises(DeprecationWarning, - os.symlink, filename, filename) - - @unittest.skipUnless(hasattr(os, 'get_terminal_size'), "requires os.get_terminal_size") class TermsizeTests(unittest.TestCase): def test_does_not_crash(self): @@ -2682,6 +2827,7 @@ class OSErrorTests(unittest.TestCase): else: encoded = os.fsencode(support.TESTFN) self.bytes_filenames.append(encoded) + self.bytes_filenames.append(bytearray(encoded)) self.bytes_filenames.append(memoryview(encoded)) self.filenames = self.bytes_filenames + self.unicode_filenames @@ -2702,16 +2848,7 @@ class OSErrorTests(unittest.TestCase): (self.bytes_filenames, os.replace, b"dst"), (self.unicode_filenames, os.rename, "dst"), (self.unicode_filenames, os.replace, "dst"), - # Issue #16414: Don't test undecodable names with listdir() - # because of a Windows bug. - # - # With the ANSI code page 932, os.listdir(b'\xe7') return an - # empty list (instead of failing), whereas os.listdir(b'\xff') - # raises a FileNotFoundError. It looks like a Windows bug: - # b'\xe7' directory does not exist, FindFirstFileA(b'\xe7') - # fails with ERROR_FILE_NOT_FOUND (2), instead of - # ERROR_PATH_NOT_FOUND (3). - (self.unicode_filenames, os.listdir,), + (self.unicode_filenames, os.listdir, ), )) else: funcs.extend(( @@ -2752,12 +2889,19 @@ class OSErrorTests(unittest.TestCase): else: funcs.append((self.filenames, os.readlink,)) + for filenames, func, *func_args in funcs: for name in filenames: try: - func(name, *func_args) + if isinstance(name, (str, bytes)): + func(name, *func_args) + else: + with self.assertWarnsRegex(DeprecationWarning, 'should be'): + func(name, *func_args) except OSError as err: - self.assertIs(err.filename, name) + self.assertIs(err.filename, name, str(func)) + except UnicodeDecodeError: + pass else: self.fail("No exception thrown by {}".format(func)) @@ -2855,6 +2999,63 @@ class FDInheritanceTests(unittest.TestCase): self.assertEqual(os.get_inheritable(slave_fd), False) +class PathTConverterTests(unittest.TestCase): + # tuples of (function name, allows fd arguments, additional arguments to + # function, cleanup function) + functions = [ + ('stat', True, (), None), + ('lstat', False, (), None), + ('access', False, (os.F_OK,), None), + ('chflags', False, (0,), None), + ('lchflags', False, (0,), None), + ('open', False, (0,), getattr(os, 'close', None)), + ] + + def test_path_t_converter(self): + str_filename = support.TESTFN + if os.name == 'nt': + bytes_fspath = bytes_filename = None + else: + bytes_filename = support.TESTFN.encode('ascii') + bytes_fspath = _PathLike(bytes_filename) + fd = os.open(_PathLike(str_filename), os.O_WRONLY|os.O_CREAT) + self.addCleanup(support.unlink, support.TESTFN) + self.addCleanup(os.close, fd) + + int_fspath = _PathLike(fd) + str_fspath = _PathLike(str_filename) + + for name, allow_fd, extra_args, cleanup_fn in self.functions: + with self.subTest(name=name): + try: + fn = getattr(os, name) + except AttributeError: + continue + + for path in (str_filename, bytes_filename, str_fspath, + bytes_fspath): + if path is None: + continue + with self.subTest(name=name, path=path): + result = fn(path, *extra_args) + if cleanup_fn is not None: + cleanup_fn(result) + + with self.assertRaisesRegex( + TypeError, 'should be string, bytes'): + fn(int_fspath, *extra_args) + + if allow_fd: + result = fn(fd, *extra_args) # should not fail + if cleanup_fn is not None: + cleanup_fn(result) + else: + with self.assertRaisesRegex( + TypeError, + 'os.PathLike'): + fn(fd, *extra_args) + + @unittest.skipUnless(hasattr(os, 'get_blocking'), 'needs os.get_blocking() and os.set_blocking()') class BlockingTests(unittest.TestCase): @@ -2878,15 +3079,18 @@ class ExportsTests(unittest.TestCase): class TestScandir(unittest.TestCase): + check_no_resource_warning = support.check_no_resource_warning + def setUp(self): self.path = os.path.realpath(support.TESTFN) + self.bytes_path = os.fsencode(self.path) self.addCleanup(support.rmtree, self.path) os.mkdir(self.path) def create_file(self, name="file.txt"): - filename = os.path.join(self.path, name) - with open(filename, "wb") as fp: - fp.write(b'python') + path = self.bytes_path if isinstance(name, bytes) else self.path + filename = os.path.join(path, name) + create_file(filename, b'python') return filename def get_entries(self, names): @@ -2909,6 +3113,7 @@ class TestScandir(unittest.TestCase): self.assertEqual(stat1, stat2) def check_entry(self, entry, name, is_dir, is_file, is_symlink): + self.assertIsInstance(entry, os.DirEntry) self.assertEqual(entry.name, name) self.assertEqual(entry.path, os.path.join(self.path, name)) self.assertEqual(entry.inode(), @@ -2974,15 +3179,16 @@ class TestScandir(unittest.TestCase): self.check_entry(entry, 'symlink_file.txt', False, True, True) def get_entry(self, name): - entries = list(os.scandir(self.path)) + path = self.bytes_path if isinstance(name, bytes) else self.path + entries = list(os.scandir(path)) self.assertEqual(len(entries), 1) entry = entries[0] self.assertEqual(entry.name, name) return entry - def create_file_entry(self): - filename = self.create_file() + def create_file_entry(self, name='file.txt'): + filename = self.create_file(name=name) return self.get_entry(os.path.basename(filename)) def test_current_directory(self): @@ -3003,6 +3209,18 @@ class TestScandir(unittest.TestCase): entry = self.create_file_entry() self.assertEqual(repr(entry), "<DirEntry 'file.txt'>") + def test_fspath_protocol(self): + entry = self.create_file_entry() + self.assertEqual(os.fspath(entry), os.path.join(self.path, 'file.txt')) + + def test_fspath_protocol_bytes(self): + bytes_filename = os.fsencode('bytesfile.txt') + bytes_entry = self.create_file_entry(name=bytes_filename) + fspath = os.fspath(bytes_entry) + self.assertIsInstance(fspath, bytes) + self.assertEqual(fspath, + os.path.join(os.fsencode(self.path),bytes_filename)) + def test_removed_dir(self): path = os.path.join(self.path, 'dir') @@ -3066,11 +3284,6 @@ class TestScandir(unittest.TestCase): entry.stat(follow_symlinks=False) def test_bytes(self): - if os.name == "nt": - # On Windows, os.scandir(bytes) must raise an exception - self.assertRaises(TypeError, os.scandir, b'.') - return - self.create_file("file.txt") path_bytes = os.fsencode(self.path) @@ -3100,6 +3313,112 @@ class TestScandir(unittest.TestCase): for obj in [1234, 1.234, {}, []]: self.assertRaises(TypeError, os.scandir, obj) + def test_close(self): + self.create_file("file.txt") + self.create_file("file2.txt") + iterator = os.scandir(self.path) + next(iterator) + iterator.close() + # multiple closes + iterator.close() + with self.check_no_resource_warning(): + del iterator + + def test_context_manager(self): + self.create_file("file.txt") + self.create_file("file2.txt") + with os.scandir(self.path) as iterator: + next(iterator) + with self.check_no_resource_warning(): + del iterator + + def test_context_manager_close(self): + self.create_file("file.txt") + self.create_file("file2.txt") + with os.scandir(self.path) as iterator: + next(iterator) + iterator.close() + + def test_context_manager_exception(self): + self.create_file("file.txt") + self.create_file("file2.txt") + with self.assertRaises(ZeroDivisionError): + with os.scandir(self.path) as iterator: + next(iterator) + 1/0 + with self.check_no_resource_warning(): + del iterator + + def test_resource_warning(self): + self.create_file("file.txt") + self.create_file("file2.txt") + iterator = os.scandir(self.path) + next(iterator) + with self.assertWarns(ResourceWarning): + del iterator + support.gc_collect() + # exhausted iterator + iterator = os.scandir(self.path) + list(iterator) + with self.check_no_resource_warning(): + del iterator + + +class TestPEP519(unittest.TestCase): + + # Abstracted so it can be overridden to test pure Python implementation + # if a C version is provided. + fspath = staticmethod(os.fspath) + + def test_return_bytes(self): + for b in b'hello', b'goodbye', b'some/path/and/file': + self.assertEqual(b, self.fspath(b)) + + def test_return_string(self): + for s in 'hello', 'goodbye', 'some/path/and/file': + self.assertEqual(s, self.fspath(s)) + + def test_fsencode_fsdecode(self): + for p in "path/like/object", b"path/like/object": + pathlike = _PathLike(p) + + self.assertEqual(p, self.fspath(pathlike)) + self.assertEqual(b"path/like/object", os.fsencode(pathlike)) + self.assertEqual("path/like/object", os.fsdecode(pathlike)) + + def test_pathlike(self): + self.assertEqual('#feelthegil', self.fspath(_PathLike('#feelthegil'))) + self.assertTrue(issubclass(_PathLike, os.PathLike)) + self.assertTrue(isinstance(_PathLike(), os.PathLike)) + + def test_garbage_in_exception_out(self): + vapor = type('blah', (), {}) + for o in int, type, os, vapor(): + self.assertRaises(TypeError, self.fspath, o) + + def test_argument_required(self): + self.assertRaises(TypeError, self.fspath) + + def test_bad_pathlike(self): + # __fspath__ returns a value other than str or bytes. + self.assertRaises(TypeError, self.fspath, _PathLike(42)) + # __fspath__ attribute that is not callable. + c = type('foo', (), {}) + c.__fspath__ = 1 + self.assertRaises(TypeError, self.fspath, c()) + # __fspath__ raises an exception. + self.assertRaises(ZeroDivisionError, self.fspath, + _PathLike(ZeroDivisionError())) + +# Only test if the C version is provided, otherwise TestPEP519 already tested +# the pure Python implementation. +if hasattr(os, "_fspath"): + class TestPEP519PurePython(TestPEP519): + + """Explicitly test the pure Python implementation of os.fspath().""" + + fspath = staticmethod(os._fspath) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index ab6577f44d..d6e6f71577 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -1,6 +1,5 @@ import parser import unittest -import sys import operator import struct from test import support @@ -139,6 +138,45 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase): self.check_suite("a = b") self.check_suite("a = b = c = d = e") + def test_var_annot(self): + self.check_suite("x: int = 5") + self.check_suite("y: List[T] = []; z: [list] = fun()") + self.check_suite("x: tuple = (1, 2)") + self.check_suite("d[f()]: int = 42") + self.check_suite("f(d[x]): str = 'abc'") + self.check_suite("x.y.z.w: complex = 42j") + self.check_suite("x: int") + self.check_suite("def f():\n" + " x: str\n" + " y: int = 5\n") + self.check_suite("class C:\n" + " x: str\n" + " y: int = 5\n") + self.check_suite("class C:\n" + " def __init__(self, x: int) -> None:\n" + " self.x: int = x\n") + # double check for nonsense + with self.assertRaises(SyntaxError): + exec("2+2: int", {}, {}) + with self.assertRaises(SyntaxError): + exec("[]: int = 5", {}, {}) + with self.assertRaises(SyntaxError): + exec("x, *y, z: int = range(5)", {}, {}) + with self.assertRaises(SyntaxError): + exec("t: tuple = 1, 2", {}, {}) + with self.assertRaises(SyntaxError): + exec("u = v: int", {}, {}) + with self.assertRaises(SyntaxError): + exec("False: int", {}, {}) + with self.assertRaises(SyntaxError): + exec("x.False: int", {}, {}) + with self.assertRaises(SyntaxError): + exec("x.y,: int", {}, {}) + with self.assertRaises(SyntaxError): + exec("[0]: int", {}, {}) + with self.assertRaises(SyntaxError): + exec("f(): int", {}, {}) + def test_simple_augmented_assignments(self): self.check_suite("a += b") self.check_suite("a -= b") @@ -633,12 +671,18 @@ class CompileTestCase(unittest.TestCase): self.assertEqual(code.co_filename, '<syntax-tree>') code = st.compile() self.assertEqual(code.co_filename, '<syntax-tree>') - for filename in ('file.py', b'file.py', - bytearray(b'file.py'), memoryview(b'file.py')): + for filename in 'file.py', b'file.py': code = parser.compilest(st, filename) self.assertEqual(code.co_filename, 'file.py') code = st.compile(filename) self.assertEqual(code.co_filename, 'file.py') + for filename in bytearray(b'file.py'), memoryview(b'file.py'): + with self.assertWarns(DeprecationWarning): + code = parser.compilest(st, filename) + self.assertEqual(code.co_filename, 'file.py') + with self.assertWarns(DeprecationWarning): + code = st.compile(filename) + self.assertEqual(code.co_filename, 'file.py') self.assertRaises(TypeError, parser.compilest, st, list(b'file.py')) self.assertRaises(TypeError, st.compile, list(b'file.py')) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 28e3ea4da6..88a93e5802 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -10,6 +10,7 @@ import tempfile import unittest from test import support +android_not_root = support.android_not_root TESTFN = support.TESTFN try: @@ -190,13 +191,18 @@ class _BasePurePathTest(object): P = self.cls p = P('a') self.assertIsInstance(p, P) + class PathLike: + def __fspath__(self): + return "a/b/c" P('a', 'b', 'c') P('/a', 'b', 'c') P('a/b/c') P('/a/b/c') + P(PathLike()) self.assertEqual(P(P('a')), P('a')) self.assertEqual(P(P('a'), 'b'), P('a/b')) self.assertEqual(P(P('a'), P('b')), P('a/b')) + self.assertEqual(P(P('a'), P('b'), P('c')), P(PathLike())) def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object @@ -384,6 +390,12 @@ class _BasePurePathTest(object): parts = p.parts self.assertEqual(parts, (sep, 'a', 'b')) + def test_fspath_common(self): + P = self.cls + p = P('a/b') + self._check_str(p.__fspath__(), ('a/b',)) + self._check_str(os.fspath(p), ('a/b',)) + def test_equivalences(self): for k, tuples in self.equivalences.items(): canon = k.replace('/', self.sep) @@ -1474,21 +1486,30 @@ class _BasePathTest(object): self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") }) self.assertEqual(set(p.glob("../xyzzy")), set()) - def _check_resolve_relative(self, p, expected): - q = p.resolve() - self.assertEqual(q, expected) - def _check_resolve_absolute(self, p, expected): - q = p.resolve() + def _check_resolve(self, p, expected, strict=True): + q = p.resolve(strict) self.assertEqual(q, expected) + # this can be used to check both relative and absolute resolutions + _check_resolve_relative = _check_resolve_absolute = _check_resolve + @with_symlinks def test_resolve_common(self): P = self.cls p = P(BASE, 'foo') with self.assertRaises(OSError) as cm: - p.resolve() + p.resolve(strict=True) self.assertEqual(cm.exception.errno, errno.ENOENT) + # Non-strict + self.assertEqual(str(p.resolve(strict=False)), + os.path.join(BASE, 'foo')) + p = P(BASE, 'foo', 'in', 'spam') + self.assertEqual(str(p.resolve(strict=False)), + os.path.join(BASE, 'foo')) + p = P(BASE, '..', 'foo', 'in', 'spam') + self.assertEqual(str(p.resolve(strict=False)), + os.path.abspath(os.path.join('foo'))) # These are all relative symlinks p = P(BASE, 'dirB', 'fileB') self._check_resolve_relative(p, p) @@ -1498,6 +1519,18 @@ class _BasePathTest(object): self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) p = P(BASE, 'dirB', 'linkD', 'fileB') self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) + # Non-strict + p = P(BASE, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo'), False) + p = P(BASE, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') + if os.name == 'nt': + # In Windows, if linkY points to dirB, 'dirA\linkY\..' + # resolves to 'dirA' without resolving linkY first. + self._check_resolve_relative(p, P(BASE, 'dirA', 'foo'), False) + else: + # In Posix, if linkY points to dirB, 'dirA/linkY/..' + # resolves to 'dirB/..' first before resolving to parent of dirB. + self._check_resolve_relative(p, P(BASE, 'foo'), False) # Now create absolute symlinks d = tempfile.mkdtemp(suffix='-dirD') self.addCleanup(support.rmtree, d) @@ -1505,6 +1538,18 @@ class _BasePathTest(object): os.symlink(join('dirB'), os.path.join(d, 'linkY')) p = P(BASE, 'dirA', 'linkX', 'linkY', 'fileB') self._check_resolve_absolute(p, P(BASE, 'dirB', 'fileB')) + # Non-strict + p = P(BASE, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(BASE, 'dirB', 'foo'), False) + p = P(BASE, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') + if os.name == 'nt': + # In Windows, if linkY points to dirB, 'dirA\linkY\..' + # resolves to 'dirA' without resolving linkY first. + self._check_resolve_relative(p, P(d, 'foo'), False) + else: + # In Posix, if linkY points to dirB, 'dirA/linkY/..' + # resolves to 'dirB/..' first before resolving to parent of dirB. + self._check_resolve_relative(p, P(BASE, 'foo'), False) @with_symlinks def test_resolve_dot(self): @@ -1514,7 +1559,11 @@ class _BasePathTest(object): self.dirlink(os.path.join('0', '0'), join('1')) self.dirlink(os.path.join('1', '1'), join('2')) q = p / '2' - self.assertEqual(q.resolve(), p) + self.assertEqual(q.resolve(strict=True), p) + r = q / '3' / '4' + self.assertRaises(FileNotFoundError, r.resolve, strict=True) + # Non-strict + self.assertEqual(r.resolve(strict=False), p / '3') def test_with(self): p = self.cls(BASE) @@ -1827,6 +1876,7 @@ class _BasePathTest(object): self.assertFalse((P / 'fileA' / 'bah').is_fifo()) @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required") + @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") def test_is_fifo_true(self): P = self.cls(BASE, 'myfifo') os.mkfifo(str(P)) @@ -1849,7 +1899,8 @@ class _BasePathTest(object): try: sock.bind(str(P)) except OSError as e: - if "AF_UNIX path too long" in str(e): + if (isinstance(e, PermissionError) or + "AF_UNIX path too long" in str(e)): self.skipTest("cannot bind Unix socket: " + str(e)) self.assertTrue(P.is_socket()) self.assertFalse(P.is_fifo()) @@ -1972,10 +2023,10 @@ class PathTest(_BasePathTest, unittest.TestCase): class PosixPathTest(_BasePathTest, unittest.TestCase): cls = pathlib.PosixPath - def _check_symlink_loop(self, *args): + def _check_symlink_loop(self, *args, strict=True): path = self.cls(*args) with self.assertRaises(RuntimeError): - print(path.resolve()) + print(path.resolve(strict)) def test_open_mode(self): old_mask = os.umask(0) @@ -2008,7 +2059,6 @@ class PosixPathTest(_BasePathTest, unittest.TestCase): @with_symlinks def test_resolve_loop(self): - # Loop detection for broken symlinks under POSIX # Loops with relative symlinks os.symlink('linkX/inside', join('linkX')) self._check_symlink_loop(BASE, 'linkX') @@ -2016,6 +2066,8 @@ class PosixPathTest(_BasePathTest, unittest.TestCase): self._check_symlink_loop(BASE, 'linkY') os.symlink('linkZ/../linkZ', join('linkZ')) self._check_symlink_loop(BASE, 'linkZ') + # Non-strict + self._check_symlink_loop(BASE, 'linkZ', 'foo', strict=False) # Loops with absolute symlinks os.symlink(join('linkU/inside'), join('linkU')) self._check_symlink_loop(BASE, 'linkU') @@ -2023,6 +2075,8 @@ class PosixPathTest(_BasePathTest, unittest.TestCase): self._check_symlink_loop(BASE, 'linkV') os.symlink(join('linkW/../linkW'), join('linkW')) self._check_symlink_loop(BASE, 'linkW') + # Non-strict + self._check_symlink_loop(BASE, 'linkW', 'foo', strict=False) def test_glob(self): P = self.cls @@ -2040,6 +2094,8 @@ class PosixPathTest(_BasePathTest, unittest.TestCase): self.assertEqual(given, expect) self.assertEqual(set(p.rglob("FILEd*")), set()) + @unittest.skipUnless(hasattr(pwd, 'getpwall'), + 'pwd module does not expose getpwall()') def test_expanduser(self): P = self.cls support.import_module('pwd') diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 66a7b5573c..0ea2af5541 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1,6 +1,7 @@ # A test suite for pdb; not very comprehensive at the moment. import doctest +import os import pdb import sys import types @@ -34,7 +35,7 @@ def test_pdb_displayhook(): """This tests the custom displayhook for pdb. >>> def test_function(foo, bar): - ... import pdb; pdb.Pdb(nosigint=True).set_trace() + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... pass >>> with PdbTestInput([ @@ -74,7 +75,7 @@ def test_pdb_basic_commands(): ... return foo.upper() >>> def test_function(): - ... import pdb; pdb.Pdb(nosigint=True).set_trace() + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... ret = test_function_2('baz') ... print(ret) @@ -173,7 +174,7 @@ def test_pdb_breakpoint_commands(): """Test basic commands related to breakpoints. >>> def test_function(): - ... import pdb; pdb.Pdb(nosigint=True).set_trace() + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... print(1) ... print(2) ... print(3) @@ -305,7 +306,7 @@ def test_list_commands(): ... return foo >>> def test_function(): - ... import pdb; pdb.Pdb(nosigint=True).set_trace() + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... ret = test_function_2('baz') >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE @@ -328,7 +329,7 @@ def test_list_commands(): -> ret = test_function_2('baz') (Pdb) list 1 def test_function(): - 2 import pdb; pdb.Pdb(nosigint=True).set_trace() + 2 import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() 3 -> ret = test_function_2('baz') [EOF] (Pdb) step @@ -391,7 +392,7 @@ def test_post_mortem(): ... print('Exception!') >>> def test_function(): - ... import pdb; pdb.Pdb(nosigint=True).set_trace() + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... test_function_2() ... print('Not reached.') @@ -424,7 +425,7 @@ def test_post_mortem(): -> 1/0 (Pdb) list 1 def test_function(): - 2 import pdb; pdb.Pdb(nosigint=True).set_trace() + 2 import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() 3 -> test_function_2() 4 print('Not reached.') [EOF] @@ -448,7 +449,7 @@ def test_pdb_skip_modules(): >>> def skip_module(): ... import string - ... import pdb; pdb.Pdb(skip=['stri*'], nosigint=True).set_trace() + ... import pdb; pdb.Pdb(skip=['stri*'], nosigint=True, readrc=False).set_trace() ... string.capwords('FOO') >>> with PdbTestInput([ @@ -477,7 +478,7 @@ def test_pdb_skip_modules_with_callback(): >>> def skip_module(): ... def callback(): ... return None - ... import pdb; pdb.Pdb(skip=['module_to_skip*'], nosigint=True).set_trace() + ... import pdb; pdb.Pdb(skip=['module_to_skip*'], nosigint=True, readrc=False).set_trace() ... mod.foo_pony(callback) >>> with PdbTestInput([ @@ -518,7 +519,7 @@ def test_pdb_continue_in_bottomframe(): """Test that "continue" and "next" work properly in bottom frame (issue #5294). >>> def test_function(): - ... import pdb, sys; inst = pdb.Pdb(nosigint=True) + ... import pdb, sys; inst = pdb.Pdb(nosigint=True, readrc=False) ... inst.set_trace() ... inst.botframe = sys._getframe() # hackery to get the right botframe ... print(1) @@ -558,8 +559,7 @@ def test_pdb_continue_in_bottomframe(): def pdb_invoke(method, arg): """Run pdb.method(arg).""" - import pdb - getattr(pdb.Pdb(nosigint=True), method)(arg) + getattr(pdb.Pdb(nosigint=True, readrc=False), method)(arg) def test_pdb_run_with_incorrect_argument(): @@ -608,7 +608,7 @@ def test_next_until_return_at_return_event(): ... x = 2 >>> def test_function(): - ... import pdb; pdb.Pdb(nosigint=True).set_trace() + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... test_function_2() ... test_function_2() ... test_function_2() @@ -674,7 +674,7 @@ def test_pdb_next_command_for_generator(): ... yield 2 >>> def test_function(): - ... import pdb; pdb.Pdb(nosigint=True).set_trace() + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... it = test_gen() ... try: ... if next(it) != 0: @@ -734,7 +734,7 @@ def test_pdb_return_command_for_generator(): ... yield 2 >>> def test_function(): - ... import pdb; pdb.Pdb(nosigint=True).set_trace() + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... it = test_gen() ... try: ... if next(it) != 0: @@ -789,7 +789,7 @@ def test_pdb_until_command_for_generator(): ... yield 2 >>> def test_function(): - ... import pdb; pdb.Pdb(nosigint=True).set_trace() + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... for i in test_gen(): ... print(i) ... print("finished") @@ -831,7 +831,7 @@ def test_pdb_next_command_in_generator_for_loop(): ... return 1 >>> def test_function(): - ... import pdb; pdb.Pdb(nosigint=True).set_trace() + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... for i in test_gen(): ... print('value', i) ... x = 123 @@ -876,7 +876,7 @@ def test_pdb_next_command_subiterator(): ... return x >>> def test_function(): - ... import pdb; pdb.Pdb(nosigint=True).set_trace() + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... for i in test_gen(): ... print('value', i) ... x = 123 @@ -1049,7 +1049,7 @@ class PdbTestCase(unittest.TestCase): import pdb def start_pdb(): - pdb.Pdb().set_trace() + pdb.Pdb(readrc=False).set_trace() x = 1 y = 1 @@ -1078,6 +1078,38 @@ class PdbTestCase(unittest.TestCase): .format(expected, stdout)) + def test_readrc_kwarg(self): + script = textwrap.dedent(""" + import pdb; pdb.Pdb(readrc=False).set_trace() + + print('hello') + """) + + save_home = os.environ.pop('HOME', None) + try: + with support.temp_cwd(): + with open('.pdbrc', 'w') as f: + f.write("invalid\n") + + with open('main.py', 'w') as f: + f.write(script) + + cmd = [sys.executable, 'main.py'] + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + with proc: + stdout, stderr = proc.communicate(b'q\n') + self.assertNotIn("NameError: name 'invalid' is not defined", + stdout.decode()) + + finally: + if save_home is not None: + os.environ['HOME'] = save_home + def tearDown(self): support.unlink(support.TESTFN) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 41e5091fab..b0336407c3 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1,9 +1,8 @@ import dis import re import sys -from io import StringIO +import textwrap import unittest -from math import copysign from test.bytecode_helper import BytecodeTestCase @@ -30,22 +29,25 @@ class TestTranforms(BytecodeTestCase): def test_global_as_constant(self): # LOAD_GLOBAL None/True/False --> LOAD_CONST None/True/False - def f(x): - None - None + def f(): + x = None + x = None return x - def g(x): - True + def g(): + x = True return x - def h(x): - False + def h(): + x = False return x + for func, elem in ((f, None), (g, True), (h, False)): self.assertNotInBytecode(func, 'LOAD_GLOBAL') self.assertInBytecode(func, 'LOAD_CONST', elem) + def f(): 'Adding a docstring made this test fail in Py2.5.0' return None + self.assertNotInBytecode(f, 'LOAD_GLOBAL') self.assertInBytecode(f, 'LOAD_CONST', None) diff --git a/Lib/test/test_pep247.py b/Lib/test/test_pep247.py deleted file mode 100644 index c17ceed810..0000000000 --- a/Lib/test/test_pep247.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -Test suite to check compliance with PEP 247, the standard API -for hashing algorithms -""" - -import hmac -import unittest -from hashlib import md5, sha1, sha224, sha256, sha384, sha512 - -class Pep247Test(unittest.TestCase): - - def check_module(self, module, key=None): - self.assertTrue(hasattr(module, 'digest_size')) - self.assertTrue(module.digest_size is None or module.digest_size > 0) - self.check_object(module.new, module.digest_size, key) - - def check_object(self, cls, digest_size, key, digestmod=None): - if key is not None: - if digestmod is None: - digestmod = md5 - obj1 = cls(key, digestmod=digestmod) - obj2 = cls(key, b'string', digestmod=digestmod) - h1 = cls(key, b'string', digestmod=digestmod).digest() - obj3 = cls(key, digestmod=digestmod) - obj3.update(b'string') - h2 = obj3.digest() - else: - obj1 = cls() - obj2 = cls(b'string') - h1 = cls(b'string').digest() - obj3 = cls() - obj3.update(b'string') - h2 = obj3.digest() - self.assertEqual(h1, h2) - self.assertTrue(hasattr(obj1, 'digest_size')) - - if digest_size is not None: - self.assertEqual(obj1.digest_size, digest_size) - - self.assertEqual(obj1.digest_size, len(h1)) - obj1.update(b'string') - obj_copy = obj1.copy() - self.assertEqual(obj1.digest(), obj_copy.digest()) - self.assertEqual(obj1.hexdigest(), obj_copy.hexdigest()) - - digest, hexdigest = obj1.digest(), obj1.hexdigest() - hd2 = "" - for byte in digest: - hd2 += '%02x' % byte - self.assertEqual(hd2, hexdigest) - - def test_md5(self): - self.check_object(md5, None, None) - - def test_sha(self): - self.check_object(sha1, None, None) - self.check_object(sha224, None, None) - self.check_object(sha256, None, None) - self.check_object(sha384, None, None) - self.check_object(sha512, None, None) - - def test_hmac(self): - self.check_module(hmac, key=b'abc') - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index d467d52bad..e6c5d08522 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -34,8 +34,6 @@ class PyUnpicklerTests(AbstractUnpickleTests): unpickler = pickle._Unpickler bad_stack_errors = (IndexError,) - bad_mark_errors = (IndexError, pickle.UnpicklingError, - TypeError, AttributeError, EOFError) truncated_errors = (pickle.UnpicklingError, EOFError, AttributeError, ValueError, struct.error, IndexError, ImportError) @@ -70,8 +68,6 @@ class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, pickler = pickle._Pickler unpickler = pickle._Unpickler bad_stack_errors = (pickle.UnpicklingError, IndexError) - bad_mark_errors = (pickle.UnpicklingError, IndexError, - TypeError, AttributeError, EOFError) truncated_errors = (pickle.UnpicklingError, EOFError, AttributeError, ValueError, struct.error, IndexError, ImportError) @@ -143,9 +139,7 @@ if has_c_implementation: class CUnpicklerTests(PyUnpicklerTests): unpickler = _pickle.Unpickler bad_stack_errors = (pickle.UnpicklingError,) - bad_mark_errors = (EOFError,) - truncated_errors = (pickle.UnpicklingError, EOFError, - AttributeError, ValueError) + truncated_errors = (pickle.UnpicklingError,) class CPicklerTests(PyPicklerTests): pickler = _pickle.Pickler @@ -341,6 +335,9 @@ class CompatPickleTests(unittest.TestCase): if (module2, name2) == ('exceptions', 'OSError'): attr = getattribute(module3, name3) self.assertTrue(issubclass(attr, OSError)) + elif (module2, name2) == ('exceptions', 'ImportError'): + attr = getattribute(module3, name3) + self.assertTrue(issubclass(attr, ImportError)) else: module, name = mapping(module2, name2) if module3[:1] != '_': @@ -407,6 +404,11 @@ class CompatPickleTests(unittest.TestCase): if exc is not OSError and issubclass(exc, OSError): self.assertEqual(reverse_mapping('builtins', name), ('exceptions', 'OSError')) + elif exc is not ImportError and issubclass(exc, ImportError): + self.assertEqual(reverse_mapping('builtins', name), + ('exceptions', 'ImportError')) + self.assertEqual(mapping('exceptions', name), + ('exceptions', name)) else: self.assertEqual(reverse_mapping('builtins', name), ('exceptions', name)) diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index bbe6875545..86bebfa026 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -1,9 +1,9 @@ -import struct import pickle import pickletools from test import support from test.pickletester import AbstractPickleTests from test.pickletester import AbstractPickleModuleTests +import unittest class OptimizedPickleTests(AbstractPickleTests, AbstractPickleModuleTests): @@ -59,8 +59,40 @@ class OptimizedPickleTests(AbstractPickleTests, AbstractPickleModuleTests): self.assertNotIn(pickle.BINPUT, pickled2) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'bytes_types', + 'UP_TO_NEWLINE', 'TAKEN_FROM_ARGUMENT1', + 'TAKEN_FROM_ARGUMENT4', 'TAKEN_FROM_ARGUMENT4U', + 'TAKEN_FROM_ARGUMENT8U', 'ArgumentDescriptor', + 'read_uint1', 'read_uint2', 'read_int4', 'read_uint4', + 'read_uint8', 'read_stringnl', 'read_stringnl_noescape', + 'read_stringnl_noescape_pair', 'read_string1', + 'read_string4', 'read_bytes1', 'read_bytes4', + 'read_bytes8', 'read_unicodestringnl', + 'read_unicodestring1', 'read_unicodestring4', + 'read_unicodestring8', 'read_decimalnl_short', + 'read_decimalnl_long', 'read_floatnl', 'read_float8', + 'read_long1', 'read_long4', + 'uint1', 'uint2', 'int4', 'uint4', 'uint8', 'stringnl', + 'stringnl_noescape', 'stringnl_noescape_pair', 'string1', + 'string4', 'bytes1', 'bytes4', 'bytes8', + 'unicodestringnl', 'unicodestring1', 'unicodestring4', + 'unicodestring8', 'decimalnl_short', 'decimalnl_long', + 'floatnl', 'float8', 'long1', 'long4', + 'StackObject', + 'pyint', 'pylong', 'pyinteger_or_bool', 'pybool', 'pyfloat', + 'pybytes_or_str', 'pystring', 'pybytes', 'pyunicode', + 'pynone', 'pytuple', 'pylist', 'pydict', 'pyset', + 'pyfrozenset', 'anyobject', 'markobject', 'stackslice', + 'OpcodeInfo', 'opcodes', 'code2op', + } + support.check__all__(self, pickletools, blacklist=blacklist) + + def test_main(): support.run_unittest(OptimizedPickleTests) + support.run_unittest(MiscTestCase) support.run_doctest(pickletools) diff --git a/Lib/test/test_pipes.py b/Lib/test/test_pipes.py index 6a7b45fb46..ad01d08481 100644 --- a/Lib/test/test_pipes.py +++ b/Lib/test/test_pipes.py @@ -2,6 +2,7 @@ import pipes import os import string import unittest +import shutil from test.support import TESTFN, run_unittest, unlink, reap_children if os.name != 'posix': @@ -18,6 +19,8 @@ class SimplePipeTests(unittest.TestCase): unlink(f) def testSimplePipe1(self): + if shutil.which('tr') is None: + self.skipTest('tr is not available') t = pipes.Template() t.append(s_command, pipes.STDIN_STDOUT) f = t.open(TESTFN, 'w') @@ -27,6 +30,8 @@ class SimplePipeTests(unittest.TestCase): self.assertEqual(f.read(), 'HELLO WORLD #1') def testSimplePipe2(self): + if shutil.which('tr') is None: + self.skipTest('tr is not available') with open(TESTFN, 'w') as f: f.write('hello world #2') t = pipes.Template() @@ -36,6 +41,8 @@ class SimplePipeTests(unittest.TestCase): self.assertEqual(f.read(), 'HELLO WORLD #2') def testSimplePipe3(self): + if shutil.which('tr') is None: + self.skipTest('tr is not available') with open(TESTFN, 'w') as f: f.write('hello world #2') t = pipes.Template() diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index a82058760d..fc04dcfd7f 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -81,8 +81,9 @@ class PkgutilTests(unittest.TestCase): self.assertEqual(res2, RESOURCE_DATA) names = [] - for loader, name, ispkg in pkgutil.iter_modules([zip_file]): - names.append(name) + for moduleinfo in pkgutil.iter_modules([zip_file]): + self.assertIsInstance(moduleinfo, pkgutil.ModuleInfo) + names.append(moduleinfo.name) self.assertEqual(names, ['test_getdata_zipfile']) del sys.path[0] @@ -413,6 +414,7 @@ class ImportlibMigrationTests(unittest.TestCase): self.assertIsNotNone(pkgutil.get_loader("test.support")) self.assertEqual(len(w.warnings), 0) + @unittest.skipIf(__name__ == '__main__', 'not compatible with __main__') def test_get_loader_handles_missing_loader_attribute(self): global __loader__ this_loader = __loader__ diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 18de110ecb..5f875eff6c 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -33,6 +33,7 @@ class PlatformTest(unittest.TestCase): 'import platform; print(platform.architecture())'] p = subprocess.Popen(cmd, stdout=subprocess.PIPE) return p.communicate() + real = os.path.realpath(sys.executable) link = os.path.abspath(support.TESTFN) os.symlink(real, link) @@ -262,7 +263,7 @@ class PlatformTest(unittest.TestCase): with warnings.catch_warnings(): warnings.filterwarnings( 'ignore', - 'dist\(\) and linux_distribution\(\) ' + r'dist\(\) and linux_distribution\(\) ' 'functions are deprecated .*', PendingDeprecationWarning, ) @@ -338,7 +339,7 @@ class PlatformTest(unittest.TestCase): with warnings.catch_warnings(): warnings.filterwarnings( 'ignore', - 'dist\(\) and linux_distribution\(\) ' + r'dist\(\) and linux_distribution\(\) ' 'functions are deprecated .*', PendingDeprecationWarning, ) diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index 692cac4bdf..c77a6bf9d0 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -7,7 +7,6 @@ import datetime import codecs import binascii import collections -import struct from test import support from io import BytesIO @@ -534,8 +533,14 @@ class TestPlistlibDeprecated(unittest.TestCase): self.assertEqual(cur, in_data) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {"PlistFormat", "PLISTHEADER"} + support.check__all__(self, plistlib, blacklist=blacklist) + + def test_main(): - support.run_unittest(TestPlistlib, TestPlistlibDeprecated) + support.run_unittest(TestPlistlib, TestPlistlibDeprecated, MiscTestCase) if __name__ == '__main__': diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index bceeb93ad1..e5b16dc98a 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -8,7 +8,6 @@ import asyncore import asynchat import socket import os -import time import errno from unittest import TestCase, skipUnless @@ -153,10 +152,12 @@ class DummyPOP3Handler(asynchat.async_chat): def cmd_stls(self, arg): if self.tls_active is False: self.push('+OK Begin TLS negotiation') - tls_sock = ssl.wrap_socket(self.socket, certfile=CERTFILE, - server_side=True, - do_handshake_on_connect=False, - suppress_ragged_eofs=False) + context = ssl.SSLContext() + context.load_cert_chain(CERTFILE) + tls_sock = context.wrap_socket(self.socket, + server_side=True, + do_handshake_on_connect=False, + suppress_ragged_eofs=False) self.del_channel() self.set_socket(tls_sock) self.tls_active = True diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 2a59c38174..029d0815e9 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1,6 +1,7 @@ "Test posix functions" from test import support +android_not_root = support.android_not_root # Skip these tests if there is no posix module. posix = support.import_module('posix') @@ -11,7 +12,6 @@ import time import os import platform import pwd -import shutil import stat import tempfile import unittest @@ -398,7 +398,7 @@ class PosixTester(unittest.TestCase): self.assertTrue(posix.stat(fp.fileno())) self.assertRaisesRegex(TypeError, - 'should be string, bytes or integer, not', + 'should be string, bytes, os.PathLike or integer, not', posix.stat, float(fp.fileno())) finally: fp.close() @@ -408,19 +408,22 @@ class PosixTester(unittest.TestCase): def test_stat(self): self.assertTrue(posix.stat(support.TESTFN)) self.assertTrue(posix.stat(os.fsencode(support.TESTFN))) - self.assertTrue(posix.stat(bytearray(os.fsencode(support.TESTFN)))) + self.assertWarnsRegex(DeprecationWarning, + 'should be string, bytes, os.PathLike or integer, not', + posix.stat, bytearray(os.fsencode(support.TESTFN))) self.assertRaisesRegex(TypeError, - 'can\'t specify None for path argument', + 'should be string, bytes, os.PathLike or integer, not', posix.stat, None) self.assertRaisesRegex(TypeError, - 'should be string, bytes or integer, not', + 'should be string, bytes, os.PathLike or integer, not', posix.stat, list(support.TESTFN)) self.assertRaisesRegex(TypeError, - 'should be string, bytes or integer, not', + 'should be string, bytes, os.PathLike or integer, not', posix.stat, list(os.fsencode(support.TESTFN))) @unittest.skipUnless(hasattr(posix, 'mkfifo'), "don't have mkfifo()") + @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") def test_mkfifo(self): support.unlink(support.TESTFN) posix.mkfifo(support.TESTFN, stat.S_IRUSR | stat.S_IWUSR) @@ -428,6 +431,7 @@ class PosixTester(unittest.TestCase): @unittest.skipUnless(hasattr(posix, 'mknod') and hasattr(stat, 'S_IFIFO'), "don't have mknod()/S_IFIFO") + @unittest.skipIf(android_not_root, "mknod not allowed, non root user") def test_mknod(self): # Test using mknod() to create a FIFO (the only use specified # by POSIX). @@ -798,7 +802,11 @@ class PosixTester(unittest.TestCase): groups = idg.read().strip() ret = idg.close() - if ret is not None or not groups: + try: + idg_groups = set(int(g) for g in groups.split()) + except ValueError: + idg_groups = set() + if ret is not None or not idg_groups: raise unittest.SkipTest("need working 'id -G'") # Issues 16698: OS X ABIs prior to 10.6 have limits on getgroups() @@ -809,12 +817,11 @@ class PosixTester(unittest.TestCase): raise unittest.SkipTest("getgroups(2) is broken prior to 10.6") # 'id -G' and 'os.getgroups()' should return the same - # groups, ignoring order and duplicates. - # #10822 - it is implementation defined whether posix.getgroups() - # includes the effective gid so we include it anyway, since id -G does - self.assertEqual( - set([int(x) for x in groups.split()]), - set(posix.getgroups() + [posix.getegid()])) + # groups, ignoring order, duplicates, and the effective gid. + # #10822/#26944 - It is implementation defined whether + # posix.getgroups() includes the effective gid. + symdiff = idg_groups.symmetric_difference(posix.getgroups()) + self.assertTrue(not symdiff or symdiff == {posix.getegid()}) # tests for the posix *at functions follow @@ -863,9 +870,9 @@ class PosixTester(unittest.TestCase): self.assertEqual(s1, s2) s2 = posix.stat(support.TESTFN, dir_fd=None) self.assertEqual(s1, s2) - self.assertRaisesRegex(TypeError, 'should be integer, not', + self.assertRaisesRegex(TypeError, 'should be integer or None, not', posix.stat, support.TESTFN, dir_fd=posix.getcwd()) - self.assertRaisesRegex(TypeError, 'should be integer, not', + self.assertRaisesRegex(TypeError, 'should be integer or None, not', posix.stat, support.TESTFN, dir_fd=float(f)) self.assertRaises(OverflowError, posix.stat, support.TESTFN, dir_fd=10**20) @@ -903,6 +910,7 @@ class PosixTester(unittest.TestCase): posix.close(f) @unittest.skipUnless(os.link in os.supports_dir_fd, "test needs dir_fd support in os.link()") + @unittest.skipIf(android_not_root, "hard link not allowed, non root user") def test_link_dir_fd(self): f = posix.open(posix.getcwd(), posix.O_RDONLY) try: @@ -926,6 +934,7 @@ class PosixTester(unittest.TestCase): @unittest.skipUnless((os.mknod in os.supports_dir_fd) and hasattr(stat, 'S_IFIFO'), "test requires both stat.S_IFIFO and dir_fd support for os.mknod()") + @unittest.skipIf(android_not_root, "mknod not allowed, non root user") def test_mknod_dir_fd(self): # Test using mknodat() to create a FIFO (the only use specified # by POSIX). @@ -1009,6 +1018,7 @@ class PosixTester(unittest.TestCase): posix.close(f) @unittest.skipUnless(os.mkfifo in os.supports_dir_fd, "test needs dir_fd support in os.mkfifo()") + @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") def test_mkfifo_dir_fd(self): support.unlink(support.TESTFN) f = posix.open(posix.getcwd(), posix.O_RDONLY) diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 0783c36b9f..8a1e33b0c8 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -596,5 +596,85 @@ class PosixCommonTest(test_genericpath.CommonTest, unittest.TestCase): attributes = ['relpath', 'samefile', 'sameopenfile', 'samestat'] +class PathLikeTests(unittest.TestCase): + + path = posixpath + + class PathLike: + def __init__(self, path=''): + self.path = path + def __fspath__(self): + if isinstance(self.path, BaseException): + raise self.path + else: + return self.path + + def setUp(self): + self.file_name = support.TESTFN.lower() + self.file_path = self.PathLike(support.TESTFN) + self.addCleanup(support.unlink, self.file_name) + with open(self.file_name, 'xb', 0) as file: + file.write(b"test_posixpath.PathLikeTests") + + def assertPathEqual(self, func): + self.assertEqual(func(self.file_path), func(self.file_name)) + + def test_path_normcase(self): + self.assertPathEqual(self.path.normcase) + + def test_path_isabs(self): + self.assertPathEqual(self.path.isabs) + + def test_path_join(self): + self.assertEqual(self.path.join('a', self.PathLike('b'), 'c'), + self.path.join('a', 'b', 'c')) + + def test_path_split(self): + self.assertPathEqual(self.path.split) + + def test_path_splitext(self): + self.assertPathEqual(self.path.splitext) + + def test_path_splitdrive(self): + self.assertPathEqual(self.path.splitdrive) + + def test_path_basename(self): + self.assertPathEqual(self.path.basename) + + def test_path_dirname(self): + self.assertPathEqual(self.path.dirname) + + def test_path_islink(self): + self.assertPathEqual(self.path.islink) + + def test_path_lexists(self): + self.assertPathEqual(self.path.lexists) + + def test_path_ismount(self): + self.assertPathEqual(self.path.ismount) + + def test_path_expanduser(self): + self.assertPathEqual(self.path.expanduser) + + def test_path_expandvars(self): + self.assertPathEqual(self.path.expandvars) + + def test_path_normpath(self): + self.assertPathEqual(self.path.normpath) + + def test_path_abspath(self): + self.assertPathEqual(self.path.abspath) + + def test_path_realpath(self): + self.assertPathEqual(self.path.realpath) + + def test_path_relpath(self): + self.assertPathEqual(self.path.relpath) + + def test_path_commonpath(self): + common_path = self.path.commonpath([self.file_path, self.file_name]) + self.assertEqual(common_path, self.file_name) + + if __name__=="__main__": unittest.main() diff --git a/Lib/test/test_pow.py b/Lib/test/test_pow.py index ba608fbb3e..cac1ae5ea2 100644 --- a/Lib/test/test_pow.py +++ b/Lib/test/test_pow.py @@ -1,4 +1,4 @@ -import test.support, unittest +import unittest class PowTest(unittest.TestCase): diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index ef5e99ee26..15f88e4fcd 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -277,7 +277,6 @@ class SmallPtyTests(unittest.TestCase): socketpair = self._socketpair() masters = [s.fileno() for s in socketpair] - os.close(masters[1]) socketpair[1].close() os.close(write_to_stdin_fd) diff --git a/Lib/test/test_pulldom.py b/Lib/test/test_pulldom.py index 1932c6bb99..3d89e3adda 100644 --- a/Lib/test/test_pulldom.py +++ b/Lib/test/test_pulldom.py @@ -1,6 +1,5 @@ import io import unittest -import sys import xml.sax from xml.sax.xmlreader import AttributesImpl diff --git a/Lib/test/test_pwd.py b/Lib/test/test_pwd.py index b7b1a4a5f6..ac9cff789e 100644 --- a/Lib/test/test_pwd.py +++ b/Lib/test/test_pwd.py @@ -4,10 +4,19 @@ from test import support pwd = support.import_module('pwd') +def _getpwall(): + # Android does not have getpwall. + if hasattr(pwd, 'getpwall'): + return pwd.getpwall() + elif hasattr(pwd, 'getpwuid'): + return [pwd.getpwuid(0)] + else: + return [] + class PwdTest(unittest.TestCase): def test_values(self): - entries = pwd.getpwall() + entries = _getpwall() for e in entries: self.assertEqual(len(e), 7) @@ -33,7 +42,7 @@ class PwdTest(unittest.TestCase): # and check afterwards (done in test_values_extended) def test_values_extended(self): - entries = pwd.getpwall() + entries = _getpwall() entriesbyname = {} entriesbyuid = {} @@ -57,12 +66,13 @@ class PwdTest(unittest.TestCase): self.assertRaises(TypeError, pwd.getpwuid, 3.14) self.assertRaises(TypeError, pwd.getpwnam) self.assertRaises(TypeError, pwd.getpwnam, 42) - self.assertRaises(TypeError, pwd.getpwall, 42) + if hasattr(pwd, 'getpwall'): + self.assertRaises(TypeError, pwd.getpwall, 42) # try to get some errors bynames = {} byuids = {} - for (n, p, u, g, gecos, d, s) in pwd.getpwall(): + for (n, p, u, g, gecos, d, s) in _getpwall(): bynames[n] = u byuids[u] = n @@ -96,13 +106,17 @@ class PwdTest(unittest.TestCase): # loop, say), pwd.getpwuid() might still be able to find data for that # uid. Using sys.maxint may provoke the same problems, but hopefully # it will be a more repeatable failure. + # Android accepts a very large span of uids including sys.maxsize and + # -1; it raises KeyError with 1 or 2 for example. fakeuid = sys.maxsize self.assertNotIn(fakeuid, byuids) - self.assertRaises(KeyError, pwd.getpwuid, fakeuid) + if not support.is_android: + self.assertRaises(KeyError, pwd.getpwuid, fakeuid) # -1 shouldn't be a valid uid because it has a special meaning in many # uid-related functions - self.assertRaises(KeyError, pwd.getpwuid, -1) + if not support.is_android: + self.assertRaises(KeyError, pwd.getpwuid, -1) # should be out of uid_t range self.assertRaises(KeyError, pwd.getpwuid, 2**128) self.assertRaises(KeyError, pwd.getpwuid, -2**128) diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index 6ffbbbda27..2cff1c526e 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -156,9 +156,9 @@ class PyclbrTest(TestCase): # These were once about the 10 longest modules cm('random', ignore=('Random',)) # from _random import Random as CoreGenerator cm('cgi', ignore=('log',)) # set with = in module - cm('pickle') + cm('pickle', ignore=('partial',)) cm('aifc', ignore=('openfp', '_aifc_params')) # set with = in module - cm('sre_parse', ignore=('dump', 'groups')) # from sre_constants import *; property + cm('sre_parse', ignore=('dump', 'groups', 'pos')) # from sre_constants import *; property cm('pdb') cm('pydoc') diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 1f7ab7ffa1..ed438d5664 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -84,6 +84,8 @@ CLASSES | Data and other attributes defined here: |\x20\x20 | NO_MEANING = 'eggs' + |\x20\x20 + | __annotations__ = {'NO_MEANING': <class 'str'>} \x20\x20\x20\x20 class C(builtins.object) | Methods defined here: @@ -196,6 +198,8 @@ Data descriptors defined here:<br> Data and other attributes defined here:<br> <dl><dt><strong>NO_MEANING</strong> = 'eggs'</dl> +<dl><dt><strong>__annotations__</strong> = {'NO_MEANING': <class 'str'>}</dl> + </td></tr></table> <p> <table width="100%%" cellspacing=0 cellpadding=2 border=0 summary="section"> <tr bgcolor="#ffc8d8"> @@ -264,7 +268,7 @@ Use help() to get the interactive help utility. Use help(str) for help on the str class.'''.replace('\n', os.linesep) # output pattern for module with bad imports -badimport_pattern = "problem in %s - ImportError: No module named %r" +badimport_pattern = "problem in %s - ModuleNotFoundError: No module named %r" expected_dynamicattribute_pattern = """ Help on class DA in module %s: @@ -639,8 +643,9 @@ class PydocDocTest(unittest.TestCase): del expected['__doc__'] del expected['__class__'] # inspect resolves descriptors on type into methods, but vars doesn't, - # so we need to update __subclasshook__. + # so we need to update __subclasshook__ and __init_subclass__. expected['__subclasshook__'] = TestClass.__subclasshook__ + expected['__init_subclass__'] = TestClass.__init_subclass__ methods = pydoc.allmethods(TestClass) self.assertDictEqual(methods, expected) @@ -868,6 +873,22 @@ class TestDescriptions(unittest.TestCase): self.assertEqual(self._get_summary_line(t.wrap), "wrap(text) method of textwrap.TextWrapper instance") + def test_field_order_for_named_tuples(self): + Person = namedtuple('Person', ['nickname', 'firstname', 'agegroup']) + s = pydoc.render_doc(Person) + self.assertLess(s.index('nickname'), s.index('firstname')) + self.assertLess(s.index('firstname'), s.index('agegroup')) + + class NonIterableFields: + _fields = None + + class NonHashableFields: + _fields = [[]] + + # Make sure these doesn't fail + pydoc.render_doc(NonIterableFields) + pydoc.render_doc(NonHashableFields) + @requires_docstrings def test_bound_builtin_method(self): s = StringIO() diff --git a/Lib/test/test_quopri.py b/Lib/test/test_quopri.py index 7cac013446..715544c8a9 100644 --- a/Lib/test/test_quopri.py +++ b/Lib/test/test_quopri.py @@ -1,6 +1,6 @@ import unittest -import sys, os, io, subprocess +import sys, io, subprocess import quopri diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index e80ed17a8c..5b6a4f06ba 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -7,6 +7,7 @@ import warnings from functools import partial from math import log, exp, pi, fsum, sin from test import support +from fractions import Fraction class TestBasicOps: # Superclass with tests common to all generators. @@ -109,6 +110,7 @@ class TestBasicOps: self.assertEqual(self.gen.sample([], 0), []) # test edge case N==k==0 # Exception raised if size of sample exceeds that of population self.assertRaises(ValueError, self.gen.sample, population, N+1) + self.assertRaises(ValueError, self.gen.sample, [], -1) def test_sample_distribution(self): # For the entire allowable range of 0 <= k <= N, validate that @@ -141,6 +143,83 @@ class TestBasicOps: def test_sample_on_dicts(self): self.assertRaises(TypeError, self.gen.sample, dict.fromkeys('abcdef'), 2) + def test_choices(self): + choices = self.gen.choices + data = ['red', 'green', 'blue', 'yellow'] + str_data = 'abcd' + range_data = range(4) + set_data = set(range(4)) + + # basic functionality + for sample in [ + choices(data, k=5), + choices(data, range(4), k=5), + choices(k=5, population=data, weights=range(4)), + choices(k=5, population=data, cum_weights=range(4)), + ]: + self.assertEqual(len(sample), 5) + self.assertEqual(type(sample), list) + self.assertTrue(set(sample) <= set(data)) + + # test argument handling + with self.assertRaises(TypeError): # missing arguments + choices(2) + + self.assertEqual(choices(data, k=0), []) # k == 0 + self.assertEqual(choices(data, k=-1), []) # negative k behaves like ``[0] * -1`` + with self.assertRaises(TypeError): + choices(data, k=2.5) # k is a float + + self.assertTrue(set(choices(str_data, k=5)) <= set(str_data)) # population is a string sequence + self.assertTrue(set(choices(range_data, k=5)) <= set(range_data)) # population is a range + with self.assertRaises(TypeError): + choices(set_data, k=2) # population is not a sequence + + self.assertTrue(set(choices(data, None, k=5)) <= set(data)) # weights is None + self.assertTrue(set(choices(data, weights=None, k=5)) <= set(data)) + with self.assertRaises(ValueError): + choices(data, [1,2], k=5) # len(weights) != len(population) + with self.assertRaises(TypeError): + choices(data, 10, k=5) # non-iterable weights + with self.assertRaises(TypeError): + choices(data, [None]*4, k=5) # non-numeric weights + for weights in [ + [15, 10, 25, 30], # integer weights + [15.1, 10.2, 25.2, 30.3], # float weights + [Fraction(1, 3), Fraction(2, 6), Fraction(3, 6), Fraction(4, 6)], # fractional weights + [True, False, True, False] # booleans (include / exclude) + ]: + self.assertTrue(set(choices(data, weights, k=5)) <= set(data)) + + with self.assertRaises(ValueError): + choices(data, cum_weights=[1,2], k=5) # len(weights) != len(population) + with self.assertRaises(TypeError): + choices(data, cum_weights=10, k=5) # non-iterable cum_weights + with self.assertRaises(TypeError): + choices(data, cum_weights=[None]*4, k=5) # non-numeric cum_weights + with self.assertRaises(TypeError): + choices(data, range(4), cum_weights=range(4), k=5) # both weights and cum_weights + for weights in [ + [15, 10, 25, 30], # integer cum_weights + [15.1, 10.2, 25.2, 30.3], # float cum_weights + [Fraction(1, 3), Fraction(2, 6), Fraction(3, 6), Fraction(4, 6)], # fractional cum_weights + ]: + self.assertTrue(set(choices(data, cum_weights=weights, k=5)) <= set(data)) + + # Test weight focused on a single element of the population + self.assertEqual(choices('abcd', [1, 0, 0, 0]), ['a']) + self.assertEqual(choices('abcd', [0, 1, 0, 0]), ['b']) + self.assertEqual(choices('abcd', [0, 0, 1, 0]), ['c']) + self.assertEqual(choices('abcd', [0, 0, 0, 1]), ['d']) + + # Test consistency with random.choice() for empty population + with self.assertRaises(IndexError): + choices([], k=1) + with self.assertRaises(IndexError): + choices([], weights=[], k=1) + with self.assertRaises(IndexError): + choices([], cum_weights=[], k=5) + def test_gauss(self): # Ensure that the seed() method initializes all the hidden state. In # particular, through 2.2.1 it failed to reset a piece of state used @@ -551,6 +630,39 @@ class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase): self.assertTrue(stop < x <= start) self.assertEqual((x+stop)%step, 0) + def test_choices_algorithms(self): + # The various ways of specifying weights should produce the same results + choices = self.gen.choices + n = 104729 + + self.gen.seed(8675309) + a = self.gen.choices(range(n), k=10000) + + self.gen.seed(8675309) + b = self.gen.choices(range(n), [1]*n, k=10000) + self.assertEqual(a, b) + + self.gen.seed(8675309) + c = self.gen.choices(range(n), cum_weights=range(1, n+1), k=10000) + self.assertEqual(a, c) + + # Amerian Roulette + population = ['Red', 'Black', 'Green'] + weights = [18, 18, 2] + cum_weights = [18, 36, 38] + expanded_population = ['Red'] * 18 + ['Black'] * 18 + ['Green'] * 2 + + self.gen.seed(9035768) + a = self.gen.choices(expanded_population, k=10000) + + self.gen.seed(9035768) + b = self.gen.choices(population, weights, k=10000) + self.assertEqual(a, b) + + self.gen.seed(9035768) + c = self.gen.choices(population, cum_weights=cum_weights, k=10000) + self.assertEqual(a, c) + def gamma(z, sqrt2pi=(2.0*pi)**0.5): # Reflection to right half of complex plane if z < 0.5: diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index c022f070bf..9e11e518f6 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -1,9 +1,10 @@ # Python test set -- built-in functions -import test.support, unittest +import unittest import sys import pickle import itertools +import test.support # pure Python implementations (3 args only), for comparison def pyrange(start, stop, step): @@ -498,29 +499,32 @@ class RangeTest(unittest.TestCase): import _testcapi rangeiter_type = type(iter(range(0))) - # rangeiter_new doesn't take keyword arguments - with self.assertRaises(TypeError): - rangeiter_type(a=1) - - # rangeiter_new takes exactly 3 arguments - self.assertRaises(TypeError, rangeiter_type) - self.assertRaises(TypeError, rangeiter_type, 1) - self.assertRaises(TypeError, rangeiter_type, 1, 1) - self.assertRaises(TypeError, rangeiter_type, 1, 1, 1, 1) - - # start, stop and stop must fit in C long - for good_val in [_testcapi.LONG_MAX, _testcapi.LONG_MIN]: - rangeiter_type(good_val, good_val, good_val) - for bad_val in [_testcapi.LONG_MAX + 1, _testcapi.LONG_MIN - 1]: - self.assertRaises(OverflowError, - rangeiter_type, bad_val, 1, 1) - self.assertRaises(OverflowError, - rangeiter_type, 1, bad_val, 1) - self.assertRaises(OverflowError, - rangeiter_type, 1, 1, bad_val) - - # step mustn't be zero - self.assertRaises(ValueError, rangeiter_type, 1, 1, 0) + self.assertWarns(DeprecationWarning, rangeiter_type, 1, 3, 1) + + with test.support.check_warnings(('', DeprecationWarning)): + # rangeiter_new doesn't take keyword arguments + with self.assertRaises(TypeError): + rangeiter_type(a=1) + + # rangeiter_new takes exactly 3 arguments + self.assertRaises(TypeError, rangeiter_type) + self.assertRaises(TypeError, rangeiter_type, 1) + self.assertRaises(TypeError, rangeiter_type, 1, 1) + self.assertRaises(TypeError, rangeiter_type, 1, 1, 1, 1) + + # start, stop and stop must fit in C long + for good_val in [_testcapi.LONG_MAX, _testcapi.LONG_MIN]: + rangeiter_type(good_val, good_val, good_val) + for bad_val in [_testcapi.LONG_MAX + 1, _testcapi.LONG_MIN - 1]: + self.assertRaises(OverflowError, + rangeiter_type, bad_val, 1, 1) + self.assertRaises(OverflowError, + rangeiter_type, 1, bad_val, 1) + self.assertRaises(OverflowError, + rangeiter_type, 1, 1, bad_val) + + # step mustn't be zero + self.assertRaises(ValueError, rangeiter_type, 1, 1, 0) def test_slice(self): def check(start, stop, step=None): diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 9acd5abbfd..b945cf094e 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -3,13 +3,13 @@ from test.support import verbose, run_unittest, gc_collect, bigmemtest, _2G, \ import io import locale import re -from re import Scanner import sre_compile -import sre_constants -import sys import string +import sys import traceback import unittest +import warnings +from re import Scanner from weakref import proxy # Misc tests from Tim Peters' re.doc @@ -115,10 +115,10 @@ class ReTests(unittest.TestCase): self.assertEqual(re.sub('(.)', re.escape(s), 'x'), s) self.assertEqual(re.sub('(.)', lambda m: s, 'x'), s) - self.assertEqual(re.sub('(?P<a>x)', '\g<a>\g<a>', 'xx'), 'xxxx') - self.assertEqual(re.sub('(?P<a>x)', '\g<a>\g<1>', 'xx'), 'xxxx') - self.assertEqual(re.sub('(?P<unk>x)', '\g<unk>\g<unk>', 'xx'), 'xxxx') - self.assertEqual(re.sub('(?P<unk>x)', '\g<1>\g<1>', 'xx'), 'xxxx') + self.assertEqual(re.sub('(?P<a>x)', r'\g<a>\g<a>', 'xx'), 'xxxx') + self.assertEqual(re.sub('(?P<a>x)', r'\g<a>\g<1>', 'xx'), 'xxxx') + self.assertEqual(re.sub('(?P<unk>x)', r'\g<unk>\g<unk>', 'xx'), 'xxxx') + self.assertEqual(re.sub('(?P<unk>x)', r'\g<1>\g<1>', 'xx'), 'xxxx') self.assertEqual(re.sub('a', r'\t\n\v\r\f\a\b', 'a'), '\t\n\v\r\f\a\b') self.assertEqual(re.sub('a', '\t\n\v\r\f\a\b', 'a'), '\t\n\v\r\f\a\b') @@ -129,11 +129,11 @@ class ReTests(unittest.TestCase): with self.assertWarns(DeprecationWarning): self.assertEqual(re.sub('a', '\\' + c, 'a'), '\\' + c) - self.assertEqual(re.sub('^\s*', 'X', 'test'), 'Xtest') + self.assertEqual(re.sub(r'^\s*', 'X', 'test'), 'Xtest') def test_bug_449964(self): # fails for group followed by other escape - self.assertEqual(re.sub(r'(?P<unk>x)', '\g<1>\g<1>\\b', 'xx'), + self.assertEqual(re.sub(r'(?P<unk>x)', r'\g<1>\g<1>\b', 'xx'), 'xx\bxx\b') def test_bug_449000(self): @@ -186,18 +186,19 @@ class ReTests(unittest.TestCase): r'octal escape value \777 outside of ' r'range 0-0o377', 0) - self.checkTemplateError('x', r'\1', 'x', 'invalid group reference') - self.checkTemplateError('x', r'\8', 'x', 'invalid group reference') - self.checkTemplateError('x', r'\9', 'x', 'invalid group reference') - self.checkTemplateError('x', r'\11', 'x', 'invalid group reference') - self.checkTemplateError('x', r'\18', 'x', 'invalid group reference') - self.checkTemplateError('x', r'\1a', 'x', 'invalid group reference') - self.checkTemplateError('x', r'\90', 'x', 'invalid group reference') - self.checkTemplateError('x', r'\99', 'x', 'invalid group reference') - self.checkTemplateError('x', r'\118', 'x', 'invalid group reference') # r'\11' + '8' - self.checkTemplateError('x', r'\11a', 'x', 'invalid group reference') - self.checkTemplateError('x', r'\181', 'x', 'invalid group reference') # r'\18' + '1' - self.checkTemplateError('x', r'\800', 'x', 'invalid group reference') # r'\80' + '0' + self.checkTemplateError('x', r'\1', 'x', 'invalid group reference 1', 1) + self.checkTemplateError('x', r'\8', 'x', 'invalid group reference 8', 1) + self.checkTemplateError('x', r'\9', 'x', 'invalid group reference 9', 1) + self.checkTemplateError('x', r'\11', 'x', 'invalid group reference 11', 1) + self.checkTemplateError('x', r'\18', 'x', 'invalid group reference 18', 1) + self.checkTemplateError('x', r'\1a', 'x', 'invalid group reference 1', 1) + self.checkTemplateError('x', r'\90', 'x', 'invalid group reference 90', 1) + self.checkTemplateError('x', r'\99', 'x', 'invalid group reference 99', 1) + self.checkTemplateError('x', r'\118', 'x', 'invalid group reference 11', 1) + self.checkTemplateError('x', r'\11a', 'x', 'invalid group reference 11', 1) + self.checkTemplateError('x', r'\181', 'x', 'invalid group reference 18', 1) + self.checkTemplateError('x', r'\800', 'x', 'invalid group reference 80', 1) + self.checkTemplateError('x', r'\8', '', 'invalid group reference 8', 1) # in python2.3 (etc), these loop endlessly in sre_parser.py self.assertEqual(re.sub('(((((((((((x)))))))))))', r'\11', 'x'), 'x') @@ -221,26 +222,26 @@ class ReTests(unittest.TestCase): self.assertEqual(re.sub('x+', '-', 'abxd'), 'ab-d') def test_symbolic_groups(self): - re.compile('(?P<a>x)(?P=a)(?(a)y)') - re.compile('(?P<a1>x)(?P=a1)(?(a1)y)') - re.compile('(?P<a1>x)\1(?(1)y)') - self.checkPatternError('(?P<a>)(?P<a>)', + re.compile(r'(?P<a>x)(?P=a)(?(a)y)') + re.compile(r'(?P<a1>x)(?P=a1)(?(a1)y)') + re.compile(r'(?P<a1>x)\1(?(1)y)') + self.checkPatternError(r'(?P<a>)(?P<a>)', "redefinition of group name 'a' as group 2; " "was group 1") - self.checkPatternError('(?P<a>(?P=a))', + self.checkPatternError(r'(?P<a>(?P=a))', "cannot refer to an open group", 10) - self.checkPatternError('(?Pxy)', 'unknown extension ?Px') - self.checkPatternError('(?P<a>)(?P=a', 'missing ), unterminated name', 11) - self.checkPatternError('(?P=', 'missing group name', 4) - self.checkPatternError('(?P=)', 'missing group name', 4) - self.checkPatternError('(?P=1)', "bad character in group name '1'", 4) - self.checkPatternError('(?P=a)', "unknown group name 'a'") - self.checkPatternError('(?P=a1)', "unknown group name 'a1'") - self.checkPatternError('(?P=a.)', "bad character in group name 'a.'", 4) - self.checkPatternError('(?P<)', 'missing >, unterminated name', 4) - self.checkPatternError('(?P<a', 'missing >, unterminated name', 4) - self.checkPatternError('(?P<', 'missing group name', 4) - self.checkPatternError('(?P<>)', 'missing group name', 4) + self.checkPatternError(r'(?Pxy)', 'unknown extension ?Px') + self.checkPatternError(r'(?P<a>)(?P=a', 'missing ), unterminated name', 11) + self.checkPatternError(r'(?P=', 'missing group name', 4) + self.checkPatternError(r'(?P=)', 'missing group name', 4) + self.checkPatternError(r'(?P=1)', "bad character in group name '1'", 4) + self.checkPatternError(r'(?P=a)', "unknown group name 'a'") + self.checkPatternError(r'(?P=a1)', "unknown group name 'a1'") + self.checkPatternError(r'(?P=a.)', "bad character in group name 'a.'", 4) + self.checkPatternError(r'(?P<)', 'missing >, unterminated name', 4) + self.checkPatternError(r'(?P<a', 'missing >, unterminated name', 4) + self.checkPatternError(r'(?P<', 'missing group name', 4) + self.checkPatternError(r'(?P<>)', 'missing group name', 4) self.checkPatternError(r'(?P<1>)', "bad character in group name '1'", 4) self.checkPatternError(r'(?P<a.>)', "bad character in group name 'a.'", 4) self.checkPatternError(r'(?(', 'missing group name', 3) @@ -259,35 +260,35 @@ class ReTests(unittest.TestCase): self.assertEqual(re.match(pat, 'xc8yz').span(), (0, 5)) def test_symbolic_refs(self): - self.checkTemplateError('(?P<a>x)', '\g<a', 'xx', + self.checkTemplateError('(?P<a>x)', r'\g<a', 'xx', 'missing >, unterminated name', 3) - self.checkTemplateError('(?P<a>x)', '\g<', 'xx', + self.checkTemplateError('(?P<a>x)', r'\g<', 'xx', 'missing group name', 3) - self.checkTemplateError('(?P<a>x)', '\g', 'xx', 'missing <', 2) - self.checkTemplateError('(?P<a>x)', '\g<a a>', 'xx', + self.checkTemplateError('(?P<a>x)', r'\g', 'xx', 'missing <', 2) + self.checkTemplateError('(?P<a>x)', r'\g<a a>', 'xx', "bad character in group name 'a a'", 3) - self.checkTemplateError('(?P<a>x)', '\g<>', 'xx', + self.checkTemplateError('(?P<a>x)', r'\g<>', 'xx', 'missing group name', 3) - self.checkTemplateError('(?P<a>x)', '\g<1a1>', 'xx', + self.checkTemplateError('(?P<a>x)', r'\g<1a1>', 'xx', "bad character in group name '1a1'", 3) self.checkTemplateError('(?P<a>x)', r'\g<2>', 'xx', - 'invalid group reference') + 'invalid group reference 2', 3) self.checkTemplateError('(?P<a>x)', r'\2', 'xx', - 'invalid group reference') + 'invalid group reference 2', 1) with self.assertRaisesRegex(IndexError, "unknown group name 'ab'"): - re.sub('(?P<a>x)', '\g<ab>', 'xx') + re.sub('(?P<a>x)', r'\g<ab>', 'xx') self.assertEqual(re.sub('(?P<a>x)|(?P<b>y)', r'\g<b>', 'xx'), '') self.assertEqual(re.sub('(?P<a>x)|(?P<b>y)', r'\2', 'xx'), '') - self.checkTemplateError('(?P<a>x)', '\g<-1>', 'xx', + self.checkTemplateError('(?P<a>x)', r'\g<-1>', 'xx', "bad character in group name '-1'", 3) # New valid/invalid identifiers in Python 3 self.assertEqual(re.sub('(?P<µ>x)', r'\g<µ>', 'xx'), 'xx') self.assertEqual(re.sub('(?P<𝔘𝔫𝔦𝔠𝔬𝔡𝔢>x)', r'\g<𝔘𝔫𝔦𝔠𝔬𝔡𝔢>', 'xx'), 'xx') - self.checkTemplateError('(?P<a>x)', '\g<©>', 'xx', + self.checkTemplateError('(?P<a>x)', r'\g<©>', 'xx', "bad character in group name '©'", 3) # Support > 100 groups. pat = '|'.join('x(?P<a%d>%x)y' % (i, i) for i in range(1, 200 + 1)) - self.assertEqual(re.sub(pat, '\g<200>', 'xc8yzxc8y'), 'c8zc8') + self.assertEqual(re.sub(pat, r'\g<200>', 'xc8yzxc8y'), 'c8zc8') def test_re_subn(self): self.assertEqual(re.subn("(?i)b+", "x", "bbbb BBBB"), ('x x', 2)) @@ -419,19 +420,77 @@ class ReTests(unittest.TestCase): self.assertEqual(pat.match('bc').groups(), ('b', None, 'b', 'c')) self.assertEqual(pat.match('bc').groups(""), ('b', "", 'b', 'c')) - # A single group - m = re.match('(a)', 'a') - self.assertEqual(m.group(0), 'a') - self.assertEqual(m.group(0), 'a') - self.assertEqual(m.group(1), 'a') - self.assertEqual(m.group(1, 1), ('a', 'a')) - pat = re.compile('(?:(?P<a1>a)|(?P<b2>b))(?P<c3>c)?') self.assertEqual(pat.match('a').group(1, 2, 3), ('a', None, None)) self.assertEqual(pat.match('b').group('a1', 'b2', 'c3'), (None, 'b', None)) self.assertEqual(pat.match('ac').group(1, 'b2', 3), ('a', None, 'c')) + def test_group(self): + class Index: + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + # A single group + m = re.match('(a)(b)', 'ab') + self.assertEqual(m.group(), 'ab') + self.assertEqual(m.group(0), 'ab') + self.assertEqual(m.group(1), 'a') + self.assertEqual(m.group(Index(1)), 'a') + self.assertRaises(IndexError, m.group, -1) + self.assertRaises(IndexError, m.group, 3) + self.assertRaises(IndexError, m.group, 1<<1000) + self.assertRaises(IndexError, m.group, Index(1<<1000)) + self.assertRaises(IndexError, m.group, 'x') + # Multiple groups + self.assertEqual(m.group(2, 1), ('b', 'a')) + self.assertEqual(m.group(Index(2), Index(1)), ('b', 'a')) + + def test_match_getitem(self): + pat = re.compile('(?:(?P<a1>a)|(?P<b2>b))(?P<c3>c)?') + + m = pat.match('a') + self.assertEqual(m['a1'], 'a') + self.assertEqual(m['b2'], None) + self.assertEqual(m['c3'], None) + self.assertEqual('a1={a1} b2={b2} c3={c3}'.format_map(m), 'a1=a b2=None c3=None') + self.assertEqual(m[0], 'a') + self.assertEqual(m[1], 'a') + self.assertEqual(m[2], None) + self.assertEqual(m[3], None) + with self.assertRaisesRegex(IndexError, 'no such group'): + m['X'] + with self.assertRaisesRegex(IndexError, 'no such group'): + m[-1] + with self.assertRaisesRegex(IndexError, 'no such group'): + m[4] + with self.assertRaisesRegex(IndexError, 'no such group'): + m[0, 1] + with self.assertRaisesRegex(IndexError, 'no such group'): + m[(0,)] + with self.assertRaisesRegex(IndexError, 'no such group'): + m[(0, 1)] + with self.assertRaisesRegex(KeyError, 'a2'): + 'a1={a2}'.format_map(m) + + m = pat.match('ac') + self.assertEqual(m['a1'], 'a') + self.assertEqual(m['b2'], None) + self.assertEqual(m['c3'], 'c') + self.assertEqual('a1={a1} b2={b2} c3={c3}'.format_map(m), 'a1=a b2=None c3=c') + self.assertEqual(m[0], 'ac') + self.assertEqual(m[1], 'a') + self.assertEqual(m[2], None) + self.assertEqual(m[3], 'c') + + # Cannot assign. + with self.assertRaises(TypeError): + m[0] = 1 + + # No len(). + self.assertRaises(TypeError, len, m) + def test_re_fullmatch(self): # Issue 16203: Proposal: add re.fullmatch() method. self.assertEqual(re.fullmatch(r"a", "a").span(), (0, 1)) @@ -463,19 +522,19 @@ class ReTests(unittest.TestCase): re.compile(r".*?").fullmatch("abcd", pos=1, endpos=3).span(), (1, 3)) def test_re_groupref_exists(self): - self.assertEqual(re.match('^(\()?([^()]+)(?(1)\))$', '(a)').groups(), + self.assertEqual(re.match(r'^(\()?([^()]+)(?(1)\))$', '(a)').groups(), ('(', 'a')) - self.assertEqual(re.match('^(\()?([^()]+)(?(1)\))$', 'a').groups(), + self.assertEqual(re.match(r'^(\()?([^()]+)(?(1)\))$', 'a').groups(), (None, 'a')) - self.assertIsNone(re.match('^(\()?([^()]+)(?(1)\))$', 'a)')) - self.assertIsNone(re.match('^(\()?([^()]+)(?(1)\))$', '(a')) + self.assertIsNone(re.match(r'^(\()?([^()]+)(?(1)\))$', 'a)')) + self.assertIsNone(re.match(r'^(\()?([^()]+)(?(1)\))$', '(a')) self.assertEqual(re.match('^(?:(a)|c)((?(1)b|d))$', 'ab').groups(), ('a', 'b')) - self.assertEqual(re.match('^(?:(a)|c)((?(1)b|d))$', 'cd').groups(), + self.assertEqual(re.match(r'^(?:(a)|c)((?(1)b|d))$', 'cd').groups(), (None, 'd')) - self.assertEqual(re.match('^(?:(a)|c)((?(1)|d))$', 'cd').groups(), + self.assertEqual(re.match(r'^(?:(a)|c)((?(1)|d))$', 'cd').groups(), (None, 'd')) - self.assertEqual(re.match('^(?:(a)|c)((?(1)|d))$', 'a').groups(), + self.assertEqual(re.match(r'^(?:(a)|c)((?(1)|d))$', 'a').groups(), ('a', '')) # Tests for bug #1177831: exercise groups other than the first group @@ -500,10 +559,11 @@ class ReTests(unittest.TestCase): 'two branches', 10) def test_re_groupref_overflow(self): - self.checkTemplateError('()', '\g<%s>' % sre_constants.MAXGROUPS, 'xx', - 'invalid group reference', 3) - self.checkPatternError(r'(?P<a>)(?(%d))' % sre_constants.MAXGROUPS, - 'invalid group reference', 10) + from sre_constants import MAXGROUPS + self.checkTemplateError('()', r'\g<%s>' % MAXGROUPS, 'xx', + 'invalid group reference %d' % MAXGROUPS, 3) + self.checkPatternError(r'(?P<a>)(?(%d))' % MAXGROUPS, + 'invalid group reference %d' % MAXGROUPS, 10) def test_re_groupref(self): self.assertEqual(re.match(r'^(\|)?([^()]+)\1$', '|a|').groups(), @@ -535,37 +595,37 @@ class ReTests(unittest.TestCase): " ") def test_repeat_minmax(self): - self.assertIsNone(re.match("^(\w){1}$", "abc")) - self.assertIsNone(re.match("^(\w){1}?$", "abc")) - self.assertIsNone(re.match("^(\w){1,2}$", "abc")) - self.assertIsNone(re.match("^(\w){1,2}?$", "abc")) - - self.assertEqual(re.match("^(\w){3}$", "abc").group(1), "c") - self.assertEqual(re.match("^(\w){1,3}$", "abc").group(1), "c") - self.assertEqual(re.match("^(\w){1,4}$", "abc").group(1), "c") - self.assertEqual(re.match("^(\w){3,4}?$", "abc").group(1), "c") - self.assertEqual(re.match("^(\w){3}?$", "abc").group(1), "c") - self.assertEqual(re.match("^(\w){1,3}?$", "abc").group(1), "c") - self.assertEqual(re.match("^(\w){1,4}?$", "abc").group(1), "c") - self.assertEqual(re.match("^(\w){3,4}?$", "abc").group(1), "c") - - self.assertIsNone(re.match("^x{1}$", "xxx")) - self.assertIsNone(re.match("^x{1}?$", "xxx")) - self.assertIsNone(re.match("^x{1,2}$", "xxx")) - self.assertIsNone(re.match("^x{1,2}?$", "xxx")) - - self.assertTrue(re.match("^x{3}$", "xxx")) - self.assertTrue(re.match("^x{1,3}$", "xxx")) - self.assertTrue(re.match("^x{3,3}$", "xxx")) - self.assertTrue(re.match("^x{1,4}$", "xxx")) - self.assertTrue(re.match("^x{3,4}?$", "xxx")) - self.assertTrue(re.match("^x{3}?$", "xxx")) - self.assertTrue(re.match("^x{1,3}?$", "xxx")) - self.assertTrue(re.match("^x{1,4}?$", "xxx")) - self.assertTrue(re.match("^x{3,4}?$", "xxx")) - - self.assertIsNone(re.match("^x{}$", "xxx")) - self.assertTrue(re.match("^x{}$", "x{}")) + self.assertIsNone(re.match(r"^(\w){1}$", "abc")) + self.assertIsNone(re.match(r"^(\w){1}?$", "abc")) + self.assertIsNone(re.match(r"^(\w){1,2}$", "abc")) + self.assertIsNone(re.match(r"^(\w){1,2}?$", "abc")) + + self.assertEqual(re.match(r"^(\w){3}$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){1,3}$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){1,4}$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){3,4}?$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){3}?$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){1,3}?$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){1,4}?$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){3,4}?$", "abc").group(1), "c") + + self.assertIsNone(re.match(r"^x{1}$", "xxx")) + self.assertIsNone(re.match(r"^x{1}?$", "xxx")) + self.assertIsNone(re.match(r"^x{1,2}$", "xxx")) + self.assertIsNone(re.match(r"^x{1,2}?$", "xxx")) + + self.assertTrue(re.match(r"^x{3}$", "xxx")) + self.assertTrue(re.match(r"^x{1,3}$", "xxx")) + self.assertTrue(re.match(r"^x{3,3}$", "xxx")) + self.assertTrue(re.match(r"^x{1,4}$", "xxx")) + self.assertTrue(re.match(r"^x{3,4}?$", "xxx")) + self.assertTrue(re.match(r"^x{3}?$", "xxx")) + self.assertTrue(re.match(r"^x{1,3}?$", "xxx")) + self.assertTrue(re.match(r"^x{1,4}?$", "xxx")) + self.assertTrue(re.match(r"^x{3,4}?$", "xxx")) + + self.assertIsNone(re.match(r"^x{}$", "xxx")) + self.assertTrue(re.match(r"^x{}$", "x{}")) self.checkPatternError(r'x{2,1}', 'min repeat greater than max repeat', 2) @@ -638,14 +698,10 @@ class ReTests(unittest.TestCase): re.purge() # for warnings for c in 'ceghijklmopqyzCEFGHIJKLMNOPQRTVXY': with self.subTest(c): - with self.assertWarns(DeprecationWarning): - self.assertEqual(re.fullmatch('\\%c' % c, c).group(), c) - self.assertIsNone(re.match('\\%c' % c, 'a')) + self.assertRaises(re.error, re.compile, '\\%c' % c) for c in 'ceghijklmopqyzABCEFGHIJKLMNOPQRTVXYZ': with self.subTest(c): - with self.assertWarns(DeprecationWarning): - self.assertEqual(re.fullmatch('[\\%c]' % c, c).group(), c) - self.assertIsNone(re.match('[\\%c]' % c, 'a')) + self.assertRaises(re.error, re.compile, '[\\%c]' % c) def test_string_boundaries(self): # See http://bugs.python.org/issue10713 @@ -692,10 +748,10 @@ class ReTests(unittest.TestCase): "a\n\nb") def test_lookahead(self): - self.assertEqual(re.match("(a(?=\s[^a]))", "a b").group(1), "a") - self.assertEqual(re.match("(a(?=\s[^a]*))", "a b").group(1), "a") - self.assertEqual(re.match("(a(?=\s[abc]))", "a b").group(1), "a") - self.assertEqual(re.match("(a(?=\s[abc]*))", "a bc").group(1), "a") + self.assertEqual(re.match(r"(a(?=\s[^a]))", "a b").group(1), "a") + self.assertEqual(re.match(r"(a(?=\s[^a]*))", "a b").group(1), "a") + self.assertEqual(re.match(r"(a(?=\s[abc]))", "a b").group(1), "a") + self.assertEqual(re.match(r"(a(?=\s[abc]*))", "a bc").group(1), "a") self.assertEqual(re.match(r"(a)(?=\s\1)", "a a").group(1), "a") self.assertEqual(re.match(r"(a)(?=\s\1*)", "a aa").group(1), "a") self.assertEqual(re.match(r"(a)(?=\s(abc|a))", "a a").group(1), "a") @@ -843,12 +899,12 @@ class ReTests(unittest.TestCase): self.assertEqual(re.match(b"abc", b"ABC", re.I|re.L).group(0), b"ABC") def test_not_literal(self): - self.assertEqual(re.search("\s([^a])", " b").group(1), "b") - self.assertEqual(re.search("\s([^a]*)", " bb").group(1), "bb") + self.assertEqual(re.search(r"\s([^a])", " b").group(1), "b") + self.assertEqual(re.search(r"\s([^a]*)", " bb").group(1), "bb") def test_search_coverage(self): - self.assertEqual(re.search("\s(b)", " b").group(1), "b") - self.assertEqual(re.search("a\s", "a ").group(0), "a ") + self.assertEqual(re.search(r"\s(b)", " b").group(1), "b") + self.assertEqual(re.search(r"a\s", "a ").group(0), "a ") def assertMatch(self, pattern, text, match=None, span=None, matcher=re.match): @@ -953,7 +1009,7 @@ class ReTests(unittest.TestCase): self.checkPatternError(r"\567", r'octal escape value \567 outside of ' r'range 0-0o377', 0) - self.checkPatternError(r"\911", 'invalid group reference', 0) + self.checkPatternError(r"\911", 'invalid group reference 91', 1) self.checkPatternError(r"\x1", r'incomplete escape \x1', 0) self.checkPatternError(r"\x1z", r'incomplete escape \x1', 0) self.checkPatternError(r"\u123", r'incomplete escape \u123', 0) @@ -998,10 +1054,8 @@ class ReTests(unittest.TestCase): self.assertTrue(re.match((r"\x%02x" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"\x%02x0" % i).encode(), bytes([i])+b"0")) self.assertTrue(re.match((r"\x%02xz" % i).encode(), bytes([i])+b"z")) - with self.assertWarns(DeprecationWarning): - self.assertTrue(re.match(br"\u1234", b'u1234')) - with self.assertWarns(DeprecationWarning): - self.assertTrue(re.match(br"\U00012345", b'U00012345')) + self.assertRaises(re.error, re.compile, br"\u1234") + self.assertRaises(re.error, re.compile, br"\U00012345") self.assertTrue(re.match(br"\0", b"\000")) self.assertTrue(re.match(br"\08", b"\0008")) self.assertTrue(re.match(br"\01", b"\001")) @@ -1009,7 +1063,7 @@ class ReTests(unittest.TestCase): self.checkPatternError(br"\567", r'octal escape value \567 outside of ' r'range 0-0o377', 0) - self.checkPatternError(br"\911", 'invalid group reference', 0) + self.checkPatternError(br"\911", 'invalid group reference 91', 1) self.checkPatternError(br"\x1", r'incomplete escape \x1', 0) self.checkPatternError(br"\x1z", r'incomplete escape \x1', 0) @@ -1023,10 +1077,8 @@ class ReTests(unittest.TestCase): self.assertTrue(re.match((r"[\x%02x]" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"[\x%02x0]" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"[\x%02xz]" % i).encode(), bytes([i]))) - with self.assertWarns(DeprecationWarning): - self.assertTrue(re.match(br"[\u1234]", b'u')) - with self.assertWarns(DeprecationWarning): - self.assertTrue(re.match(br"[\U00012345]", b'U')) + self.assertRaises(re.error, re.compile, br"[\u1234]") + self.assertRaises(re.error, re.compile, br"[\U00012345]") self.checkPatternError(br"[\567]", r'octal escape value \567 outside of ' r'range 0-0o377', 1) @@ -1054,8 +1106,8 @@ class ReTests(unittest.TestCase): self.assertIsNone(re.match(r'(a)?a','a').lastindex) self.assertEqual(re.match(r'(a)(b)?b','ab').lastindex, 1) self.assertEqual(re.match(r'(?P<a>a)(?P<b>b)?b','ab').lastgroup, 'a') - self.assertEqual(re.match("(?P<a>a(b))", "ab").lastgroup, 'a') - self.assertEqual(re.match("((a))", "a").lastindex, 1) + self.assertEqual(re.match(r"(?P<a>a(b))", "ab").lastgroup, 'a') + self.assertEqual(re.match(r"((a))", "a").lastindex, 1) def test_bug_418626(self): # bugs 418626 at al. -- Testing Greg Chapman's addition of op code @@ -1227,7 +1279,7 @@ class ReTests(unittest.TestCase): '\uff10', # '\N{FULLWIDTH DIGIT ZERO}', category 'Nd' ] for x in decimal_digits: - self.assertEqual(re.match('^\d$', x).group(0), x) + self.assertEqual(re.match(r'^\d$', x).group(0), x) not_decimal_digits = [ '\u2165', # '\N{ROMAN NUMERAL SIX}', category 'Nl' @@ -1236,7 +1288,7 @@ class ReTests(unittest.TestCase): '\u32b4', # '\N{CIRCLED NUMBER THIRTY NINE}', category 'No' ] for x in not_decimal_digits: - self.assertIsNone(re.match('^\d$', x)) + self.assertIsNone(re.match(r'^\d$', x)) def test_empty_array(self): # SF buf 1647541 @@ -1278,6 +1330,22 @@ class ReTests(unittest.TestCase): self.assertTrue(re.match('(?ixu) ' + upper_char, lower_char)) self.assertTrue(re.match('(?ixu) ' + lower_char, upper_char)) + p = upper_char + '(?i)' + with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.match(p, lower_char)) + self.assertEqual( + str(warns.warnings[0].message), + 'Flags not at the start of the expression %s' % p + ) + + p = upper_char + '(?i)%s' % ('.?' * 100) + with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.match(p, lower_char)) + self.assertEqual( + str(warns.warnings[0].message), + 'Flags not at the start of the expression %s (truncated)' % p[:20] + ) + def test_dollar_matches_twice(self): "$ matches the end of string, and just before the terminating \n" pattern = re.compile('$') @@ -1308,29 +1376,29 @@ class ReTests(unittest.TestCase): for flags in (0, re.UNICODE): pat = re.compile('\xc0', flags | re.IGNORECASE) self.assertTrue(pat.match('\xe0')) - pat = re.compile('\w', flags) + pat = re.compile(r'\w', flags) self.assertTrue(pat.match('\xe0')) pat = re.compile('\xc0', re.ASCII | re.IGNORECASE) self.assertIsNone(pat.match('\xe0')) pat = re.compile('(?a)\xc0', re.IGNORECASE) self.assertIsNone(pat.match('\xe0')) - pat = re.compile('\w', re.ASCII) + pat = re.compile(r'\w', re.ASCII) self.assertIsNone(pat.match('\xe0')) - pat = re.compile('(?a)\w') + pat = re.compile(r'(?a)\w') self.assertIsNone(pat.match('\xe0')) # Bytes patterns for flags in (0, re.ASCII): pat = re.compile(b'\xc0', flags | re.IGNORECASE) self.assertIsNone(pat.match(b'\xe0')) - pat = re.compile(b'\w', flags) + pat = re.compile(br'\w', flags) self.assertIsNone(pat.match(b'\xe0')) # Incompatibilities - self.assertRaises(ValueError, re.compile, b'\w', re.UNICODE) - self.assertRaises(ValueError, re.compile, b'(?u)\w') - self.assertRaises(ValueError, re.compile, '\w', re.UNICODE | re.ASCII) - self.assertRaises(ValueError, re.compile, '(?u)\w', re.ASCII) - self.assertRaises(ValueError, re.compile, '(?a)\w', re.UNICODE) - self.assertRaises(ValueError, re.compile, '(?au)\w') + self.assertRaises(ValueError, re.compile, br'\w', re.UNICODE) + self.assertRaises(ValueError, re.compile, br'(?u)\w') + self.assertRaises(ValueError, re.compile, r'\w', re.UNICODE | re.ASCII) + self.assertRaises(ValueError, re.compile, r'(?u)\w', re.ASCII) + self.assertRaises(ValueError, re.compile, r'(?a)\w', re.UNICODE) + self.assertRaises(ValueError, re.compile, r'(?au)\w') def test_locale_flag(self): import locale @@ -1361,27 +1429,59 @@ class ReTests(unittest.TestCase): pat = re.compile(bpat, re.IGNORECASE) if bletter: self.assertIsNone(pat.match(bletter)) - pat = re.compile(b'\w', re.LOCALE) + pat = re.compile(br'\w', re.LOCALE) if bletter: self.assertTrue(pat.match(bletter)) - pat = re.compile(b'(?L)\w') + pat = re.compile(br'(?L)\w') if bletter: self.assertTrue(pat.match(bletter)) - pat = re.compile(b'\w') + pat = re.compile(br'\w') if bletter: self.assertIsNone(pat.match(bletter)) # Incompatibilities - self.assertWarns(DeprecationWarning, re.compile, '', re.LOCALE) - self.assertWarns(DeprecationWarning, re.compile, '(?L)') - self.assertWarns(DeprecationWarning, re.compile, b'', re.LOCALE | re.ASCII) - self.assertWarns(DeprecationWarning, re.compile, b'(?L)', re.ASCII) - self.assertWarns(DeprecationWarning, re.compile, b'(?a)', re.LOCALE) - self.assertWarns(DeprecationWarning, re.compile, b'(?aL)') + self.assertRaises(ValueError, re.compile, '', re.LOCALE) + self.assertRaises(ValueError, re.compile, '(?L)') + self.assertRaises(ValueError, re.compile, b'', re.LOCALE | re.ASCII) + self.assertRaises(ValueError, re.compile, b'(?L)', re.ASCII) + self.assertRaises(ValueError, re.compile, b'(?a)', re.LOCALE) + self.assertRaises(ValueError, re.compile, b'(?aL)') + + def test_scoped_flags(self): + self.assertTrue(re.match(r'(?i:a)b', 'Ab')) + self.assertIsNone(re.match(r'(?i:a)b', 'aB')) + self.assertIsNone(re.match(r'(?-i:a)b', 'Ab', re.IGNORECASE)) + self.assertTrue(re.match(r'(?-i:a)b', 'aB', re.IGNORECASE)) + self.assertIsNone(re.match(r'(?i:(?-i:a)b)', 'Ab')) + self.assertTrue(re.match(r'(?i:(?-i:a)b)', 'aB')) + + self.assertTrue(re.match(r'(?x: a) b', 'a b')) + self.assertIsNone(re.match(r'(?x: a) b', ' a b')) + self.assertTrue(re.match(r'(?-x: a) b', ' ab', re.VERBOSE)) + self.assertIsNone(re.match(r'(?-x: a) b', 'ab', re.VERBOSE)) + + self.checkPatternError(r'(?a:\w)', + 'bad inline flags: cannot turn on global flag', 3) + self.checkPatternError(r'(?a)(?-a:\w)', + 'bad inline flags: cannot turn off global flag', 8) + self.checkPatternError(r'(?i-i:a)', + 'bad inline flags: flag turned on and off', 5) + + self.checkPatternError(r'(?-', 'missing flag', 3) + self.checkPatternError(r'(?-+', 'missing flag', 3) + self.checkPatternError(r'(?-z', 'unknown flag', 3) + self.checkPatternError(r'(?-i', 'missing :', 4) + self.checkPatternError(r'(?-i)', 'missing :', 4) + self.checkPatternError(r'(?-i+', 'missing :', 4) + self.checkPatternError(r'(?-iz', 'unknown flag', 4) + self.checkPatternError(r'(?i:', 'missing ), unterminated subpattern', 0) + self.checkPatternError(r'(?i', 'missing -, : or )', 3) + self.checkPatternError(r'(?i+', 'missing -, : or )', 3) + self.checkPatternError(r'(?iz', 'unknown flag', 3) def test_bug_6509(self): # Replacement strings of both types must parse properly. # all strings - pat = re.compile('a(\w)') + pat = re.compile(r'a(\w)') self.assertEqual(pat.sub('b\\1', 'ac'), 'bc') pat = re.compile('a(.)') self.assertEqual(pat.sub('b\\1', 'a\u1234'), 'b\u1234') @@ -1389,7 +1489,7 @@ class ReTests(unittest.TestCase): self.assertEqual(pat.sub(lambda m: 'str', 'a5'), 'str') # all bytes - pat = re.compile(b'a(\w)') + pat = re.compile(br'a(\w)') self.assertEqual(pat.sub(b'b\\1', b'ac'), b'bc') pat = re.compile(b'a(.)') self.assertEqual(pat.sub(b'b\\1', b'a\xCD'), b'b\xCD') @@ -1427,13 +1527,6 @@ class ReTests(unittest.TestCase): # Test behaviour when not given a string or pattern as parameter self.assertRaises(TypeError, re.compile, 0) - def test_bug_13899(self): - # Issue #13899: re pattern r"[\A]" should work like "A" but matches - # nothing. Ditto B and Z. - with self.assertWarns(DeprecationWarning): - self.assertEqual(re.findall(r'[\A\B\b\C\Z]', 'AB\bCZ'), - ['A', 'B', '\b', 'C', 'Z']) - @bigmemtest(size=_2G, memuse=1) def test_large_search(self, size): # Issue #10182: indices were 32-bit-truncated. @@ -1518,7 +1611,7 @@ class ReTests(unittest.TestCase): for string in (b'[abracadabra]', B(b'[abracadabra]'), bytearray(b'[abracadabra]'), memoryview(b'[abracadabra]')): - m = re.search(rb'(.+)(.*?)\1', string) + m = re.search(br'(.+)(.*?)\1', string) self.assertEqual(repr(m), "<%s.%s object; " "span=(1, 12), match=b'abracadabra'>" % (type(m).__module__, type(m).__qualname__)) @@ -1547,9 +1640,9 @@ class ReTests(unittest.TestCase): with captured_stdout() as out: re.compile(pat, re.DEBUG) dump = '''\ -SUBPATTERN 1 +SUBPATTERN 1 0 0 LITERAL 46 -SUBPATTERN None +SUBPATTERN None 0 0 BRANCH IN LITERAL 99 @@ -1557,7 +1650,7 @@ SUBPATTERN None OR LITERAL 112 LITERAL 121 -SUBPATTERN None +SUBPATTERN None 0 0 GROUPREF_EXISTS 1 AT AT_END ELSE @@ -1673,12 +1766,64 @@ SUBPATTERN None self.checkPatternError(r'(?P', 'unexpected end of pattern', 3) self.checkPatternError(r'(?z)', 'unknown extension ?z', 1) self.checkPatternError(r'(?iz)', 'unknown flag', 3) - self.checkPatternError(r'(?i', 'missing )', 3) + self.checkPatternError(r'(?i', 'missing -, : or )', 3) self.checkPatternError(r'(?#abc', 'missing ), unterminated comment', 0) self.checkPatternError(r'(?<', 'unexpected end of pattern', 3) self.checkPatternError(r'(?<>)', 'unknown extension ?<>', 1) self.checkPatternError(r'(?', 'unexpected end of pattern', 2) + def test_enum(self): + # Issue #28082: Check that str(flag) returns a human readable string + # instead of an integer + self.assertIn('ASCII', str(re.A)) + self.assertIn('DOTALL', str(re.S)) + + def test_pattern_compare(self): + pattern1 = re.compile('abc', re.IGNORECASE) + + # equal to itself + self.assertEqual(pattern1, pattern1) + self.assertFalse(pattern1 != pattern1) + + # equal + re.purge() + pattern2 = re.compile('abc', re.IGNORECASE) + self.assertEqual(hash(pattern2), hash(pattern1)) + self.assertEqual(pattern2, pattern1) + + # not equal: different pattern + re.purge() + pattern3 = re.compile('XYZ', re.IGNORECASE) + # Don't test hash(pattern3) != hash(pattern1) because there is no + # warranty that hash values are different + self.assertNotEqual(pattern3, pattern1) + + # not equal: different flag (flags=0) + re.purge() + pattern4 = re.compile('abc') + self.assertNotEqual(pattern4, pattern1) + + # only == and != comparison operators are supported + with self.assertRaises(TypeError): + pattern1 < pattern2 + + def test_pattern_compare_bytes(self): + pattern1 = re.compile(b'abc') + + # equal: test bytes patterns + re.purge() + pattern2 = re.compile(b'abc') + self.assertEqual(hash(pattern2), hash(pattern1)) + self.assertEqual(pattern2, pattern1) + + # not equal: pattern of a different types (str vs bytes), + # comparison must not raise a BytesWarning + re.purge() + pattern3 = re.compile('abc') + with warnings.catch_warnings(): + warnings.simplefilter('error', BytesWarning) + self.assertNotEqual(pattern3, pattern1) + def test_bug_29444(self): s = bytearray(b'abcdefgh') m = re.search(b'[a-h]+', s) diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 2c73df291f..06a9149374 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -121,6 +121,21 @@ class TestReadline(unittest.TestCase): TERM='xterm-256color') self.assertEqual(stdout, b'') + auto_history_script = """\ +import readline +readline.set_auto_history({}) +input() +print("History length:", readline.get_current_history_length()) +""" + + def test_auto_history_enabled(self): + output = run_pty(self.auto_history_script.format(True)) + self.assertIn(b"History length: 1\r\n", output) + + def test_auto_history_disabled(self): + output = run_pty(self.auto_history_script.format(False)) + self.assertIn(b"History length: 0\r\n", output) + def test_nonascii(self): try: readline.add_history("\xEB\xEF") diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e2c2f2da86..751df1537d 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1,21 +1,48 @@ """ Tests of regrtest.py. + +Note: test_regrtest cannot be run twice in parallel. """ -import argparse +import contextlib import faulthandler -import getopt +import io import os.path +import platform +import re +import subprocess +import sys +import sysconfig +import tempfile +import textwrap import unittest -from test import regrtest, support +from test import libregrtest +from test import support -class ParseArgsTestCase(unittest.TestCase): - """Test regrtest's argument parsing.""" +Py_DEBUG = hasattr(sys, 'getobjects') +ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') +ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) + +TEST_INTERRUPTED = textwrap.dedent(""" + from signal import SIGINT + try: + from _testcapi import raise_signal + raise_signal(SIGINT) + except ImportError: + import os + os.kill(os.getpid(), SIGINT) + """) + + +class ParseArgsTestCase(unittest.TestCase): + """ + Test regrtest's argument parsing, function _parse_args(). + """ def checkError(self, args, msg): with support.captured_stderr() as err, self.assertRaises(SystemExit): - regrtest._parse_args(args) + libregrtest._parse_args(args) self.assertIn(msg, err.getvalue()) def test_help(self): @@ -23,82 +50,82 @@ class ParseArgsTestCase(unittest.TestCase): with self.subTest(opt=opt): with support.captured_stdout() as out, \ self.assertRaises(SystemExit): - regrtest._parse_args([opt]) + libregrtest._parse_args([opt]) self.assertIn('Run Python regression tests.', out.getvalue()) @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'), "faulthandler.dump_traceback_later() required") def test_timeout(self): - ns = regrtest._parse_args(['--timeout', '4.2']) + ns = libregrtest._parse_args(['--timeout', '4.2']) self.assertEqual(ns.timeout, 4.2) self.checkError(['--timeout'], 'expected one argument') self.checkError(['--timeout', 'foo'], 'invalid float value') def test_wait(self): - ns = regrtest._parse_args(['--wait']) + ns = libregrtest._parse_args(['--wait']) self.assertTrue(ns.wait) def test_slaveargs(self): - ns = regrtest._parse_args(['--slaveargs', '[[], {}]']) + ns = libregrtest._parse_args(['--slaveargs', '[[], {}]']) self.assertEqual(ns.slaveargs, '[[], {}]') self.checkError(['--slaveargs'], 'expected one argument') def test_start(self): for opt in '-S', '--start': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'foo']) + ns = libregrtest._parse_args([opt, 'foo']) self.assertEqual(ns.start, 'foo') self.checkError([opt], 'expected one argument') def test_verbose(self): - ns = regrtest._parse_args(['-v']) + ns = libregrtest._parse_args(['-v']) self.assertEqual(ns.verbose, 1) - ns = regrtest._parse_args(['-vvv']) + ns = libregrtest._parse_args(['-vvv']) self.assertEqual(ns.verbose, 3) - ns = regrtest._parse_args(['--verbose']) + ns = libregrtest._parse_args(['--verbose']) self.assertEqual(ns.verbose, 1) - ns = regrtest._parse_args(['--verbose'] * 3) + ns = libregrtest._parse_args(['--verbose'] * 3) self.assertEqual(ns.verbose, 3) - ns = regrtest._parse_args([]) + ns = libregrtest._parse_args([]) self.assertEqual(ns.verbose, 0) def test_verbose2(self): for opt in '-w', '--verbose2': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.verbose2) def test_verbose3(self): for opt in '-W', '--verbose3': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.verbose3) def test_quiet(self): for opt in '-q', '--quiet': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) def test_slow(self): - for opt in '-o', '--slow': + for opt in '-o', '--slowest': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.print_slow) def test_header(self): - ns = regrtest._parse_args(['--header']) + ns = libregrtest._parse_args(['--header']) self.assertTrue(ns.header) def test_randomize(self): for opt in '-r', '--randomize': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.randomize) def test_randseed(self): - ns = regrtest._parse_args(['--randseed', '12345']) + ns = libregrtest._parse_args(['--randseed', '12345']) self.assertEqual(ns.random_seed, 12345) self.assertTrue(ns.randomize) self.checkError(['--randseed'], 'expected one argument') @@ -107,7 +134,7 @@ class ParseArgsTestCase(unittest.TestCase): def test_fromfile(self): for opt in '-f', '--fromfile': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'foo']) + ns = libregrtest._parse_args([opt, 'foo']) self.assertEqual(ns.fromfile, 'foo') self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo', '-s'], "don't go together") @@ -115,42 +142,42 @@ class ParseArgsTestCase(unittest.TestCase): def test_exclude(self): for opt in '-x', '--exclude': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.exclude) def test_single(self): for opt in '-s', '--single': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.single) self.checkError([opt, '-f', 'foo'], "don't go together") def test_match(self): for opt in '-m', '--match': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'pattern']) + ns = libregrtest._parse_args([opt, 'pattern']) self.assertEqual(ns.match_tests, 'pattern') self.checkError([opt], 'expected one argument') def test_failfast(self): for opt in '-G', '--failfast': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '-v']) + ns = libregrtest._parse_args([opt, '-v']) self.assertTrue(ns.failfast) - ns = regrtest._parse_args([opt, '-W']) + ns = libregrtest._parse_args([opt, '-W']) self.assertTrue(ns.failfast) self.checkError([opt], '-G/--failfast needs either -v or -W') def test_use(self): for opt in '-u', '--use': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'gui,network']) + ns = libregrtest._parse_args([opt, 'gui,network']) self.assertEqual(ns.use_resources, ['gui', 'network']) - ns = regrtest._parse_args([opt, 'gui,none,network']) + ns = libregrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(regrtest.RESOURCE_NAMES) + expected = list(libregrtest.RESOURCE_NAMES) expected.remove('gui') - ns = regrtest._parse_args([opt, 'all,-gui']) + ns = libregrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid resource') @@ -158,31 +185,31 @@ class ParseArgsTestCase(unittest.TestCase): def test_memlimit(self): for opt in '-M', '--memlimit': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '4G']) + ns = libregrtest._parse_args([opt, '4G']) self.assertEqual(ns.memlimit, '4G') self.checkError([opt], 'expected one argument') def test_testdir(self): - ns = regrtest._parse_args(['--testdir', 'foo']) + ns = libregrtest._parse_args(['--testdir', 'foo']) self.assertEqual(ns.testdir, os.path.join(support.SAVEDCWD, 'foo')) self.checkError(['--testdir'], 'expected one argument') def test_runleaks(self): for opt in '-L', '--runleaks': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.runleaks) def test_huntrleaks(self): for opt in '-R', '--huntrleaks': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, ':']) + ns = libregrtest._parse_args([opt, ':']) self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt')) - ns = regrtest._parse_args([opt, '6:']) + ns = libregrtest._parse_args([opt, '6:']) self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt')) - ns = regrtest._parse_args([opt, ':3']) + ns = libregrtest._parse_args([opt, ':3']) self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt')) - ns = regrtest._parse_args([opt, '6:3:leaks.log']) + ns = libregrtest._parse_args([opt, '6:3:leaks.log']) self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log')) self.checkError([opt], 'expected one argument') self.checkError([opt, '6'], @@ -193,27 +220,25 @@ class ParseArgsTestCase(unittest.TestCase): def test_multiprocess(self): for opt in '-j', '--multiprocess': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '2']) + ns = libregrtest._parse_args([opt, '2']) self.assertEqual(ns.use_mp, 2) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') self.checkError([opt, '2', '-T'], "don't go together") self.checkError([opt, '2', '-l'], "don't go together") - self.checkError([opt, '2', '-M', '4G'], "don't go together") self.checkError([opt, '0', '-T'], "don't go together") self.checkError([opt, '0', '-l'], "don't go together") - self.checkError([opt, '0', '-M', '4G'], "don't go together") def test_coverage(self): for opt in '-T', '--coverage': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.trace) def test_coverdir(self): for opt in '-D', '--coverdir': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'foo']) + ns = libregrtest._parse_args([opt, 'foo']) self.assertEqual(ns.coverdir, os.path.join(support.SAVEDCWD, 'foo')) self.checkError([opt], 'expected one argument') @@ -221,13 +246,13 @@ class ParseArgsTestCase(unittest.TestCase): def test_nocoverdir(self): for opt in '-N', '--nocoverdir': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertIsNone(ns.coverdir) def test_threshold(self): for opt in '-t', '--threshold': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '1000']) + ns = libregrtest._parse_args([opt, '1000']) self.assertEqual(ns.threshold, 1000) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') @@ -235,13 +260,16 @@ class ParseArgsTestCase(unittest.TestCase): def test_nowindows(self): for opt in '-n', '--nowindows': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + with contextlib.redirect_stderr(io.StringIO()) as stderr: + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.nowindows) + err = stderr.getvalue() + self.assertIn('the --nowindows (-n) option is deprecated', err) def test_forever(self): for opt in '-F', '--forever': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.forever) @@ -249,32 +277,32 @@ class ParseArgsTestCase(unittest.TestCase): self.checkError(['--xxx'], 'usage:') def test_long_option__partial(self): - ns = regrtest._parse_args(['--qui']) + ns = libregrtest._parse_args(['--qui']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) def test_two_options(self): - ns = regrtest._parse_args(['--quiet', '--exclude']) + ns = libregrtest._parse_args(['--quiet', '--exclude']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) self.assertTrue(ns.exclude) def test_option_with_empty_string_value(self): - ns = regrtest._parse_args(['--start', '']) + ns = libregrtest._parse_args(['--start', '']) self.assertEqual(ns.start, '') def test_arg(self): - ns = regrtest._parse_args(['foo']) + ns = libregrtest._parse_args(['foo']) self.assertEqual(ns.args, ['foo']) def test_option_and_arg(self): - ns = regrtest._parse_args(['--quiet', 'foo']) + ns = libregrtest._parse_args(['--quiet', 'foo']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) self.assertEqual(ns.args, ['foo']) def test_arg_option_arg(self): - ns = regrtest._parse_args(['test_unaryop', '-v', 'test_binop']) + ns = libregrtest._parse_args(['test_unaryop', '-v', 'test_binop']) self.assertEqual(ns.verbose, 1) self.assertEqual(ns.args, ['test_unaryop', 'test_binop']) @@ -283,6 +311,496 @@ class ParseArgsTestCase(unittest.TestCase): 'unrecognized arguments: --unknown-option') +class BaseTestCase(unittest.TestCase): + TEST_UNIQUE_ID = 1 + TESTNAME_PREFIX = 'test_regrtest_' + TESTNAME_REGEX = r'test_[a-zA-Z0-9_]+' + + def setUp(self): + self.testdir = os.path.realpath(os.path.dirname(__file__)) + + self.tmptestdir = tempfile.mkdtemp() + self.addCleanup(support.rmtree, self.tmptestdir) + + def create_test(self, name=None, code=''): + if not name: + name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID + BaseTestCase.TEST_UNIQUE_ID += 1 + + # test_regrtest cannot be run twice in parallel because + # of setUp() and create_test() + name = self.TESTNAME_PREFIX + name + path = os.path.join(self.tmptestdir, name + '.py') + + self.addCleanup(support.unlink, path) + # Use 'x' mode to ensure that we do not override existing tests + try: + with open(path, 'x', encoding='utf-8') as fp: + fp.write(code) + except PermissionError as exc: + if not sysconfig.is_python_build(): + self.skipTest("cannot write %s: %s" % (path, exc)) + raise + return name + + def regex_search(self, regex, output): + match = re.search(regex, output, re.MULTILINE) + if not match: + self.fail("%r not found in %r" % (regex, output)) + return match + + def check_line(self, output, regex): + regex = re.compile(r'^' + regex, re.MULTILINE) + self.assertRegex(output, regex) + + def parse_executed_tests(self, output): + regex = (r'^[0-9]+:[0-9]+:[0-9]+ \[ *[0-9]+(?:/ *[0-9]+)?\] (%s)' + % self.TESTNAME_REGEX) + parser = re.finditer(regex, output, re.MULTILINE) + return list(match.group(1) for match in parser) + + def check_executed_tests(self, output, tests, skipped=(), failed=(), + omitted=(), randomize=False, interrupted=False): + if isinstance(tests, str): + tests = [tests] + if isinstance(skipped, str): + skipped = [skipped] + if isinstance(failed, str): + failed = [failed] + if isinstance(omitted, str): + omitted = [omitted] + ntest = len(tests) + nskipped = len(skipped) + nfailed = len(failed) + nomitted = len(omitted) + + executed = self.parse_executed_tests(output) + if randomize: + self.assertEqual(set(executed), set(tests), output) + else: + self.assertEqual(executed, tests, output) + + def plural(count): + return 's' if count != 1 else '' + + def list_regex(line_format, tests): + count = len(tests) + names = ' '.join(sorted(tests)) + regex = line_format % (count, plural(count)) + regex = r'%s:\n %s$' % (regex, names) + return regex + + if skipped: + regex = list_regex('%s test%s skipped', skipped) + self.check_line(output, regex) + + if failed: + regex = list_regex('%s test%s failed', failed) + self.check_line(output, regex) + + if omitted: + regex = list_regex('%s test%s omitted', omitted) + self.check_line(output, regex) + + good = ntest - nskipped - nfailed - nomitted + if good: + regex = r'%s test%s OK\.$' % (good, plural(good)) + if not skipped and not failed and good > 1: + regex = 'All %s' % regex + self.check_line(output, regex) + + if interrupted: + self.check_line(output, 'Test suite interrupted by signal SIGINT.') + + if nfailed: + result = 'FAILURE' + elif interrupted: + result = 'INTERRUPTED' + else: + result = 'SUCCESS' + self.check_line(output, 'Tests result: %s' % result) + + def parse_random_seed(self, output): + match = self.regex_search(r'Using random seed ([0-9]+)', output) + randseed = int(match.group(1)) + self.assertTrue(0 <= randseed <= 10000000, randseed) + return randseed + + def run_command(self, args, input=None, exitcode=0, **kw): + if not input: + input = '' + if 'stderr' not in kw: + kw['stderr'] = subprocess.PIPE + proc = subprocess.run(args, + universal_newlines=True, + input=input, + stdout=subprocess.PIPE, + **kw) + if proc.returncode != exitcode: + msg = ("Command %s failed with exit code %s\n" + "\n" + "stdout:\n" + "---\n" + "%s\n" + "---\n" + % (str(args), proc.returncode, proc.stdout)) + if proc.stderr: + msg += ("\n" + "stderr:\n" + "---\n" + "%s" + "---\n" + % proc.stderr) + self.fail(msg) + return proc + + + def run_python(self, args, **kw): + args = [sys.executable, '-X', 'faulthandler', '-I', *args] + proc = self.run_command(args, **kw) + return proc.stdout + + +class ProgramsTestCase(BaseTestCase): + """ + Test various ways to run the Python test suite. Use options close + to options used on the buildbot. + """ + + NTEST = 4 + + def setUp(self): + super().setUp() + + # Create NTEST tests doing nothing + self.tests = [self.create_test() for index in range(self.NTEST)] + + self.python_args = ['-Wd', '-E', '-bb'] + self.regrtest_args = ['-uall', '-rwW', + '--testdir=%s' % self.tmptestdir] + if hasattr(faulthandler, 'dump_traceback_later'): + self.regrtest_args.extend(('--timeout', '3600', '-j4')) + if sys.platform == 'win32': + self.regrtest_args.append('-n') + + def check_output(self, output): + self.parse_random_seed(output) + self.check_executed_tests(output, self.tests, randomize=True) + + def run_tests(self, args): + output = self.run_python(args) + self.check_output(output) + + def test_script_regrtest(self): + # Lib/test/regrtest.py + script = os.path.join(self.testdir, 'regrtest.py') + + args = [*self.python_args, script, *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_test(self): + # -m test + args = [*self.python_args, '-m', 'test', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_regrtest(self): + # -m test.regrtest + args = [*self.python_args, '-m', 'test.regrtest', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_autotest(self): + # -m test.autotest + args = [*self.python_args, '-m', 'test.autotest', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_from_test_autotest(self): + # from test import autotest + code = 'from test import autotest' + args = [*self.python_args, '-c', code, + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_script_autotest(self): + # Lib/test/autotest.py + script = os.path.join(self.testdir, 'autotest.py') + args = [*self.python_args, script, *self.regrtest_args, *self.tests] + self.run_tests(args) + + @unittest.skipUnless(sysconfig.is_python_build(), + 'run_tests.py script is not installed') + def test_tools_script_run_tests(self): + # Tools/scripts/run_tests.py + script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') + args = [script, *self.regrtest_args, *self.tests] + self.run_tests(args) + + def run_batch(self, *args): + proc = self.run_command(args) + self.check_output(proc.stdout) + + @unittest.skipUnless(sysconfig.is_python_build(), + 'test.bat script is not installed') + @unittest.skipUnless(sys.platform == 'win32', 'Windows only') + def test_tools_buildbot_test(self): + # Tools\buildbot\test.bat + script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat') + test_args = ['--testdir=%s' % self.tmptestdir] + if platform.architecture()[0] == '64bit': + test_args.append('-x64') # 64-bit build + if not Py_DEBUG: + test_args.append('+d') # Release build, use python.exe + self.run_batch(script, *test_args, *self.tests) + + @unittest.skipUnless(sys.platform == 'win32', 'Windows only') + def test_pcbuild_rt(self): + # PCbuild\rt.bat + script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat') + rt_args = ["-q"] # Quick, don't run tests twice + if platform.architecture()[0] == '64bit': + rt_args.append('-x64') # 64-bit build + if Py_DEBUG: + rt_args.append('-d') # Debug build, use python_d.exe + self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) + + +class ArgsTestCase(BaseTestCase): + """ + Test arguments of the Python test suite. + """ + + def run_tests(self, *testargs, **kw): + cmdargs = ['-m', 'test', '--testdir=%s' % self.tmptestdir, *testargs] + return self.run_python(cmdargs, **kw) + + def test_failing_test(self): + # test a failing test + code = textwrap.dedent(""" + import unittest + + class FailingTest(unittest.TestCase): + def test_failing(self): + self.fail("bug") + """) + test_ok = self.create_test('ok') + test_failing = self.create_test('failing', code=code) + tests = [test_ok, test_failing] + + output = self.run_tests(*tests, exitcode=1) + self.check_executed_tests(output, tests, failed=test_failing) + + def test_resources(self): + # test -u command line option + tests = {} + for resource in ('audio', 'network'): + code = 'from test import support\nsupport.requires(%r)' % resource + tests[resource] = self.create_test(resource, code) + test_names = sorted(tests.values()) + + # -u all: 2 resources enabled + output = self.run_tests('-u', 'all', *test_names) + self.check_executed_tests(output, test_names) + + # -u audio: 1 resource enabled + output = self.run_tests('-uaudio', *test_names) + self.check_executed_tests(output, test_names, + skipped=tests['network']) + + # no option: 0 resources enabled + output = self.run_tests(*test_names) + self.check_executed_tests(output, test_names, + skipped=test_names) + + def test_random(self): + # test -r and --randseed command line option + code = textwrap.dedent(""" + import random + print("TESTRANDOM: %s" % random.randint(1, 1000)) + """) + test = self.create_test('random', code) + + # first run to get the output with the random seed + output = self.run_tests('-r', test) + randseed = self.parse_random_seed(output) + match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) + test_random = int(match.group(1)) + + # try to reproduce with the random seed + output = self.run_tests('-r', '--randseed=%s' % randseed, test) + randseed2 = self.parse_random_seed(output) + self.assertEqual(randseed2, randseed) + + match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) + test_random2 = int(match.group(1)) + self.assertEqual(test_random2, test_random) + + def test_fromfile(self): + # test --fromfile + tests = [self.create_test() for index in range(5)] + + # Write the list of files using a format similar to regrtest output: + # [1/2] test_1 + # [2/2] test_2 + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + # test format '0:00:00 [2/7] test_opcodes -- test_grammar took 0 sec' + with open(filename, "w") as fp: + previous = None + for index, name in enumerate(tests, 1): + line = ("00:00:%02i [%s/%s] %s" + % (index, index, len(tests), name)) + if previous: + line += " -- %s took 0 sec" % previous + print(line, file=fp) + previous = name + + output = self.run_tests('--fromfile', filename) + self.check_executed_tests(output, tests) + + # test format '[2/7] test_opcodes' + with open(filename, "w") as fp: + for index, name in enumerate(tests, 1): + print("[%s/%s] %s" % (index, len(tests), name), file=fp) + + output = self.run_tests('--fromfile', filename) + self.check_executed_tests(output, tests) + + # test format 'test_opcodes' + with open(filename, "w") as fp: + for name in tests: + print(name, file=fp) + + output = self.run_tests('--fromfile', filename) + self.check_executed_tests(output, tests) + + def test_interrupted(self): + code = TEST_INTERRUPTED + test = self.create_test('sigint', code=code) + output = self.run_tests(test, exitcode=1) + self.check_executed_tests(output, test, omitted=test, + interrupted=True) + + def test_slowest(self): + # test --slowest + tests = [self.create_test() for index in range(3)] + output = self.run_tests("--slowest", *tests) + self.check_executed_tests(output, tests) + regex = ('10 slowest tests:\n' + '(?:- %s: .*\n){%s}' + % (self.TESTNAME_REGEX, len(tests))) + self.check_line(output, regex) + + def test_slow_interrupted(self): + # Issue #25373: test --slowest with an interrupted test + code = TEST_INTERRUPTED + test = self.create_test("sigint", code=code) + + try: + import threading + tests = (False, True) + except ImportError: + tests = (False,) + for multiprocessing in tests: + if multiprocessing: + args = ("--slowest", "-j2", test) + else: + args = ("--slowest", test) + output = self.run_tests(*args, exitcode=1) + self.check_executed_tests(output, test, + omitted=test, interrupted=True) + + regex = ('10 slowest tests:\n') + self.check_line(output, regex) + + def test_coverage(self): + # test --coverage + test = self.create_test('coverage') + output = self.run_tests("--coverage", test) + self.check_executed_tests(output, [test]) + regex = (r'lines +cov% +module +\(path\)\n' + r'(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') + self.check_line(output, regex) + + def test_wait(self): + # test --wait + test = self.create_test('wait') + output = self.run_tests("--wait", test, input='key') + self.check_line(output, 'Press any key to continue') + + def test_forever(self): + # test --forever + code = textwrap.dedent(""" + import builtins + import unittest + + class ForeverTester(unittest.TestCase): + def test_run(self): + # Store the state in the builtins module, because the test + # module is reload at each run + if 'RUN' in builtins.__dict__: + builtins.__dict__['RUN'] += 1 + if builtins.__dict__['RUN'] >= 3: + self.fail("fail at the 3rd runs") + else: + builtins.__dict__['RUN'] = 1 + """) + test = self.create_test('forever', code=code) + output = self.run_tests('--forever', test, exitcode=1) + self.check_executed_tests(output, [test]*3, failed=test) + + @unittest.skipUnless(Py_DEBUG, 'need a debug build') + def test_huntrleaks_fd_leak(self): + # test --huntrleaks for file descriptor leak + code = textwrap.dedent(""" + import os + import unittest + + # Issue #25306: Disable popups and logs to stderr on assertion + # failures in MSCRT + try: + import msvcrt + msvcrt.CrtSetReportMode + except (ImportError, AttributeError): + # no Windows, o release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, 0) + + class FDLeakTest(unittest.TestCase): + def test_leak(self): + fd = os.open(__file__, os.O_RDONLY) + # bug: never cloes the file descriptor + """) + test = self.create_test('huntrleaks', code=code) + + filename = 'reflog.txt' + self.addCleanup(support.unlink, filename) + output = self.run_tests('--huntrleaks', '3:3:', test, + exitcode=1, + stderr=subprocess.STDOUT) + self.check_executed_tests(output, [test], failed=test) + + line = 'beginning 6 repetitions\n123456\n......\n' + self.check_line(output, re.escape(line)) + + line2 = '%s leaked [1, 1, 1] file descriptors, sum=3\n' % test + self.assertIn(line2, output) + + with open(filename) as fp: + reflog = fp.read() + self.assertIn(line2, reflog) + + def test_list_tests(self): + # test --list-tests + tests = [self.create_test() for i in range(5)] + output = self.run_tests('--list-tests', *tests) + self.assertEqual(output.rstrip().splitlines(), + tests) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_richcmp.py b/Lib/test/test_richcmp.py index 1582caad97..58729a9fea 100644 --- a/Lib/test/test_richcmp.py +++ b/Lib/test/test_richcmp.py @@ -253,6 +253,31 @@ class MiscTest(unittest.TestCase): self.assertTrue(a != b) self.assertTrue(a < b) + def test_exception_message(self): + class Spam: + pass + + tests = [ + (lambda: 42 < None, r"'<' .* of 'int' and 'NoneType'"), + (lambda: None < 42, r"'<' .* of 'NoneType' and 'int'"), + (lambda: 42 > None, r"'>' .* of 'int' and 'NoneType'"), + (lambda: "foo" < None, r"'<' .* of 'str' and 'NoneType'"), + (lambda: "foo" >= 666, r"'>=' .* of 'str' and 'int'"), + (lambda: 42 <= None, r"'<=' .* of 'int' and 'NoneType'"), + (lambda: 42 >= None, r"'>=' .* of 'int' and 'NoneType'"), + (lambda: 42 < [], r"'<' .* of 'int' and 'list'"), + (lambda: () > [], r"'>' .* of 'tuple' and 'list'"), + (lambda: None >= None, r"'>=' .* of 'NoneType' and 'NoneType'"), + (lambda: Spam() < 42, r"'<' .* of 'Spam' and 'int'"), + (lambda: 42 < Spam(), r"'<' .* of 'int' and 'Spam'"), + (lambda: Spam() <= Spam(), r"'<=' .* of 'Spam' and 'Spam'"), + ] + for i, test in enumerate(tests): + with self.subTest(test=i): + with self.assertRaisesRegex(TypeError, test[1]): + test[0]() + + class DictTest(unittest.TestCase): def test_dicts(self): diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 853e77330a..0dc1080ca3 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -1,11 +1,12 @@ import unittest -import unittest.mock +from unittest.mock import patch import builtins import rlcompleter class CompleteMe: """ Trivial class used in testing rlcompleter.Completer. """ spam = 1 + _ham = 2 class TestRlcompleter(unittest.TestCase): @@ -52,18 +53,32 @@ class TestRlcompleter(unittest.TestCase): ['str.{}('.format(x) for x in dir(str) if x.startswith('s')]) self.assertEqual(self.stdcompleter.attr_matches('tuple.foospamegg'), []) + expected = sorted({'None.%s%s' % (x, '(' if x != '__doc__' else '') + for x in dir(None)}) + self.assertEqual(self.stdcompleter.attr_matches('None.'), expected) + self.assertEqual(self.stdcompleter.attr_matches('None._'), expected) + self.assertEqual(self.stdcompleter.attr_matches('None.__'), expected) # test with a customized namespace self.assertEqual(self.completer.attr_matches('CompleteMe.sp'), ['CompleteMe.spam']) self.assertEqual(self.completer.attr_matches('Completeme.egg'), []) + self.assertEqual(self.completer.attr_matches('CompleteMe.'), + ['CompleteMe.mro(', 'CompleteMe.spam']) + self.assertEqual(self.completer.attr_matches('CompleteMe._'), + ['CompleteMe._ham']) + matches = self.completer.attr_matches('CompleteMe.__') + for x in matches: + self.assertTrue(x.startswith('CompleteMe.__'), x) + self.assertIn('CompleteMe.__name__', matches) + self.assertIn('CompleteMe.__new__(', matches) - CompleteMe.me = CompleteMe - self.assertEqual(self.completer.attr_matches('CompleteMe.me.me.sp'), - ['CompleteMe.me.me.spam']) - self.assertEqual(self.completer.attr_matches('egg.s'), - ['egg.{}('.format(x) for x in dir(str) - if x.startswith('s')]) + with patch.object(CompleteMe, "me", CompleteMe, create=True): + self.assertEqual(self.completer.attr_matches('CompleteMe.me.me.sp'), + ['CompleteMe.me.me.spam']) + self.assertEqual(self.completer.attr_matches('egg.s'), + ['egg.{}('.format(x) for x in dir(str) + if x.startswith('s')]) def test_excessive_getattr(self): # Ensure getattr() is invoked no more than once per attribute @@ -78,14 +93,27 @@ class TestRlcompleter(unittest.TestCase): self.assertEqual(completer.complete('f.b', 0), 'f.bar') self.assertEqual(f.calls, 1) + def test_uncreated_attr(self): + # Attributes like properties and slots should be completed even when + # they haven't been created on an instance + class Foo: + __slots__ = ("bar",) + completer = rlcompleter.Completer(dict(f=Foo())) + self.assertEqual(completer.complete('f.', 0), 'f.bar') + @unittest.mock.patch('rlcompleter._readline_available', False) def test_complete(self): completer = rlcompleter.Completer() self.assertEqual(completer.complete('', 0), '\t') - self.assertEqual(completer.complete('a', 0), 'and') - self.assertEqual(completer.complete('a', 1), 'as') - self.assertEqual(completer.complete('as', 2), 'assert') - self.assertEqual(completer.complete('an', 0), 'and') + self.assertEqual(completer.complete('a', 0), 'and ') + self.assertEqual(completer.complete('a', 1), 'as ') + self.assertEqual(completer.complete('as', 2), 'assert ') + self.assertEqual(completer.complete('an', 0), 'and ') + self.assertEqual(completer.complete('pa', 0), 'pass') + self.assertEqual(completer.complete('Fa', 0), 'False') + self.assertEqual(completer.complete('el', 0), 'elif ') + self.assertEqual(completer.complete('el', 1), 'else') + self.assertEqual(completer.complete('tr', 0), 'try:') def test_duplicate_globals(self): namespace = { @@ -98,9 +126,10 @@ class TestRlcompleter(unittest.TestCase): completer = rlcompleter.Completer(namespace) self.assertEqual(completer.complete('False', 0), 'False') self.assertIsNone(completer.complete('False', 1)) # No duplicates - self.assertEqual(completer.complete('assert', 0), 'assert') + # Space or colon added due to being a reserved keyword + self.assertEqual(completer.complete('assert', 0), 'assert ') self.assertIsNone(completer.complete('assert', 1)) - self.assertEqual(completer.complete('try', 0), 'try') + self.assertEqual(completer.complete('try', 0), 'try:') self.assertIsNone(completer.complete('try', 1)) # No opening bracket "(" because we overrode the built-in class self.assertEqual(completer.complete('memoryview', 0), 'memoryview') diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index d01266f330..0f64ba8b06 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -1,8 +1,8 @@ import io +import os import unittest import urllib.robotparser -from urllib.error import URLError, HTTPError -from urllib.request import urlopen +from collections import namedtuple from test import support from http.server import BaseHTTPRequestHandler, HTTPServer try: @@ -11,246 +11,239 @@ except ImportError: threading = None -class RobotTestCase(unittest.TestCase): - def __init__(self, index=None, parser=None, url=None, good=None, agent=None): - # workaround to make unittest discovery work (see #17066) - if not isinstance(index, int): - return - unittest.TestCase.__init__(self) - if good: - self.str = "RobotTest(%d, good, %s)" % (index, url) - else: - self.str = "RobotTest(%d, bad, %s)" % (index, url) - self.parser = parser - self.url = url - self.good = good - self.agent = agent - - def runTest(self): - if isinstance(self.url, tuple): - agent, url = self.url - else: - url = self.url - agent = self.agent - if self.good: - self.assertTrue(self.parser.can_fetch(agent, url)) - else: - self.assertFalse(self.parser.can_fetch(agent, url)) - - def __str__(self): - return self.str - -tests = unittest.TestSuite() - -def RobotTest(index, robots_txt, good_urls, bad_urls, - agent="test_robotparser"): - - lines = io.StringIO(robots_txt).readlines() - parser = urllib.robotparser.RobotFileParser() - parser.parse(lines) - for url in good_urls: - tests.addTest(RobotTestCase(index, parser, url, 1, agent)) - for url in bad_urls: - tests.addTest(RobotTestCase(index, parser, url, 0, agent)) - -# Examples from http://www.robotstxt.org/wc/norobots.html (fetched 2002) - -# 1. -doc = """ +class BaseRobotTest: + robots_txt = '' + agent = 'test_robotparser' + good = [] + bad = [] + + def setUp(self): + lines = io.StringIO(self.robots_txt).readlines() + self.parser = urllib.robotparser.RobotFileParser() + self.parser.parse(lines) + + def get_agent_and_url(self, url): + if isinstance(url, tuple): + agent, url = url + return agent, url + return self.agent, url + + def test_good_urls(self): + for url in self.good: + agent, url = self.get_agent_and_url(url) + with self.subTest(url=url, agent=agent): + self.assertTrue(self.parser.can_fetch(agent, url)) + + def test_bad_urls(self): + for url in self.bad: + agent, url = self.get_agent_and_url(url) + with self.subTest(url=url, agent=agent): + self.assertFalse(self.parser.can_fetch(agent, url)) + + +class UserAgentWildcardTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ User-agent: * Disallow: /cyberworld/map/ # This is an infinite virtual URL space Disallow: /tmp/ # these will soon disappear Disallow: /foo.html -""" - -good = ['/','/test.html'] -bad = ['/cyberworld/map/index.html','/tmp/xxx','/foo.html'] + """ + good = ['/', '/test.html'] + bad = ['/cyberworld/map/index.html', '/tmp/xxx', '/foo.html'] -RobotTest(1, doc, good, bad) -# 2. -doc = """ +class CrawlDelayAndCustomAgentTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ # robots.txt for http://www.example.com/ User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. User-agent: cybermapper Disallow: + """ + good = ['/', '/test.html', ('cybermapper', '/cyberworld/map/index.html')] + bad = ['/cyberworld/map/index.html'] -""" - -good = ['/','/test.html',('cybermapper','/cyberworld/map/index.html')] -bad = ['/cyberworld/map/index.html'] - -RobotTest(2, doc, good, bad) -# 3. -doc = """ +class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ # go away User-agent: * Disallow: / -""" - -good = [] -bad = ['/cyberworld/map/index.html','/','/tmp/'] - -RobotTest(3, doc, good, bad) - -# Examples from http://www.robotstxt.org/wc/norobots-rfc.html (fetched 2002) - -# 4. -doc = """ + """ + good = [] + bad = ['/cyberworld/map/index.html', '/', '/tmp/'] + + +class BaseRequestRateTest(BaseRobotTest): + + def test_request_rate(self): + for url in self.good + self.bad: + agent, url = self.get_agent_and_url(url) + with self.subTest(url=url, agent=agent): + if self.crawl_delay: + self.assertEqual( + self.parser.crawl_delay(agent), self.crawl_delay + ) + if self.request_rate: + self.assertEqual( + self.parser.request_rate(agent).requests, + self.request_rate.requests + ) + self.assertEqual( + self.parser.request_rate(agent).seconds, + self.request_rate.seconds + ) + + +class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase): + robots_txt = """\ User-agent: figtree +Crawl-delay: 3 +Request-rate: 9/30 Disallow: /tmp Disallow: /a%3cd.html Disallow: /a%2fb.html Disallow: /%7ejoe/index.html -""" + """ + agent = 'figtree' + request_rate = namedtuple('req_rate', 'requests seconds')(9, 30) + crawl_delay = 3 + good = [('figtree', '/foo.html')] + bad = ['/tmp', '/tmp.html', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', + '/a%2fb.html', '/~joe/index.html'] + -good = [] # XFAIL '/a/b.html' -bad = ['/tmp','/tmp.html','/tmp/a.html', - '/a%3cd.html','/a%3Cd.html','/a%2fb.html', - '/~joe/index.html' - ] +class DifferentAgentTest(CrawlDelayAndRequestRateTest): + agent = 'FigTree Robot libwww-perl/5.04' + # these are not actually tested, but we still need to parse it + # in order to accommodate the input parameters + request_rate = None + crawl_delay = None -RobotTest(4, doc, good, bad, 'figtree') -RobotTest(5, doc, good, bad, 'FigTree Robot libwww-perl/5.04') -# 6. -doc = """ +class InvalidRequestRateTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ User-agent: * Disallow: /tmp/ Disallow: /a%3Cd.html Disallow: /a/b.html Disallow: /%7ejoe/index.html -""" - -good = ['/tmp',] # XFAIL: '/a%2fb.html' -bad = ['/tmp/','/tmp/a.html', - '/a%3cd.html','/a%3Cd.html',"/a/b.html", - '/%7Ejoe/index.html'] - -RobotTest(6, doc, good, bad) - -# From bug report #523041 - -# 7. -doc = """ +Crawl-delay: 3 +Request-rate: 9/banana + """ + good = ['/tmp'] + bad = ['/tmp/', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', '/a/b.html', + '/%7Ejoe/index.html'] + crawl_delay = 3 + + +class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase): + # From bug report #523041 + robots_txt = """\ User-Agent: * Disallow: /. -""" - -good = ['/foo.html'] -bad = [] # Bug report says "/" should be denied, but that is not in the RFC +Crawl-delay: pears + """ + good = ['/foo.html'] + # bug report says "/" should be denied, but that is not in the RFC + bad = [] -RobotTest(7, doc, good, bad) -# From Google: http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40364 - -# 8. -doc = """ +class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase): + # also test that Allow and Diasallow works well with each other + robots_txt = """\ User-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ -""" - -good = ['/folder1/myfile.html'] -bad = ['/folder1/anotherfile.html'] - -RobotTest(8, doc, good, bad, agent="Googlebot") - -# 9. This file is incorrect because "Googlebot" is a substring of -# "Googlebot-Mobile", so test 10 works just like test 9. -doc = """ +Request-rate: whale/banana + """ + agent = 'Googlebot' + good = ['/folder1/myfile.html'] + bad = ['/folder1/anotherfile.html'] + + +class UserAgentOrderingTest(BaseRobotTest, unittest.TestCase): + # the order of User-agent should be correct. note + # that this file is incorrect because "Googlebot" is a + # substring of "Googlebot-Mobile" + robots_txt = """\ User-agent: Googlebot Disallow: / User-agent: Googlebot-Mobile Allow: / -""" - -good = [] -bad = ['/something.jpg'] + """ + agent = 'Googlebot' + bad = ['/something.jpg'] -RobotTest(9, doc, good, bad, agent="Googlebot") -good = [] -bad = ['/something.jpg'] +class UserAgentGoogleMobileTest(UserAgentOrderingTest): + agent = 'Googlebot-Mobile' -RobotTest(10, doc, good, bad, agent="Googlebot-Mobile") - -# 11. Get the order correct. -doc = """ -User-agent: Googlebot-Mobile -Allow: / -User-agent: Googlebot -Disallow: / -""" - -good = [] -bad = ['/something.jpg'] - -RobotTest(11, doc, good, bad, agent="Googlebot") - -good = ['/something.jpg'] -bad = [] - -RobotTest(12, doc, good, bad, agent="Googlebot-Mobile") - - -# 13. Google also got the order wrong in #8. You need to specify the -# URLs from more specific to more general. -doc = """ +class GoogleURLOrderingTest(BaseRobotTest, unittest.TestCase): + # Google also got the order wrong. You need + # to specify the URLs from more specific to more general + robots_txt = """\ User-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ -""" - -good = ['/folder1/myfile.html'] -bad = ['/folder1/anotherfile.html'] - -RobotTest(13, doc, good, bad, agent="googlebot") + """ + agent = 'googlebot' + good = ['/folder1/myfile.html'] + bad = ['/folder1/anotherfile.html'] -# 14. For issue #6325 (query string support) -doc = """ +class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase): + # see issue #6325 for details + robots_txt = """\ User-agent: * Disallow: /some/path?name=value -""" + """ + good = ['/some/path'] + bad = ['/some/path?name=value'] -good = ['/some/path'] -bad = ['/some/path?name=value'] -RobotTest(14, doc, good, bad) - -# 15. For issue #4108 (obey first * entry) -doc = """ +class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase): + # obey first * entry (#4108) + robots_txt = """\ User-agent: * Disallow: /some/path User-agent: * Disallow: /another/path -""" - -good = ['/another/path'] -bad = ['/some/path'] + """ + good = ['/another/path'] + bad = ['/some/path'] -RobotTest(15, doc, good, bad) -# 16. Empty query (issue #17403). Normalizing the url first. -doc = """ +class EmptyQueryStringTest(BaseRobotTest, unittest.TestCase): + # normalize the URL first (#17403) + robots_txt = """\ User-agent: * Allow: /some/path? Disallow: /another/path? -""" + """ + good = ['/some/path?'] + bad = ['/another/path?'] -good = ['/some/path?'] -bad = ['/another/path?'] -RobotTest(16, doc, good, bad) +class DefaultEntryTest(BaseRequestRateTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 +Disallow: /cyberworld/map/ + """ + request_rate = namedtuple('req_rate', 'requests seconds')(3, 15) + crawl_delay = 1 + good = ['/', '/test.html'] + bad = ['/cyberworld/map/index.html'] class RobotHandler(BaseHTTPRequestHandler): @@ -283,9 +276,7 @@ class PasswordProtectedSiteTestCase(unittest.TestCase): self.t.join() self.server.server_close() - def runTest(self): - self.testPasswordProtectedSite() - + @support.reap_threads def testPasswordProtectedSite(self): addr = self.server.server_address url = 'http://' + support.HOST + ':' + str(addr[1]) @@ -295,26 +286,47 @@ class PasswordProtectedSiteTestCase(unittest.TestCase): parser.read() self.assertFalse(parser.can_fetch("*", robots_url)) - def __str__(self): - return '%s' % self.__class__.__name__ class NetworkTestCase(unittest.TestCase): - @unittest.skip('does not handle the gzip encoding delivered by pydotorg') - def testPythonOrg(self): + base_url = 'http://www.pythontest.net/' + robots_txt = '{}elsewhere/robots.txt'.format(base_url) + + @classmethod + def setUpClass(cls): support.requires('network') - with support.transient_internet('www.python.org'): - parser = urllib.robotparser.RobotFileParser( - "http://www.python.org/robots.txt") - parser.read() - self.assertTrue( - parser.can_fetch("*", "http://www.python.org/robots.txt")) - -def load_tests(loader, suite, pattern): - suite = unittest.makeSuite(NetworkTestCase) - suite.addTest(tests) - suite.addTest(PasswordProtectedSiteTestCase()) - return suite + with support.transient_internet(cls.base_url): + cls.parser = urllib.robotparser.RobotFileParser(cls.robots_txt) + cls.parser.read() + + def url(self, path): + return '{}{}{}'.format( + self.base_url, path, '/' if not os.path.splitext(path)[1] else '' + ) + + def test_basic(self): + self.assertFalse(self.parser.disallow_all) + self.assertFalse(self.parser.allow_all) + self.assertGreater(self.parser.mtime(), 0) + self.assertFalse(self.parser.crawl_delay('*')) + self.assertFalse(self.parser.request_rate('*')) + + def test_can_fetch(self): + self.assertTrue(self.parser.can_fetch('*', self.url('elsewhere'))) + self.assertFalse(self.parser.can_fetch('Nutch', self.base_url)) + self.assertFalse(self.parser.can_fetch('Nutch', self.url('brian'))) + self.assertFalse(self.parser.can_fetch('Nutch', self.url('webstats'))) + self.assertFalse(self.parser.can_fetch('*', self.url('webstats'))) + self.assertTrue(self.parser.can_fetch('*', self.base_url)) + + def test_read_404(self): + parser = urllib.robotparser.RobotFileParser(self.url('i-robot.txt')) + parser.read() + self.assertTrue(parser.allow_all) + self.assertFalse(parser.disallow_all) + self.assertEqual(parser.mtime(), 0) + self.assertIsNone(parser.crawl_delay('*')) + self.assertIsNone(parser.request_rate('*')) if __name__=='__main__': unittest.main() diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index db55db702b..02b4d62567 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -577,13 +577,14 @@ from ..uncle.cousin import nephew self.addCleanup(self._del_pkg, pkg_dir) for depth in range(2, max_depth+1): self._add_relative_modules(pkg_dir, "", depth) - for finder, mod_name, ispkg in pkgutil.walk_packages([pkg_dir]): - self.assertIsInstance(finder, + for moduleinfo in pkgutil.walk_packages([pkg_dir]): + self.assertIsInstance(moduleinfo, pkgutil.ModuleInfo) + self.assertIsInstance(moduleinfo.module_finder, importlib.machinery.FileFinder) - if ispkg: - expected_packages.remove(mod_name) + if moduleinfo.ispkg: + expected_packages.remove(moduleinfo.name) else: - expected_modules.remove(mod_name) + expected_modules.remove(moduleinfo.name) self.assertEqual(len(expected_packages), 0, expected_packages) self.assertEqual(len(expected_modules), 0, expected_modules) diff --git a/Lib/test/test_sched.py b/Lib/test/test_sched.py index fe8e785092..ebf8856462 100644 --- a/Lib/test/test_sched.py +++ b/Lib/test/test_sched.py @@ -2,7 +2,6 @@ import queue import sched import time import unittest -from test import support try: import threading except ImportError: @@ -173,17 +172,23 @@ class TestCase(unittest.TestCase): self.assertEqual(scheduler.queue, [e1, e2, e3, e4, e5]) def test_args_kwargs(self): - flag = [] - + seq = [] def fun(*a, **b): - flag.append(None) - self.assertEqual(a, (1,2,3)) - self.assertEqual(b, {"foo":1}) + seq.append((a, b)) + now = time.time() scheduler = sched.scheduler(time.time, time.sleep) - z = scheduler.enterabs(0.01, 1, fun, argument=(1,2,3), kwargs={"foo":1}) + scheduler.enterabs(now, 1, fun) + scheduler.enterabs(now, 1, fun, argument=(1, 2)) + scheduler.enterabs(now, 1, fun, argument=('a', 'b')) + scheduler.enterabs(now, 1, fun, argument=(1, 2), kwargs={"foo": 3}) scheduler.run() - self.assertEqual(flag, [None]) + self.assertCountEqual(seq, [ + ((), {}), + ((1, 2), {}), + (('a', 'b'), {}), + ((1, 2), {'foo': 3}) + ]) def test_run_non_blocking(self): l = [] diff --git a/Lib/test/test_secrets.py b/Lib/test/test_secrets.py new file mode 100644 index 0000000000..d31d07e01f --- /dev/null +++ b/Lib/test/test_secrets.py @@ -0,0 +1,124 @@ +"""Test the secrets module. + +As most of the functions in secrets are thin wrappers around functions +defined elsewhere, we don't need to test them exhaustively. +""" + + +import secrets +import unittest +import string + + +# === Unit tests === + +class Compare_Digest_Tests(unittest.TestCase): + """Test secrets.compare_digest function.""" + + def test_equal(self): + # Test compare_digest functionality with equal (byte/text) strings. + for s in ("a", "bcd", "xyz123"): + a = s*100 + b = s*100 + self.assertTrue(secrets.compare_digest(a, b)) + self.assertTrue(secrets.compare_digest(a.encode('utf-8'), b.encode('utf-8'))) + + def test_unequal(self): + # Test compare_digest functionality with unequal (byte/text) strings. + self.assertFalse(secrets.compare_digest("abc", "abcd")) + self.assertFalse(secrets.compare_digest(b"abc", b"abcd")) + for s in ("x", "mn", "a1b2c3"): + a = s*100 + "q" + b = s*100 + "k" + self.assertFalse(secrets.compare_digest(a, b)) + self.assertFalse(secrets.compare_digest(a.encode('utf-8'), b.encode('utf-8'))) + + def test_bad_types(self): + # Test that compare_digest raises with mixed types. + a = 'abcde' + b = a.encode('utf-8') + assert isinstance(a, str) + assert isinstance(b, bytes) + self.assertRaises(TypeError, secrets.compare_digest, a, b) + self.assertRaises(TypeError, secrets.compare_digest, b, a) + + def test_bool(self): + # Test that compare_digest returns a bool. + self.assertIsInstance(secrets.compare_digest("abc", "abc"), bool) + self.assertIsInstance(secrets.compare_digest("abc", "xyz"), bool) + + +class Random_Tests(unittest.TestCase): + """Test wrappers around SystemRandom methods.""" + + def test_randbits(self): + # Test randbits. + errmsg = "randbits(%d) returned %d" + for numbits in (3, 12, 30): + for i in range(6): + n = secrets.randbits(numbits) + self.assertTrue(0 <= n < 2**numbits, errmsg % (numbits, n)) + + def test_choice(self): + # Test choice. + items = [1, 2, 4, 8, 16, 32, 64] + for i in range(10): + self.assertTrue(secrets.choice(items) in items) + + def test_randbelow(self): + # Test randbelow. + for i in range(2, 10): + self.assertIn(secrets.randbelow(i), range(i)) + self.assertRaises(ValueError, secrets.randbelow, 0) + self.assertRaises(ValueError, secrets.randbelow, -1) + + +class Token_Tests(unittest.TestCase): + """Test token functions.""" + + def test_token_defaults(self): + # Test that token_* functions handle default size correctly. + for func in (secrets.token_bytes, secrets.token_hex, + secrets.token_urlsafe): + with self.subTest(func=func): + name = func.__name__ + try: + func() + except TypeError: + self.fail("%s cannot be called with no argument" % name) + try: + func(None) + except TypeError: + self.fail("%s cannot be called with None" % name) + size = secrets.DEFAULT_ENTROPY + self.assertEqual(len(secrets.token_bytes(None)), size) + self.assertEqual(len(secrets.token_hex(None)), 2*size) + + def test_token_bytes(self): + # Test token_bytes. + for n in (1, 8, 17, 100): + with self.subTest(n=n): + self.assertIsInstance(secrets.token_bytes(n), bytes) + self.assertEqual(len(secrets.token_bytes(n)), n) + + def test_token_hex(self): + # Test token_hex. + for n in (1, 12, 25, 90): + with self.subTest(n=n): + s = secrets.token_hex(n) + self.assertIsInstance(s, str) + self.assertEqual(len(s), 2*n) + self.assertTrue(all(c in string.hexdigits for c in s)) + + def test_token_urlsafe(self): + # Test token_urlsafe. + legal = string.ascii_letters + string.digits + '-_' + for n in (1, 11, 28, 76): + with self.subTest(n=n): + s = secrets.token_urlsafe(n) + self.assertIsInstance(s, str) + self.assertTrue(all(c in legal for c in s)) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index 7594303132..0202981d67 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -6,10 +6,11 @@ import operator import copy import pickle from random import randrange, shuffle -import sys import warnings import collections import collections.abc +import itertools +import string class PassThru(Exception): pass @@ -729,6 +730,25 @@ class TestFrozenSet(TestJointOps, unittest.TestCase): addhashvalue(hash(frozenset([e for e, m in elemmasks if m&i]))) self.assertEqual(len(hashvalues), 2**n) + def zf_range(n): + # https://en.wikipedia.org/wiki/Set-theoretic_definition_of_natural_numbers + nums = [frozenset()] + for i in range(n-1): + num = frozenset(nums) + nums.append(num) + return nums[:n] + + def powerset(s): + for i in range(len(s)+1): + yield from map(frozenset, itertools.combinations(s, i)) + + for n in range(18): + t = 2 ** n + mask = t - 1 + for nums in (range, zf_range): + u = len({h & mask for h in map(hash, powerset(nums(n)))}) + self.assertGreater(4*u, t) + class FrozenSetSubclass(frozenset): pass diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index 55b533d5ab..fd35788e81 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -173,16 +173,124 @@ class ShlexTest(unittest.TestCase): "%s: %s != %s" % (self.data[i][0], l, self.data[i][1:])) + def testSyntaxSplitAmpersandAndPipe(self): + """Test handling of syntax splitting of &, |""" + # Could take these forms: &&, &, |&, ;&, ;;& + # of course, the same applies to | and || + # these should all parse to the same output + for delimiter in ('&&', '&', '|&', ';&', ';;&', + '||', '|', '&|', ';|', ';;|'): + src = ['echo hi %s echo bye' % delimiter, + 'echo hi%secho bye' % delimiter] + ref = ['echo', 'hi', delimiter, 'echo', 'bye'] + for ss in src: + s = shlex.shlex(ss, punctuation_chars=True) + result = list(s) + self.assertEqual(ref, result, "While splitting '%s'" % ss) + + def testSyntaxSplitSemicolon(self): + """Test handling of syntax splitting of ;""" + # Could take these forms: ;, ;;, ;&, ;;& + # these should all parse to the same output + for delimiter in (';', ';;', ';&', ';;&'): + src = ['echo hi %s echo bye' % delimiter, + 'echo hi%s echo bye' % delimiter, + 'echo hi%secho bye' % delimiter] + ref = ['echo', 'hi', delimiter, 'echo', 'bye'] + for ss in src: + s = shlex.shlex(ss, punctuation_chars=True) + result = list(s) + self.assertEqual(ref, result, "While splitting '%s'" % ss) + + def testSyntaxSplitRedirect(self): + """Test handling of syntax splitting of >""" + # of course, the same applies to <, | + # these should all parse to the same output + for delimiter in ('<', '|'): + src = ['echo hi %s out' % delimiter, + 'echo hi%s out' % delimiter, + 'echo hi%sout' % delimiter] + ref = ['echo', 'hi', delimiter, 'out'] + for ss in src: + s = shlex.shlex(ss, punctuation_chars=True) + result = list(s) + self.assertEqual(ref, result, "While splitting '%s'" % ss) + + def testSyntaxSplitParen(self): + """Test handling of syntax splitting of ()""" + # these should all parse to the same output + src = ['( echo hi )', + '(echo hi)'] + ref = ['(', 'echo', 'hi', ')'] + for ss in src: + s = shlex.shlex(ss, punctuation_chars=True) + result = list(s) + self.assertEqual(ref, result, "While splitting '%s'" % ss) + + def testSyntaxSplitCustom(self): + """Test handling of syntax splitting with custom chars""" + ref = ['~/a', '&', '&', 'b-c', '--color=auto', '||', 'd', '*.py?'] + ss = "~/a && b-c --color=auto || d *.py?" + s = shlex.shlex(ss, punctuation_chars="|") + result = list(s) + self.assertEqual(ref, result, "While splitting '%s'" % ss) + + def testTokenTypes(self): + """Test that tokens are split with types as expected.""" + for source, expected in ( + ('a && b || c', + [('a', 'a'), ('&&', 'c'), ('b', 'a'), + ('||', 'c'), ('c', 'a')]), + ): + s = shlex.shlex(source, punctuation_chars=True) + observed = [] + while True: + t = s.get_token() + if t == s.eof: + break + if t[0] in s.punctuation_chars: + tt = 'c' + else: + tt = 'a' + observed.append((t, tt)) + self.assertEqual(observed, expected) + + def testPunctuationInWordChars(self): + """Test that any punctuation chars are removed from wordchars""" + s = shlex.shlex('a_b__c', punctuation_chars='_') + self.assertNotIn('_', s.wordchars) + self.assertEqual(list(s), ['a', '_', 'b', '__', 'c']) + + def testPunctuationWithWhitespaceSplit(self): + """Test that with whitespace_split, behaviour is as expected""" + s = shlex.shlex('a && b || c', punctuation_chars='&') + # whitespace_split is False, so splitting will be based on + # punctuation_chars + self.assertEqual(list(s), ['a', '&&', 'b', '|', '|', 'c']) + s = shlex.shlex('a && b || c', punctuation_chars='&') + s.whitespace_split = True + # whitespace_split is True, so splitting will be based on + # white space + self.assertEqual(list(s), ['a', '&&', 'b', '||', 'c']) + + def testPunctuationWithPosix(self): + """Test that punctuation_chars and posix behave correctly together.""" + # see Issue #29132 + s = shlex.shlex('f >"abc"', posix=True, punctuation_chars=True) + self.assertEqual(list(s), ['f', '>', 'abc']) + s = shlex.shlex('f >\\"abc\\"', posix=True, punctuation_chars=True) + self.assertEqual(list(s), ['f', '>', '"abc"']) + def testEmptyStringHandling(self): """Test that parsing of empty strings is correctly handled.""" # see Issue #21999 expected = ['', ')', 'abc'] - - s = shlex.shlex("'')abc", posix=True) - slist = list(s) - self.assertEqual(slist, expected) + for punct in (False, True): + s = shlex.shlex("'')abc", posix=True, punctuation_chars=punct) + slist = list(s) + self.assertEqual(slist, expected) expected = ["''", ')', 'abc'] - s = shlex.shlex("'')abc") + s = shlex.shlex("'')abc", punctuation_chars=True) self.assertEqual(list(s), expected) def testQuote(self): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 709feebf01..2ad3a21c86 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -23,7 +23,8 @@ import zipfile import warnings from test import support -from test.support import TESTFN, check_warnings, captured_stdout +from test.support import (TESTFN, check_warnings, captured_stdout, + android_not_root) TESTFN2 = TESTFN + "2" @@ -115,9 +116,7 @@ class TestShutil(unittest.TestCase): write_file(os.path.join(victim, 'somefile'), 'foo') victim = os.fsencode(victim) self.assertIsInstance(victim, bytes) - win = (os.name == 'nt') - with self.assertWarns(DeprecationWarning) if win else ExitStack(): - shutil.rmtree(victim) + shutil.rmtree(victim) @support.skip_unless_symlink def test_rmtree_fails_on_symlink(self): @@ -772,6 +771,7 @@ class TestShutil(unittest.TestCase): @unittest.skipIf(os.name == 'nt', 'temporarily disabled on Windows') @unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') + @unittest.skipIf(android_not_root, "hard links not allowed, non root user") def test_dont_copy_file_onto_link_to_itself(self): # bug 851123. os.mkdir(TESTFN) @@ -824,6 +824,7 @@ class TestShutil(unittest.TestCase): # Issue #3002: copyfile and copytree block indefinitely on named pipes @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") def test_copyfile_named_pipe(self): os.mkfifo(TESTFN) try: @@ -834,6 +835,7 @@ class TestShutil(unittest.TestCase): finally: os.remove(TESTFN) + @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') @support.skip_unless_symlink def test_copytree_named_pipe(self): @@ -1312,10 +1314,10 @@ class TestShutil(unittest.TestCase): shutil.chown(filename) with self.assertRaises(LookupError): - shutil.chown(filename, user='non-exising username') + shutil.chown(filename, user='non-existing username') with self.assertRaises(LookupError): - shutil.chown(filename, group='non-exising groupname') + shutil.chown(filename, group='non-existing groupname') with self.assertRaises(TypeError): shutil.chown(filename, b'spam') @@ -1864,7 +1866,8 @@ class TermsizeTests(unittest.TestCase): """ try: size = subprocess.check_output(['stty', 'size']).decode().split() - except (FileNotFoundError, subprocess.CalledProcessError): + except (FileNotFoundError, PermissionError, + subprocess.CalledProcessError): self.skipTest("stty invocation failed") expected = (int(size[1]), int(size[0])) # reversed order diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 1b80ff0963..ab42ed70cc 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -22,29 +22,6 @@ except ImportError: _testcapi = None -class HandlerBCalled(Exception): - pass - - -def exit_subprocess(): - """Use os._exit(0) to exit the current subprocess. - - Otherwise, the test catches the SystemExit and continues executing - in parallel with the original test, so you wind up with an - exponential number of tests running concurrently. - """ - os._exit(0) - - -def ignoring_eintr(__func, *args, **kwargs): - try: - return __func(*args, **kwargs) - except OSError as e: - if e.errno != errno.EINTR: - raise - return None - - class GenericTests(unittest.TestCase): @unittest.skipIf(threading is None, "test needs threading module") @@ -63,145 +40,6 @@ class GenericTests(unittest.TestCase): @unittest.skipIf(sys.platform == "win32", "Not valid on Windows") -class InterProcessSignalTests(unittest.TestCase): - MAX_DURATION = 20 # Entire test should last at most 20 sec. - - def setUp(self): - self.using_gc = gc.isenabled() - gc.disable() - - def tearDown(self): - if self.using_gc: - gc.enable() - - def format_frame(self, frame, limit=None): - return ''.join(traceback.format_stack(frame, limit=limit)) - - def handlerA(self, signum, frame): - self.a_called = True - - def handlerB(self, signum, frame): - self.b_called = True - raise HandlerBCalled(signum, self.format_frame(frame)) - - def wait(self, child): - """Wait for child to finish, ignoring EINTR.""" - while True: - try: - child.wait() - return - except OSError as e: - if e.errno != errno.EINTR: - raise - - def run_test(self): - # Install handlers. This function runs in a sub-process, so we - # don't worry about re-setting the default handlers. - signal.signal(signal.SIGHUP, self.handlerA) - signal.signal(signal.SIGUSR1, self.handlerB) - signal.signal(signal.SIGUSR2, signal.SIG_IGN) - signal.signal(signal.SIGALRM, signal.default_int_handler) - - # Variables the signals will modify: - self.a_called = False - self.b_called = False - - # Let the sub-processes know who to send signals to. - pid = os.getpid() - - child = ignoring_eintr(subprocess.Popen, ['kill', '-HUP', str(pid)]) - if child: - self.wait(child) - if not self.a_called: - time.sleep(1) # Give the signal time to be delivered. - self.assertTrue(self.a_called) - self.assertFalse(self.b_called) - self.a_called = False - - # Make sure the signal isn't delivered while the previous - # Popen object is being destroyed, because __del__ swallows - # exceptions. - del child - try: - child = subprocess.Popen(['kill', '-USR1', str(pid)]) - # This wait should be interrupted by the signal's exception. - self.wait(child) - time.sleep(1) # Give the signal time to be delivered. - self.fail('HandlerBCalled exception not raised') - except HandlerBCalled: - self.assertTrue(self.b_called) - self.assertFalse(self.a_called) - - child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)]) - if child: - self.wait(child) # Nothing should happen. - - try: - signal.alarm(1) - # The race condition in pause doesn't matter in this case, - # since alarm is going to raise a KeyboardException, which - # will skip the call. - signal.pause() - # But if another signal arrives before the alarm, pause - # may return early. - time.sleep(1) - except KeyboardInterrupt: - pass - except: - self.fail("Some other exception woke us from pause: %s" % - traceback.format_exc()) - else: - self.fail("pause returned of its own accord, and the signal" - " didn't arrive after another second.") - - # Issue 3864, unknown if this affects earlier versions of freebsd also - @unittest.skipIf(sys.platform=='freebsd6', - 'inter process signals not reliable (do not mix well with threading) ' - 'on freebsd6') - def test_main(self): - # This function spawns a child process to insulate the main - # test-running process from all the signals. It then - # communicates with that child process over a pipe and - # re-raises information about any exceptions the child - # raises. The real work happens in self.run_test(). - os_done_r, os_done_w = os.pipe() - with closing(os.fdopen(os_done_r, 'rb')) as done_r, \ - closing(os.fdopen(os_done_w, 'wb')) as done_w: - child = os.fork() - if child == 0: - # In the child process; run the test and report results - # through the pipe. - try: - done_r.close() - # Have to close done_w again here because - # exit_subprocess() will skip the enclosing with block. - with closing(done_w): - try: - self.run_test() - except: - pickle.dump(traceback.format_exc(), done_w) - else: - pickle.dump(None, done_w) - except: - print('Uh oh, raised from pickle.') - traceback.print_exc() - finally: - exit_subprocess() - - done_w.close() - # Block for up to MAX_DURATION seconds for the test to finish. - r, w, x = select.select([done_r], [], [], self.MAX_DURATION) - if done_r in r: - tb = pickle.load(done_r) - if tb: - self.fail(tb) - else: - os.kill(child, signal.SIGKILL) - self.fail('Test deadlocked after %d seconds.' % - self.MAX_DURATION) - - -@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") class PosixTests(unittest.TestCase): def trivial_signal_handler(self, *args): pass @@ -224,6 +62,15 @@ class PosixTests(unittest.TestCase): signal.signal(signal.SIGHUP, hup) self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + # Issue 3864, unknown if this affects earlier versions of freebsd also + @unittest.skipIf(sys.platform=='freebsd6', + 'inter process signals not reliable (do not mix well with threading) ' + 'on freebsd6') + def test_interprocess_signal(self): + dirname = os.path.dirname(__file__) + script = os.path.join(dirname, 'signalinterproctester.py') + assert_python_ok(script) + @unittest.skipUnless(sys.platform == "win32", "Windows specific") class WindowsSignalTests(unittest.TestCase): diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index da20a3d21a..d245fd5e1b 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -14,6 +14,7 @@ import re import encodings import urllib.request import urllib.error +import shutil import subprocess import sysconfig from copy import copy @@ -75,7 +76,7 @@ class HelperFunctionsTests(unittest.TestCase): def test_init_pathinfo(self): dir_set = site._init_pathinfo() for entry in [site.makepath(path)[1] for path in sys.path - if path and os.path.isdir(path)]: + if path and os.path.exists(path)]: self.assertIn(entry, dir_set, "%s from sys.path not found in set returned " "by _init_pathinfo(): %s" % (entry, dir_set)) @@ -138,10 +139,8 @@ class HelperFunctionsTests(unittest.TestCase): re.escape(os.path.join(pth_dir, pth_fn))) # XXX: ditto previous XXX comment. self.assertRegex(err_out.getvalue(), 'Traceback') - self.assertRegex(err_out.getvalue(), 'ImportError') + self.assertRegex(err_out.getvalue(), 'ModuleNotFoundError') - @unittest.skipIf(sys.platform == "win32", "Windows does not raise an " - "error for file paths containing null characters") def test_addpackage_import_bad_pth_file(self): # Issue 5258 pth_dir, pth_fn = self.make_pth("abc\x00def\n") @@ -243,13 +242,14 @@ class HelperFunctionsTests(unittest.TestCase): self.assertEqual(len(dirs), 2) wanted = os.path.join('/Library', sysconfig.get_config_var("PYTHONFRAMEWORK"), - sys.version[:3], + '%d.%d' % sys.version_info[:2], 'site-packages') self.assertEqual(dirs[1], wanted) elif os.sep == '/': # OS X non-framwework builds, Linux, FreeBSD, etc self.assertEqual(len(dirs), 1) - wanted = os.path.join('xoxo', 'lib', 'python' + sys.version[:3], + wanted = os.path.join('xoxo', 'lib', + 'python%d.%d' % sys.version_info[:2], 'site-packages') self.assertEqual(dirs[0], wanted) else: @@ -446,10 +446,9 @@ class StartupImportTests(unittest.TestCase): popen = subprocess.Popen([sys.executable, '-I', '-v', '-c', 'import sys; print(set(sys.modules))'], stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + encoding='utf-8') stdout, stderr = popen.communicate() - stdout = stdout.decode('utf-8') - stderr = stderr.decode('utf-8') modules = eval(stdout) self.assertIn('site', modules) @@ -469,7 +468,100 @@ class StartupImportTests(unittest.TestCase): 'heapq', 'itertools', 'keyword', 'operator', 'reprlib', 'types', 'weakref' }.difference(sys.builtin_module_names) - self.assertFalse(modules.intersection(collection_mods), stderr) + # http://bugs.python.org/issue28095 + if sys.platform != 'darwin': + self.assertFalse(modules.intersection(collection_mods), stderr) + + def test_startup_interactivehook(self): + r = subprocess.Popen([sys.executable, '-c', + 'import sys; sys.exit(hasattr(sys, "__interactivehook__"))']).wait() + self.assertTrue(r, "'__interactivehook__' not added by site") + + def test_startup_interactivehook_isolated(self): + # issue28192 readline is not automatically enabled in isolated mode + r = subprocess.Popen([sys.executable, '-I', '-c', + 'import sys; sys.exit(hasattr(sys, "__interactivehook__"))']).wait() + self.assertFalse(r, "'__interactivehook__' added in isolated mode") + + def test_startup_interactivehook_isolated_explicit(self): + # issue28192 readline can be explicitly enabled in isolated mode + r = subprocess.Popen([sys.executable, '-I', '-c', + 'import site, sys; site.enablerlcompleter(); sys.exit(hasattr(sys, "__interactivehook__"))']).wait() + self.assertTrue(r, "'__interactivehook__' not added by enablerlcompleter()") + + @classmethod + def _create_underpth_exe(self, lines): + exe_file = os.path.join(os.getenv('TEMP'), os.path.split(sys.executable)[1]) + shutil.copy(sys.executable, exe_file) + + _pth_file = os.path.splitext(exe_file)[0] + '._pth' + try: + with open(_pth_file, 'w') as f: + for line in lines: + print(line, file=f) + return exe_file + except: + os.unlink(_pth_file) + os.unlink(exe_file) + raise + + @classmethod + def _cleanup_underpth_exe(self, exe_file): + _pth_file = os.path.splitext(exe_file)[0] + '._pth' + os.unlink(_pth_file) + os.unlink(exe_file) + + @unittest.skipUnless(sys.platform == 'win32', "only supported on Windows") + def test_underpth_nosite_file(self): + libpath = os.path.dirname(os.path.dirname(encodings.__file__)) + exe_prefix = os.path.dirname(sys.executable) + exe_file = self._create_underpth_exe([ + 'fake-path-name', + *[libpath for _ in range(200)], + '# comment', + 'import site' + ]) + + try: + env = os.environ.copy() + env['PYTHONPATH'] = 'from-env' + env['PATH'] = '{};{}'.format(exe_prefix, os.getenv('PATH')) + rc = subprocess.call([exe_file, '-c', + 'import sys; sys.exit(sys.flags.no_site and ' + 'len(sys.path) > 200 and ' + '%r in sys.path and %r in sys.path and %r not in sys.path)' % ( + os.path.join(sys.prefix, 'fake-path-name'), + libpath, + os.path.join(sys.prefix, 'from-env'), + )], env=env) + finally: + self._cleanup_underpth_exe(exe_file) + self.assertEqual(rc, 0) + + @unittest.skipUnless(sys.platform == 'win32', "only supported on Windows") + def test_underpth_file(self): + libpath = os.path.dirname(os.path.dirname(encodings.__file__)) + exe_prefix = os.path.dirname(sys.executable) + exe_file = self._create_underpth_exe([ + 'fake-path-name', + *[libpath for _ in range(200)], + '# comment', + 'import site' + ]) + try: + env = os.environ.copy() + env['PYTHONPATH'] = 'from-env' + env['PATH'] = '{};{}'.format(exe_prefix, os.getenv('PATH')) + rc = subprocess.call([exe_file, '-c', + 'import sys; sys.exit(not sys.flags.no_site and ' + '%r in sys.path and %r in sys.path and %r not in sys.path)' % ( + os.path.join(sys.prefix, 'fake-path-name'), + libpath, + os.path.join(sys.prefix, 'from-env'), + )], env=env) + finally: + self._cleanup_underpth_exe(exe_file) + self.assertEqual(rc, 0) if __name__ == "__main__": diff --git a/Lib/test/test_smtpd.py b/Lib/test/test_smtpd.py index 88dbfdf6f0..3eebe948ad 100644 --- a/Lib/test/test_smtpd.py +++ b/Lib/test/test_smtpd.py @@ -53,10 +53,6 @@ class SMTPDServerTest(unittest.TestCase): write_line(b'DATA') self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n') - def test_decode_data_default_warns(self): - with self.assertWarns(DeprecationWarning): - smtpd.SMTPServer((support.HOST, 0), ('b', 0)) - def test_decode_data_and_enable_SMTPUTF8_raises(self): self.assertRaises( ValueError, @@ -108,10 +104,9 @@ class DebuggingServerTest(unittest.TestCase): """)) def test_process_message_with_decode_data_false(self): - server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0), - decode_data=False) + server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0)) conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + channel = smtpd.SMTPChannel(server, conn, addr) with support.captured_stdout() as s: self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n') stdout = s.getvalue() @@ -175,13 +170,11 @@ class TestFamilyDetection(unittest.TestCase): @unittest.skipUnless(support.IPV6_ENABLED, "IPv6 not enabled") def test_socket_uses_IPv6(self): - server = smtpd.SMTPServer((support.HOSTv6, 0), (support.HOST, 0), - decode_data=False) + server = smtpd.SMTPServer((support.HOSTv6, 0), (support.HOST, 0)) self.assertEqual(server.socket.family, socket.AF_INET6) def test_socket_uses_IPv4(self): - server = smtpd.SMTPServer((support.HOST, 0), (support.HOSTv6, 0), - decode_data=False) + server = smtpd.SMTPServer((support.HOST, 0), (support.HOSTv6, 0)) self.assertEqual(server.socket.family, socket.AF_INET) @@ -204,18 +197,18 @@ class TestRcptOptionParsing(unittest.TestCase): channel.handle_read() def test_params_rejected(self): - server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False) + server = DummyServer((support.HOST, 0), ('b', 0)) conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + channel = smtpd.SMTPChannel(server, conn, addr) self.write_line(channel, b'EHLO example') self.write_line(channel, b'MAIL from: <foo@example.com> size=20') self.write_line(channel, b'RCPT to: <foo@example.com> foo=bar') self.assertEqual(channel.socket.last, self.error_response) def test_nothing_accepted(self): - server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False) + server = DummyServer((support.HOST, 0), ('b', 0)) conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + channel = smtpd.SMTPChannel(server, conn, addr) self.write_line(channel, b'EHLO example') self.write_line(channel, b'MAIL from: <foo@example.com> size=20') self.write_line(channel, b'RCPT to: <foo@example.com>') @@ -257,9 +250,9 @@ class TestMailOptionParsing(unittest.TestCase): self.assertEqual(channel.socket.last, b'250 OK\r\n') def test_with_decode_data_false(self): - server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False) + server = DummyServer((support.HOST, 0), ('b', 0)) conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + channel = smtpd.SMTPChannel(server, conn, addr) self.write_line(channel, b'EHLO example') for line in [ b'MAIL from: <foo@example.com> size=20 SMTPUTF8', @@ -765,13 +758,6 @@ class SMTPDChannelTest(unittest.TestCase): with support.check_warnings(('', DeprecationWarning)): self.channel._SMTPChannel__addr = 'spam' - def test_decode_data_default_warning(self): - with self.assertWarns(DeprecationWarning): - server = DummyServer((support.HOST, 0), ('b', 0)) - conn, addr = self.server.accept() - with self.assertWarns(DeprecationWarning): - smtpd.SMTPChannel(server, conn, addr) - @unittest.skipUnless(support.IPV6_ENABLED, "IPv6 not enabled") class SMTPDChannelIPv6Test(SMTPDChannelTest): def setUp(self): @@ -845,12 +831,9 @@ class SMTPDChannelWithDecodeDataFalse(unittest.TestCase): smtpd.socket = asyncore.socket = mock_socket self.old_debugstream = smtpd.DEBUGSTREAM self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer((support.HOST, 0), ('b', 0), - decode_data=False) + self.server = DummyServer((support.HOST, 0), ('b', 0)) conn, addr = self.server.accept() - # Set decode_data to False - self.channel = smtpd.SMTPChannel(self.server, conn, addr, - decode_data=False) + self.channel = smtpd.SMTPChannel(self.server, conn, addr) def tearDown(self): asyncore.close_all() @@ -1015,5 +998,16 @@ class SMTPDChannelTestWithEnableSMTPUTF8True(unittest.TestCase): self.write_line(b'test\r\n.') self.assertEqual(self.channel.socket.last[0:3], b'250') + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = { + "program", "Devnull", "DEBUGSTREAM", "NEWLINE", "COMMASPACE", + "DATA_SIZE_DEFAULT", "usage", "Options", "parseargs", + + } + support.check__all__(self, smtpd, blacklist=blacklist) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 70c03f9c46..97dc3cd76b 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -13,7 +13,6 @@ import queue import sys import os import array -import platform import contextlib from weakref import proxy import signal @@ -66,10 +65,22 @@ def _have_socket_rds(): s.close() return True +def _have_socket_alg(): + """Check whether AF_ALG sockets are supported on this host.""" + try: + s = socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET, 0) + except (AttributeError, OSError): + return False + else: + s.close() + return True + HAVE_SOCKET_CAN = _have_socket_can() HAVE_SOCKET_RDS = _have_socket_rds() +HAVE_SOCKET_ALG = _have_socket_alg() + # Size in bytes of the int type SIZEOF_INT = array.array("i").itemsize @@ -267,8 +278,14 @@ class ThreadableTest: def clientRun(self, test_func): self.server_ready.wait() - self.clientSetUp() - self.client_ready.set() + try: + self.clientSetUp() + except BaseException as e: + self.queue.put(e) + self.clientTearDown() + return + finally: + self.client_ready.set() if self.server_crashed: self.clientTearDown() return @@ -509,8 +526,11 @@ class ConnectedStreamTestMixin(SocketListeningTestMixin, self.serv_conn = self.cli def clientTearDown(self): - self.serv_conn.close() - self.serv_conn = None + try: + self.serv_conn.close() + self.serv_conn = None + except AttributeError: + pass super().clientTearDown() @@ -529,7 +549,7 @@ class UnixSocketTestBase(SocketTestBase): def bindSock(self, sock): path = tempfile.mktemp(dir=self.dir_path) - sock.bind(path) + support.bind_unix_socket(sock, path) self.addCleanup(support.unlink, path) class UnixStreamBase(UnixSocketTestBase): @@ -1161,6 +1181,17 @@ class GeneralModuleTests(unittest.TestCase): sock.close() self.assertRaises(OSError, sock.send, b"spam") + def testCloseException(self): + sock = socket.socket() + socket.socket(fileno=sock.fileno()).close() + try: + sock.close() + except OSError as err: + # Winsock apparently raises ENOTSOCK + self.assertIn(err.errno, (errno.EBADF, errno.ENOTSOCK)) + else: + self.fail("close() should raise EBADF/ENOTSOCK") + def testNewAttributes(self): # testing .family, .type and .protocol @@ -1207,6 +1238,22 @@ class GeneralModuleTests(unittest.TestCase): self.assertRaises(ValueError, s.ioctl, -1, None) s.ioctl(socket.SIO_KEEPALIVE_VALS, (1, 100, 100)) + @unittest.skipUnless(os.name == "nt", "Windows specific") + @unittest.skipUnless(hasattr(socket, 'SIO_LOOPBACK_FAST_PATH'), + 'Loopback fast path support required for this test') + def test_sio_loopback_fast_path(self): + s = socket.socket() + self.addCleanup(s.close) + try: + s.ioctl(socket.SIO_LOOPBACK_FAST_PATH, True) + except OSError as exc: + WSAEOPNOTSUPP = 10045 + if exc.winerror == WSAEOPNOTSUPP: + self.skipTest("SIO_LOOPBACK_FAST_PATH is defined but " + "doesn't implemented in this Windows version") + raise + self.assertRaises(TypeError, s.ioctl, socket.SIO_LOOPBACK_FAST_PATH, None) + def testGetaddrinfo(self): try: socket.getaddrinfo('localhost', 80) @@ -2843,6 +2890,7 @@ class SCMRightsTest(SendrecvmsgServerTimeoutBase): nbytes = self.sendmsgToServer([msg]) self.assertEqual(nbytes, len(msg)) + @unittest.skipIf(sys.platform == "darwin", "see issue #24725") def testFDPassEmpty(self): # Try to pass an empty FD array. Can receive either no array # or an empty array. @@ -4514,6 +4562,19 @@ class TestExceptions(unittest.TestCase): self.assertTrue(issubclass(socket.gaierror, OSError)) self.assertTrue(issubclass(socket.timeout, OSError)) + def test_setblocking_invalidfd(self): + # Regression test for issue #28471 + + sock0 = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + sock = socket.socket( + socket.AF_INET, socket.SOCK_STREAM, 0, sock0.fileno()) + sock0.close() + self.addCleanup(sock.detach) + + with self.assertRaises(OSError): + sock.setblocking(False) + + @unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') class TestLinuxAbstractNamespace(unittest.TestCase): @@ -4579,7 +4640,7 @@ class TestUnixDomain(unittest.TestCase): def bind(self, sock, path): # Bind the socket try: - sock.bind(path) + support.bind_unix_socket(sock, path) except OSError as e: if str(e) == "AF_UNIX path too long": self.skipTest( @@ -5325,6 +5386,183 @@ class SendfileUsingSendfileTest(SendfileUsingSendTest): return getattr(sock, "_sendfile_use_sendfile") +@unittest.skipUnless(HAVE_SOCKET_ALG, 'AF_ALG required') +class LinuxKernelCryptoAPI(unittest.TestCase): + # tests for AF_ALG + def create_alg(self, typ, name): + sock = socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET, 0) + try: + sock.bind((typ, name)) + except FileNotFoundError as e: + # type / algorithm is not available + sock.close() + raise unittest.SkipTest(str(e), typ, name) + else: + return sock + + def test_sha256(self): + expected = bytes.fromhex("ba7816bf8f01cfea414140de5dae2223b00361a396" + "177a9cb410ff61f20015ad") + with self.create_alg('hash', 'sha256') as algo: + op, _ = algo.accept() + with op: + op.sendall(b"abc") + self.assertEqual(op.recv(512), expected) + + op, _ = algo.accept() + with op: + op.send(b'a', socket.MSG_MORE) + op.send(b'b', socket.MSG_MORE) + op.send(b'c', socket.MSG_MORE) + op.send(b'') + self.assertEqual(op.recv(512), expected) + + def test_hmac_sha1(self): + expected = bytes.fromhex("effcdf6ae5eb2fa2d27416d5f184df9c259a7c79") + with self.create_alg('hash', 'hmac(sha1)') as algo: + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, b"Jefe") + op, _ = algo.accept() + with op: + op.sendall(b"what do ya want for nothing?") + self.assertEqual(op.recv(512), expected) + + # Although it should work with 3.19 and newer the test blocks on + # Ubuntu 15.10 with Kernel 4.2.0-19. + @support.requires_linux_version(4, 3) + def test_aes_cbc(self): + key = bytes.fromhex('06a9214036b8a15b512e03d534120006') + iv = bytes.fromhex('3dafba429d9eb430b422da802c9fac41') + msg = b"Single block msg" + ciphertext = bytes.fromhex('e353779c1079aeb82708942dbe77181a') + msglen = len(msg) + with self.create_alg('skcipher', 'cbc(aes)') as algo: + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, key) + op, _ = algo.accept() + with op: + op.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, iv=iv, + flags=socket.MSG_MORE) + op.sendall(msg) + self.assertEqual(op.recv(msglen), ciphertext) + + op, _ = algo.accept() + with op: + op.sendmsg_afalg([ciphertext], + op=socket.ALG_OP_DECRYPT, iv=iv) + self.assertEqual(op.recv(msglen), msg) + + # long message + multiplier = 1024 + longmsg = [msg] * multiplier + op, _ = algo.accept() + with op: + op.sendmsg_afalg(longmsg, + op=socket.ALG_OP_ENCRYPT, iv=iv) + enc = op.recv(msglen * multiplier) + self.assertEqual(len(enc), msglen * multiplier) + self.assertTrue(enc[:msglen], ciphertext) + + op, _ = algo.accept() + with op: + op.sendmsg_afalg([enc], + op=socket.ALG_OP_DECRYPT, iv=iv) + dec = op.recv(msglen * multiplier) + self.assertEqual(len(dec), msglen * multiplier) + self.assertEqual(dec, msg * multiplier) + + @support.requires_linux_version(4, 3) # see test_aes_cbc + def test_aead_aes_gcm(self): + key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') + iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') + plain = bytes.fromhex('c3b3c41f113a31b73d9a5cd432103069') + assoc = bytes.fromhex('24825602bd12a984e0092d3e448eda5f') + expected_ct = bytes.fromhex('93fe7d9e9bfd10348a5606e5cafa7354') + expected_tag = bytes.fromhex('0032a1dc85f1c9786925a2e71d8272dd') + + taglen = len(expected_tag) + assoclen = len(assoc) + + with self.create_alg('aead', 'gcm(aes)') as algo: + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, key) + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_AEAD_AUTHSIZE, + None, taglen) + + # send assoc, plain and tag buffer in separate steps + op, _ = algo.accept() + with op: + op.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, iv=iv, + assoclen=assoclen, flags=socket.MSG_MORE) + op.sendall(assoc, socket.MSG_MORE) + op.sendall(plain, socket.MSG_MORE) + op.sendall(b'\x00' * taglen) + res = op.recv(assoclen + len(plain) + taglen) + self.assertEqual(expected_ct, res[assoclen:-taglen]) + self.assertEqual(expected_tag, res[-taglen:]) + + # now with msg + op, _ = algo.accept() + with op: + msg = assoc + plain + b'\x00' * taglen + op.sendmsg_afalg([msg], op=socket.ALG_OP_ENCRYPT, iv=iv, + assoclen=assoclen) + res = op.recv(assoclen + len(plain) + taglen) + self.assertEqual(expected_ct, res[assoclen:-taglen]) + self.assertEqual(expected_tag, res[-taglen:]) + + # create anc data manually + pack_uint32 = struct.Struct('I').pack + op, _ = algo.accept() + with op: + msg = assoc + plain + b'\x00' * taglen + op.sendmsg( + [msg], + ([socket.SOL_ALG, socket.ALG_SET_OP, pack_uint32(socket.ALG_OP_ENCRYPT)], + [socket.SOL_ALG, socket.ALG_SET_IV, pack_uint32(len(iv)) + iv], + [socket.SOL_ALG, socket.ALG_SET_AEAD_ASSOCLEN, pack_uint32(assoclen)], + ) + ) + res = op.recv(len(msg)) + self.assertEqual(expected_ct, res[assoclen:-taglen]) + self.assertEqual(expected_tag, res[-taglen:]) + + # decrypt and verify + op, _ = algo.accept() + with op: + msg = assoc + expected_ct + expected_tag + op.sendmsg_afalg([msg], op=socket.ALG_OP_DECRYPT, iv=iv, + assoclen=assoclen) + res = op.recv(len(msg)) + self.assertEqual(plain, res[assoclen:-taglen]) + + @support.requires_linux_version(4, 3) # see test_aes_cbc + def test_drbg_pr_sha256(self): + # deterministic random bit generator, prediction resistance, sha256 + with self.create_alg('rng', 'drbg_pr_sha256') as algo: + extra_seed = os.urandom(32) + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, extra_seed) + op, _ = algo.accept() + with op: + rn = op.recv(32) + self.assertEqual(len(rn), 32) + + def test_sendmsg_afalg_args(self): + sock = socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET, 0) + with sock: + with self.assertRaises(TypeError): + sock.sendmsg_afalg() + + with self.assertRaises(TypeError): + sock.sendmsg_afalg(op=None) + + with self.assertRaises(TypeError): + sock.sendmsg_afalg(1) + + with self.assertRaises(TypeError): + sock.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, assoclen=None) + + with self.assertRaises(TypeError): + sock.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, assoclen=-1) + + def test_main(): tests = [GeneralModuleTests, BasicTCPTest, TCPCloserTest, TCPTimeoutTest, TestExceptions, BufferIOTest, BasicTCPTest2, BasicUDPTest, UDPTimeoutTest ] @@ -5351,6 +5589,7 @@ def test_main(): tests.extend([TIPCTest, TIPCThreadableTest]) tests.extend([BasicCANTest, CANTest]) tests.extend([BasicRDSTest, RDSTest]) + tests.append(LinuxKernelCryptoAPI) tests.extend([ CmsgMacroTests, SendmsgUDPTest, diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 0d0f86fca4..140a6abf9e 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -3,12 +3,11 @@ Test suite for socketserver. """ import contextlib +import io import os import select import signal import socket -import select -import errno import tempfile import unittest import socketserver @@ -46,7 +45,7 @@ def receive(sock, n, timeout=20): else: raise RuntimeError("timed out on %r" % (sock,)) -if HAVE_UNIX_SOCKETS: +if HAVE_UNIX_SOCKETS and HAVE_FORKING: class ForkingUnixStreamServer(socketserver.ForkingMixIn, socketserver.UnixStreamServer): pass @@ -58,6 +57,7 @@ if HAVE_UNIX_SOCKETS: @contextlib.contextmanager def simple_subprocess(testcase): + """Tests that a custom child process is not waited on (Issue 1540386)""" pid = os.fork() if pid == 0: # Don't raise an exception; it would be caught by the test harness. @@ -103,7 +103,6 @@ class SocketServerTest(unittest.TestCase): class MyServer(svrcls): def handle_error(self, request, client_address): self.close_request(request) - self.server_close() raise class MyHandler(hdlrbase): @@ -279,6 +278,182 @@ class SocketServerTest(unittest.TestCase): socketserver.TCPServer((HOST, -1), socketserver.StreamRequestHandler) + def test_context_manager(self): + with socketserver.TCPServer((HOST, 0), + socketserver.StreamRequestHandler) as server: + pass + self.assertEqual(-1, server.socket.fileno()) + + +class ErrorHandlerTest(unittest.TestCase): + """Test that the servers pass normal exceptions from the handler to + handle_error(), and that exiting exceptions like SystemExit and + KeyboardInterrupt are not passed.""" + + def tearDown(self): + test.support.unlink(test.support.TESTFN) + + def test_sync_handled(self): + BaseErrorTestServer(ValueError) + self.check_result(handled=True) + + def test_sync_not_handled(self): + with self.assertRaises(SystemExit): + BaseErrorTestServer(SystemExit) + self.check_result(handled=False) + + @unittest.skipUnless(threading, 'Threading required for this test.') + def test_threading_handled(self): + ThreadingErrorTestServer(ValueError) + self.check_result(handled=True) + + @unittest.skipUnless(threading, 'Threading required for this test.') + def test_threading_not_handled(self): + ThreadingErrorTestServer(SystemExit) + self.check_result(handled=False) + + @requires_forking + def test_forking_handled(self): + ForkingErrorTestServer(ValueError) + self.check_result(handled=True) + + @requires_forking + def test_forking_not_handled(self): + ForkingErrorTestServer(SystemExit) + self.check_result(handled=False) + + def check_result(self, handled): + with open(test.support.TESTFN) as log: + expected = 'Handler called\n' + 'Error handled\n' * handled + self.assertEqual(log.read(), expected) + + +class BaseErrorTestServer(socketserver.TCPServer): + def __init__(self, exception): + self.exception = exception + super().__init__((HOST, 0), BadHandler) + with socket.create_connection(self.server_address): + pass + try: + self.handle_request() + finally: + self.server_close() + self.wait_done() + + def handle_error(self, request, client_address): + with open(test.support.TESTFN, 'a') as log: + log.write('Error handled\n') + + def wait_done(self): + pass + + +class BadHandler(socketserver.BaseRequestHandler): + def handle(self): + with open(test.support.TESTFN, 'a') as log: + log.write('Handler called\n') + raise self.server.exception('Test error') + + +class ThreadingErrorTestServer(socketserver.ThreadingMixIn, + BaseErrorTestServer): + def __init__(self, *pos, **kw): + self.done = threading.Event() + super().__init__(*pos, **kw) + + def shutdown_request(self, *pos, **kw): + super().shutdown_request(*pos, **kw) + self.done.set() + + def wait_done(self): + self.done.wait() + + +if HAVE_FORKING: + class ForkingErrorTestServer(socketserver.ForkingMixIn, BaseErrorTestServer): + def wait_done(self): + [child] = self.active_children + os.waitpid(child, 0) + self.active_children.clear() + + +class SocketWriterTest(unittest.TestCase): + def test_basics(self): + class Handler(socketserver.StreamRequestHandler): + def handle(self): + self.server.wfile = self.wfile + self.server.wfile_fileno = self.wfile.fileno() + self.server.request_fileno = self.request.fileno() + + server = socketserver.TCPServer((HOST, 0), Handler) + self.addCleanup(server.server_close) + s = socket.socket( + server.address_family, socket.SOCK_STREAM, socket.IPPROTO_TCP) + with s: + s.connect(server.server_address) + server.handle_request() + self.assertIsInstance(server.wfile, io.BufferedIOBase) + self.assertEqual(server.wfile_fileno, server.request_fileno) + + @unittest.skipUnless(threading, 'Threading required for this test.') + def test_write(self): + # Test that wfile.write() sends data immediately, and that it does + # not truncate sends when interrupted by a Unix signal + pthread_kill = test.support.get_attribute(signal, 'pthread_kill') + + class Handler(socketserver.StreamRequestHandler): + def handle(self): + self.server.sent1 = self.wfile.write(b'write data\n') + # Should be sent immediately, without requiring flush() + self.server.received = self.rfile.readline() + big_chunk = b'\0' * test.support.SOCK_MAX_SIZE + self.server.sent2 = self.wfile.write(big_chunk) + + server = socketserver.TCPServer((HOST, 0), Handler) + self.addCleanup(server.server_close) + interrupted = threading.Event() + + def signal_handler(signum, frame): + interrupted.set() + + original = signal.signal(signal.SIGUSR1, signal_handler) + self.addCleanup(signal.signal, signal.SIGUSR1, original) + response1 = None + received2 = None + main_thread = threading.get_ident() + + def run_client(): + s = socket.socket(server.address_family, socket.SOCK_STREAM, + socket.IPPROTO_TCP) + with s, s.makefile('rb') as reader: + s.connect(server.server_address) + nonlocal response1 + response1 = reader.readline() + s.sendall(b'client response\n') + + reader.read(100) + # The main thread should now be blocking in a send() syscall. + # But in theory, it could get interrupted by other signals, + # and then retried. So keep sending the signal in a loop, in + # case an earlier signal happens to be delivered at an + # inconvenient moment. + while True: + pthread_kill(main_thread, signal.SIGUSR1) + if interrupted.wait(timeout=float(1)): + break + nonlocal received2 + received2 = len(reader.read()) + + background = threading.Thread(target=run_client) + background.start() + server.handle_request() + background.join() + self.assertEqual(server.sent1, len(response1)) + self.assertEqual(response1, b'write data\n') + self.assertEqual(server.received, b'client response\n') + self.assertEqual(server.sent2, test.support.SOCK_MAX_SIZE) + self.assertEqual(received2, test.support.SOCK_MAX_SIZE - 100) + class MiscTestCase(unittest.TestCase): diff --git a/Lib/test/test_sort.py b/Lib/test/test_sort.py index a5d0ebfd5a..98ccab5c39 100644 --- a/Lib/test/test_sort.py +++ b/Lib/test/test_sort.py @@ -1,6 +1,5 @@ from test import support import random -import sys import unittest from functools import cmp_to_key diff --git a/Lib/test/test_spwd.py b/Lib/test/test_spwd.py index bea7ab1ba5..e893f3a847 100644 --- a/Lib/test/test_spwd.py +++ b/Lib/test/test_spwd.py @@ -56,5 +56,20 @@ class TestSpwdRoot(unittest.TestCase): self.assertRaises(TypeError, spwd.getspnam, bytes_name) +@unittest.skipUnless(hasattr(os, 'geteuid') and os.geteuid() != 0, + 'non-root user required') +class TestSpwdNonRoot(unittest.TestCase): + + def test_getspnam_exception(self): + name = 'bin' + try: + with self.assertRaises(PermissionError) as cm: + spwd.getspnam(name) + except KeyError as exc: + self.skipTest("spwd entry %r doesn't exist: %s" % (name, exc)) + else: + self.assertEqual(str(cm.exception), '[Errno 13] Permission denied') + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 551558304f..d203cddbdf 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -21,6 +21,13 @@ import functools ssl = support.import_module("ssl") +try: + import threading +except ImportError: + _have_threads = False +else: + _have_threads = True + PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) HOST = support.HOST IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL') @@ -56,12 +63,12 @@ CRLFILE = data_file("revocation.crl") # Two keys and certs signed by the same CA (for SNI tests) SIGNED_CERTFILE = data_file("keycert3.pem") SIGNED_CERTFILE2 = data_file("keycert4.pem") -SIGNING_CA = data_file("pycacert.pem") +# Same certificate as pycacert.pem, but without extra text in file +SIGNING_CA = data_file("capath", "ceff1710.0") # cert with all kinds of subject alt names ALLSANFILE = data_file("allsans.pem") REMOTE_HOST = "self-signed.pythontest.net" -REMOTE_ROOT_CERT = data_file("selfsigned_pythontestdotnet.pem") EMPTYCERT = data_file("nullcert.pem") BADCERT = data_file("badcert.pem") @@ -73,6 +80,12 @@ NULLBYTECERT = data_file("nullbytecert.pem") DHFILE = data_file("dh1024.pem") BYTES_DHFILE = os.fsencode(DHFILE) +# Not defined in all versions of OpenSSL +OP_NO_COMPRESSION = getattr(ssl, "OP_NO_COMPRESSION", 0) +OP_SINGLE_DH_USE = getattr(ssl, "OP_SINGLE_DH_USE", 0) +OP_SINGLE_ECDH_USE = getattr(ssl, "OP_SINGLE_ECDH_USE", 0) +OP_CIPHER_SERVER_PREFERENCE = getattr(ssl, "OP_CIPHER_SERVER_PREFERENCE", 0) + def handle_error(prefix): exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) @@ -130,6 +143,21 @@ def skip_if_broken_ubuntu_ssl(func): needs_sni = unittest.skipUnless(ssl.HAS_SNI, "SNI support needed for this test") +def test_wrap_socket(sock, ssl_version=ssl.PROTOCOL_TLS, *, + cert_reqs=ssl.CERT_NONE, ca_certs=None, + ciphers=None, certfile=None, keyfile=None, + **kwargs): + context = ssl.SSLContext(ssl_version) + if cert_reqs is not None: + context.verify_mode = cert_reqs + if ca_certs is not None: + context.load_verify_locations(ca_certs) + if certfile is not None or keyfile is not None: + context.load_cert_chain(certfile, keyfile) + if ciphers is not None: + context.set_ciphers(ciphers) + return context.wrap_socket(sock, **kwargs) + class BasicSocketTests(unittest.TestCase): def test_constants(self): @@ -350,17 +378,17 @@ class BasicSocketTests(unittest.TestCase): # Issue #7943: an SSL object doesn't create reference cycles with # itself. s = socket.socket(socket.AF_INET) - ss = ssl.wrap_socket(s) + ss = test_wrap_socket(s) wr = weakref.ref(ss) with support.check_warnings(("", ResourceWarning)): del ss - self.assertEqual(wr(), None) + self.assertEqual(wr(), None) def test_wrapped_unconnected(self): # Methods on an unconnected SSLSocket propagate the original # OSError raise by the underlying socket object. s = socket.socket(socket.AF_INET) - with ssl.wrap_socket(s) as ss: + with test_wrap_socket(s) as ss: self.assertRaises(OSError, ss.recv, 1) self.assertRaises(OSError, ss.recv_into, bytearray(b'x')) self.assertRaises(OSError, ss.recvfrom, 1) @@ -374,10 +402,10 @@ class BasicSocketTests(unittest.TestCase): for timeout in (None, 0.0, 5.0): s = socket.socket(socket.AF_INET) s.settimeout(timeout) - with ssl.wrap_socket(s) as ss: + with test_wrap_socket(s) as ss: self.assertEqual(timeout, ss.gettimeout()) - def test_errors(self): + def test_errors_sslwrap(self): sock = socket.socket() self.assertRaisesRegex(ValueError, "certfile must be specified", @@ -387,10 +415,10 @@ class BasicSocketTests(unittest.TestCase): ssl.wrap_socket, sock, server_side=True) self.assertRaisesRegex(ValueError, "certfile must be specified for server-side operations", - ssl.wrap_socket, sock, server_side=True, certfile="") + ssl.wrap_socket, sock, server_side=True, certfile="") with ssl.wrap_socket(sock, server_side=True, certfile=CERTFILE) as s: self.assertRaisesRegex(ValueError, "can't connect in server-side mode", - s.connect, (HOST, 8080)) + s.connect, (HOST, 8080)) with self.assertRaises(OSError) as cm: with socket.socket() as sock: ssl.wrap_socket(sock, certfile=NONEXISTINGCERT) @@ -413,7 +441,7 @@ class BasicSocketTests(unittest.TestCase): sock = socket.socket() self.addCleanup(sock.close) with self.assertRaises(ssl.SSLError): - ssl.wrap_socket(sock, + test_wrap_socket(sock, certfile=certfile, ssl_version=ssl.PROTOCOL_TLSv1) @@ -600,7 +628,7 @@ class BasicSocketTests(unittest.TestCase): s.listen() c = socket.socket(socket.AF_INET) c.connect(s.getsockname()) - with ssl.wrap_socket(c, do_handshake_on_connect=False) as ss: + with test_wrap_socket(c, do_handshake_on_connect=False) as ss: with self.assertRaises(ValueError): ss.get_channel_binding("unknown-type") s.close() @@ -610,15 +638,15 @@ class BasicSocketTests(unittest.TestCase): def test_tls_unique_channel_binding(self): # unconnected should return None for known type s = socket.socket(socket.AF_INET) - with ssl.wrap_socket(s) as ss: + with test_wrap_socket(s) as ss: self.assertIsNone(ss.get_channel_binding("tls-unique")) # the same for server-side s = socket.socket(socket.AF_INET) - with ssl.wrap_socket(s, server_side=True, certfile=CERTFILE) as ss: + with test_wrap_socket(s, server_side=True, certfile=CERTFILE) as ss: self.assertIsNone(ss.get_channel_binding("tls-unique")) def test_dealloc_warn(self): - ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) + ss = test_wrap_socket(socket.socket(socket.AF_INET)) r = repr(ss) with self.assertWarns(ResourceWarning) as cm: ss = None @@ -737,7 +765,7 @@ class BasicSocketTests(unittest.TestCase): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.addCleanup(s.close) with self.assertRaises(NotImplementedError) as cx: - ssl.wrap_socket(s, cert_reqs=ssl.CERT_NONE) + test_wrap_socket(s, cert_reqs=ssl.CERT_NONE) self.assertEqual(str(cx.exception), "only stream sockets are supported") ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) with self.assertRaises(NotImplementedError) as cx: @@ -809,6 +837,22 @@ class BasicSocketTests(unittest.TestCase): self.cert_time_ok("Feb 9 00:00:00 2007 GMT", 1170979200.0) self.cert_time_fail(local_february_name() + " 9 00:00:00 2007 GMT") + def test_connect_ex_error(self): + server = socket.socket(socket.AF_INET) + self.addCleanup(server.close) + port = support.bind_port(server) # Reserve port but don't listen + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) + rc = s.connect_ex((HOST, port)) + # Issue #19919: Windows machines or VMs hosted on Windows + # machines sometimes return EWOULDBLOCK. + errors = ( + errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, + errno.EWOULDBLOCK, + ) + self.assertIn(rc, errors) + class ContextTests(unittest.TestCase): @@ -834,13 +878,22 @@ class ContextTests(unittest.TestCase): with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): ctx.set_ciphers("^$:,;?*'dorothyx") + @unittest.skipIf(ssl.OPENSSL_VERSION_INFO < (1, 0, 2, 0, 0), 'OpenSSL too old') + def test_get_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ciphers('AESGCM') + names = set(d['name'] for d in ctx.get_ciphers()) + self.assertIn('AES256-GCM-SHA384', names) + self.assertIn('AES128-GCM-SHA256', names) + @skip_if_broken_ubuntu_ssl def test_options(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) - if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0): - default |= ssl.OP_NO_COMPRESSION + # SSLContext also enables these by default + default |= (OP_NO_COMPRESSION | OP_CIPHER_SERVER_PREFERENCE | + OP_SINGLE_DH_USE | OP_SINGLE_ECDH_USE) self.assertEqual(default, ctx.options) ctx.options |= ssl.OP_NO_TLSv1 self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options) @@ -1205,16 +1258,29 @@ class ContextTests(unittest.TestCase): stats["x509"] += 1 self.assertEqual(ctx.cert_store_stats(), stats) + def _assert_context_options(self, ctx): + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + if OP_NO_COMPRESSION != 0: + self.assertEqual(ctx.options & OP_NO_COMPRESSION, + OP_NO_COMPRESSION) + if OP_SINGLE_DH_USE != 0: + self.assertEqual(ctx.options & OP_SINGLE_DH_USE, + OP_SINGLE_DH_USE) + if OP_SINGLE_ECDH_USE != 0: + self.assertEqual(ctx.options & OP_SINGLE_ECDH_USE, + OP_SINGLE_ECDH_USE) + if OP_CIPHER_SERVER_PREFERENCE != 0: + self.assertEqual(ctx.options & OP_CIPHER_SERVER_PREFERENCE, + OP_CIPHER_SERVER_PREFERENCE) + def test_create_default_context(self): ctx = ssl.create_default_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) self.assertTrue(ctx.check_hostname) - self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) - self.assertEqual( - ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), - getattr(ssl, "OP_NO_COMPRESSION", 0), - ) + self._assert_context_options(ctx) + with open(SIGNING_CA) as f: cadata = f.read() @@ -1222,40 +1288,24 @@ class ContextTests(unittest.TestCase): cadata=cadata) self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) - self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) - self.assertEqual( - ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), - getattr(ssl, "OP_NO_COMPRESSION", 0), - ) + self._assert_context_options(ctx) ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) - self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) - self.assertEqual( - ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), - getattr(ssl, "OP_NO_COMPRESSION", 0), - ) - self.assertEqual( - ctx.options & getattr(ssl, "OP_SINGLE_DH_USE", 0), - getattr(ssl, "OP_SINGLE_DH_USE", 0), - ) - self.assertEqual( - ctx.options & getattr(ssl, "OP_SINGLE_ECDH_USE", 0), - getattr(ssl, "OP_SINGLE_ECDH_USE", 0), - ) + self._assert_context_options(ctx) def test__create_stdlib_context(self): ctx = ssl._create_stdlib_context() self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) self.assertFalse(ctx.check_hostname) - self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self._assert_context_options(ctx) ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1) self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) - self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self._assert_context_options(ctx) ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1, cert_reqs=ssl.CERT_REQUIRED, @@ -1263,12 +1313,12 @@ class ContextTests(unittest.TestCase): self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) self.assertTrue(ctx.check_hostname) - self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self._assert_context_options(ctx) ctx = ssl._create_stdlib_context(purpose=ssl.Purpose.CLIENT_AUTH) self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) - self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self._assert_context_options(ctx) def test_check_hostname(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) @@ -1292,6 +1342,17 @@ class ContextTests(unittest.TestCase): ctx.check_hostname = False self.assertFalse(ctx.check_hostname) + def test_context_client_server(self): + # PROTOCOL_TLS_CLIENT has sane defaults + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + + # PROTOCOL_TLS_SERVER has different but also sane defaults + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + class SSLErrorTests(unittest.TestCase): @@ -1397,140 +1458,105 @@ class MemoryBIOTests(unittest.TestCase): self.assertRaises(TypeError, bio.write, 1) -class NetworkedTests(unittest.TestCase): +@unittest.skipUnless(_have_threads, "Needs threading module") +class SimpleBackgroundTests(unittest.TestCase): - def test_connect(self): - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_NONE) - try: - s.connect((REMOTE_HOST, 443)) - self.assertEqual({}, s.getpeercert()) - finally: - s.close() + """Tests that connect to a simple server running in the background""" - # this should fail because we have no verification certs - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED) - self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", - s.connect, (REMOTE_HOST, 443)) - s.close() + def setUp(self): + server = ThreadedEchoServer(SIGNED_CERTFILE) + self.server_addr = (HOST, server.port) + server.__enter__() + self.addCleanup(server.__exit__, None, None, None) - # this should succeed because we specify the root cert - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT) - try: - s.connect((REMOTE_HOST, 443)) - self.assertTrue(s.getpeercert()) - finally: - s.close() + def test_connect(self): + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE) as s: + s.connect(self.server_addr) + self.assertEqual({}, s.getpeercert()) + self.assertFalse(s.server_side) + + # this should succeed because we specify the root cert + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA) as s: + s.connect(self.server_addr) + self.assertTrue(s.getpeercert()) + self.assertFalse(s.server_side) + + def test_connect_fail(self): + # This should fail because we have no verification certs. Connection + # failure crashes ThreadedEchoServer, so run this in an independent + # test method. + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, self.server_addr) def test_connect_ex(self): # Issue #11326: check connect_ex() implementation - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT) - try: - self.assertEqual(0, s.connect_ex((REMOTE_HOST, 443))) - self.assertTrue(s.getpeercert()) - finally: - s.close() + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA) + self.addCleanup(s.close) + self.assertEqual(0, s.connect_ex(self.server_addr)) + self.assertTrue(s.getpeercert()) def test_non_blocking_connect_ex(self): # Issue #11326: non-blocking connect_ex() should allow handshake # to proceed after the socket gets ready. - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT, - do_handshake_on_connect=False) + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA, + do_handshake_on_connect=False) + self.addCleanup(s.close) + s.setblocking(False) + rc = s.connect_ex(self.server_addr) + # EWOULDBLOCK under Windows, EINPROGRESS elsewhere + self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) + # Wait for connect to finish + select.select([], [s], [], 5.0) + # Non-blocking handshake + while True: try: - s.setblocking(False) - rc = s.connect_ex((REMOTE_HOST, 443)) - # EWOULDBLOCK under Windows, EINPROGRESS elsewhere - self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) - # Wait for connect to finish + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], [], 5.0) + except ssl.SSLWantWriteError: select.select([], [s], [], 5.0) - # Non-blocking handshake - while True: - try: - s.do_handshake() - break - except ssl.SSLWantReadError: - select.select([s], [], [], 5.0) - except ssl.SSLWantWriteError: - select.select([], [s], [], 5.0) - # SSL established - self.assertTrue(s.getpeercert()) - finally: - s.close() - - def test_timeout_connect_ex(self): - # Issue #12065: on a timeout, connect_ex() should return the original - # errno (mimicking the behaviour of non-SSL sockets). - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT, - do_handshake_on_connect=False) - try: - s.settimeout(0.0000001) - rc = s.connect_ex((REMOTE_HOST, 443)) - if rc == 0: - self.skipTest("REMOTE_HOST responded too quickly") - self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) - finally: - s.close() - - def test_connect_ex_error(self): - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT) - try: - rc = s.connect_ex((REMOTE_HOST, 444)) - # Issue #19919: Windows machines or VMs hosted on Windows - # machines sometimes return EWOULDBLOCK. - errors = ( - errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, - errno.EWOULDBLOCK, - ) - self.assertIn(rc, errors) - finally: - s.close() + # SSL established + self.assertTrue(s.getpeercert()) def test_connect_with_context(self): - with support.transient_internet(REMOTE_HOST): - # Same as test_connect, but with a separately created context - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - self.assertEqual({}, s.getpeercert()) - finally: - s.close() - # Same with a server hostname - s = ctx.wrap_socket(socket.socket(socket.AF_INET), - server_hostname=REMOTE_HOST) - s.connect((REMOTE_HOST, 443)) - s.close() - # This should fail because we have no verification certs - ctx.verify_mode = ssl.CERT_REQUIRED - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", - s.connect, (REMOTE_HOST, 443)) - s.close() - # This should succeed because we specify the root cert - ctx.load_verify_locations(REMOTE_ROOT_CERT) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - cert = s.getpeercert() - self.assertTrue(cert) - finally: - s.close() + # Same as test_connect, but with a separately created context + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + self.assertEqual({}, s.getpeercert()) + # Same with a server hostname + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="dummy") as s: + s.connect(self.server_addr) + ctx.verify_mode = ssl.CERT_REQUIRED + # This should succeed because we specify the root cert + ctx.load_verify_locations(SIGNING_CA) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + def test_connect_with_context_fail(self): + # This should fail because we have no verification certs. Connection + # failure crashes ThreadedEchoServer, so run this in an independent + # test method. + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + self.addCleanup(s.close) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, self.server_addr) def test_connect_capath(self): # Verify server certificates using the `capath` argument @@ -1538,198 +1564,130 @@ class NetworkedTests(unittest.TestCase): # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must # contain both versions of each certificate (same content, different # filename) for this test to be portable across OpenSSL releases. - with support.transient_internet(REMOTE_HOST): - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(capath=CAPATH) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - cert = s.getpeercert() - self.assertTrue(cert) - finally: - s.close() - # Same with a bytes `capath` argument - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(capath=BYTES_CAPATH) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - cert = s.getpeercert() - self.assertTrue(cert) - finally: - s.close() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + # Same with a bytes `capath` argument + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=BYTES_CAPATH) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) def test_connect_cadata(self): - with open(REMOTE_ROOT_CERT) as f: + with open(SIGNING_CA) as f: pem = f.read() der = ssl.PEM_cert_to_DER_cert(pem) - with support.transient_internet(REMOTE_HOST): - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(cadata=pem) - with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: - s.connect((REMOTE_HOST, 443)) - cert = s.getpeercert() - self.assertTrue(cert) + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=pem) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) - # same with DER - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(cadata=der) - with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: - s.connect((REMOTE_HOST, 443)) - cert = s.getpeercert() - self.assertTrue(cert) + # same with DER + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=der) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) @unittest.skipIf(os.name == "nt", "Can't use a socket as a file under Windows") def test_makefile_close(self): # Issue #5238: creating a file-like object with makefile() shouldn't # delay closing the underlying "real socket" (here tested with its # file descriptor, hence skipping the test under Windows). - with support.transient_internet(REMOTE_HOST): - ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) - ss.connect((REMOTE_HOST, 443)) - fd = ss.fileno() - f = ss.makefile() - f.close() - # The fd is still open + ss = test_wrap_socket(socket.socket(socket.AF_INET)) + ss.connect(self.server_addr) + fd = ss.fileno() + f = ss.makefile() + f.close() + # The fd is still open + os.read(fd, 0) + # Closing the SSL socket should close the fd too + ss.close() + gc.collect() + with self.assertRaises(OSError) as e: os.read(fd, 0) - # Closing the SSL socket should close the fd too - ss.close() - gc.collect() - with self.assertRaises(OSError) as e: - os.read(fd, 0) - self.assertEqual(e.exception.errno, errno.EBADF) + self.assertEqual(e.exception.errno, errno.EBADF) def test_non_blocking_handshake(self): - with support.transient_internet(REMOTE_HOST): - s = socket.socket(socket.AF_INET) - s.connect((REMOTE_HOST, 443)) - s.setblocking(False) - s = ssl.wrap_socket(s, - cert_reqs=ssl.CERT_NONE, - do_handshake_on_connect=False) - count = 0 - while True: - try: - count += 1 - s.do_handshake() - break - except ssl.SSLWantReadError: - select.select([s], [], []) - except ssl.SSLWantWriteError: - select.select([], [s], []) - s.close() - if support.verbose: - sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) + s = socket.socket(socket.AF_INET) + s.connect(self.server_addr) + s.setblocking(False) + s = test_wrap_socket(s, + cert_reqs=ssl.CERT_NONE, + do_handshake_on_connect=False) + self.addCleanup(s.close) + count = 0 + while True: + try: + count += 1 + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], []) + except ssl.SSLWantWriteError: + select.select([], [s], []) + if support.verbose: + sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) def test_get_server_certificate(self): - def _test_get_server_certificate(host, port, cert=None): - with support.transient_internet(host): - pem = ssl.get_server_certificate((host, port)) - if not pem: - self.fail("No server certificate on %s:%s!" % (host, port)) + _test_get_server_certificate(self, *self.server_addr, cert=SIGNING_CA) - try: - pem = ssl.get_server_certificate((host, port), - ca_certs=CERTFILE) - except ssl.SSLError as x: - #should fail - if support.verbose: - sys.stdout.write("%s\n" % x) - else: - self.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) - - pem = ssl.get_server_certificate((host, port), - ca_certs=cert) - if not pem: - self.fail("No server certificate on %s:%s!" % (host, port)) - if support.verbose: - sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) - - _test_get_server_certificate(REMOTE_HOST, 443, REMOTE_ROOT_CERT) - if support.IPV6_ENABLED: - _test_get_server_certificate('ipv6.google.com', 443) + def test_get_server_certificate_fail(self): + # Connection failure crashes ThreadedEchoServer, so run this in an + # independent test method + _test_get_server_certificate_fail(self, *self.server_addr) def test_ciphers(self): - remote = (REMOTE_HOST, 443) - with support.transient_internet(remote[0]): - with ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s: - s.connect(remote) - with ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s: - s.connect(remote) - # Error checking can happen at instantiation or when connecting - with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): - with socket.socket(socket.AF_INET) as sock: - s = ssl.wrap_socket(sock, - cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") - s.connect(remote) - - def test_algorithms(self): - # Issue #8484: all algorithms should be available when verifying a - # certificate. - # SHA256 was added in OpenSSL 0.9.8 - if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15): - self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION) - # sha256.tbs-internet.com needs SNI to use the correct certificate - if not ssl.HAS_SNI: - self.skipTest("SNI needed for this test") - # https://sha2.hboeck.de/ was used until 2011-01-08 (no route to host) - remote = ("sha256.tbs-internet.com", 443) - sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem") - with support.transient_internet("sha256.tbs-internet.com"): - ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(sha256_cert) - s = ctx.wrap_socket(socket.socket(socket.AF_INET), - server_hostname="sha256.tbs-internet.com") - try: - s.connect(remote) - if support.verbose: - sys.stdout.write("\nCipher with %r is %r\n" % - (remote, s.cipher())) - sys.stdout.write("Certificate is:\n%s\n" % - pprint.pformat(s.getpeercert())) - finally: - s.close() + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s: + s.connect(self.server_addr) + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s: + s.connect(self.server_addr) + # Error checking can happen at instantiation or when connecting + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + with socket.socket(socket.AF_INET) as sock: + s = test_wrap_socket(sock, + cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") + s.connect(self.server_addr) def test_get_ca_certs_capath(self): # capath certs are loaded on request - with support.transient_internet(REMOTE_HOST): - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(capath=CAPATH) - self.assertEqual(ctx.get_ca_certs(), []) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - cert = s.getpeercert() - self.assertTrue(cert) - finally: - s.close() - self.assertEqual(len(ctx.get_ca_certs()), 1) + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + self.assertEqual(ctx.get_ca_certs(), []) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + self.assertEqual(len(ctx.get_ca_certs()), 1) @needs_sni def test_context_setget(self): # Check that the context of a connected socket can be replaced. - with support.transient_internet(REMOTE_HOST): - ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - s = socket.socket(socket.AF_INET) - with ctx1.wrap_socket(s) as ss: - ss.connect((REMOTE_HOST, 443)) - self.assertIs(ss.context, ctx1) - self.assertIs(ss._sslobj.context, ctx1) - ss.context = ctx2 - self.assertIs(ss.context, ctx2) - self.assertIs(ss._sslobj.context, ctx2) - - -class NetworkedBIOTests(unittest.TestCase): + ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = socket.socket(socket.AF_INET) + with ctx1.wrap_socket(s) as ss: + ss.connect(self.server_addr) + self.assertIs(ss.context, ctx1) + self.assertIs(ss._sslobj.context, ctx1) + ss.context = ctx2 + self.assertIs(ss.context, ctx2) + self.assertIs(ss._sslobj.context, ctx2) def ssl_io_loop(self, sock, incoming, outgoing, func, *args, **kwargs): # A simple IO loop. Call func(*args) depending on the error we get @@ -1765,64 +1723,128 @@ class NetworkedBIOTests(unittest.TestCase): % (count, func.__name__)) return ret - def test_handshake(self): + def test_bio_handshake(self): + sock = socket.socket(socket.AF_INET) + self.addCleanup(sock.close) + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(SIGNING_CA) + ctx.check_hostname = True + sslobj = ctx.wrap_bio(incoming, outgoing, False, 'localhost') + self.assertIs(sslobj._sslobj.owner, sslobj) + self.assertIsNone(sslobj.cipher()) + self.assertIsNotNone(sslobj.shared_ciphers()) + self.assertRaises(ValueError, sslobj.getpeercert) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertIsNone(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + self.assertTrue(sslobj.cipher()) + self.assertIsNotNone(sslobj.shared_ciphers()) + self.assertTrue(sslobj.getpeercert()) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertTrue(sslobj.get_channel_binding('tls-unique')) + try: + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + except ssl.SSLSyscallError: + # If the server shuts down the TCP connection without sending a + # secure shutdown message, this is reported as SSL_ERROR_SYSCALL + pass + self.assertRaises(ssl.SSLError, sslobj.write, b'foo') + + def test_bio_read_write_data(self): + sock = socket.socket(socket.AF_INET) + self.addCleanup(sock.close) + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_NONE + sslobj = ctx.wrap_bio(incoming, outgoing, False) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + req = b'FOO\n' + self.ssl_io_loop(sock, incoming, outgoing, sslobj.write, req) + buf = self.ssl_io_loop(sock, incoming, outgoing, sslobj.read, 1024) + self.assertEqual(buf, b'foo\n') + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + + +class NetworkedTests(unittest.TestCase): + + def test_timeout_connect_ex(self): + # Issue #12065: on a timeout, connect_ex() should return the original + # errno (mimicking the behaviour of non-SSL sockets). with support.transient_internet(REMOTE_HOST): - sock = socket.socket(socket.AF_INET) - sock.connect((REMOTE_HOST, 443)) - incoming = ssl.MemoryBIO() - outgoing = ssl.MemoryBIO() - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + do_handshake_on_connect=False) + self.addCleanup(s.close) + s.settimeout(0.0000001) + rc = s.connect_ex((REMOTE_HOST, 443)) + if rc == 0: + self.skipTest("REMOTE_HOST responded too quickly") + self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) + + @unittest.skipUnless(support.IPV6_ENABLED, 'Needs IPv6') + def test_get_server_certificate_ipv6(self): + with support.transient_internet('ipv6.google.com'): + _test_get_server_certificate(self, 'ipv6.google.com', 443) + _test_get_server_certificate_fail(self, 'ipv6.google.com', 443) + + def test_algorithms(self): + # Issue #8484: all algorithms should be available when verifying a + # certificate. + # SHA256 was added in OpenSSL 0.9.8 + if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15): + self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION) + # sha256.tbs-internet.com needs SNI to use the correct certificate + if not ssl.HAS_SNI: + self.skipTest("SNI needed for this test") + # https://sha2.hboeck.de/ was used until 2011-01-08 (no route to host) + remote = ("sha256.tbs-internet.com", 443) + sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem") + with support.transient_internet("sha256.tbs-internet.com"): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(REMOTE_ROOT_CERT) - ctx.check_hostname = True - sslobj = ctx.wrap_bio(incoming, outgoing, False, REMOTE_HOST) - self.assertIs(sslobj._sslobj.owner, sslobj) - self.assertIsNone(sslobj.cipher()) - self.assertIsNotNone(sslobj.shared_ciphers()) - self.assertRaises(ValueError, sslobj.getpeercert) - if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: - self.assertIsNone(sslobj.get_channel_binding('tls-unique')) - self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) - self.assertTrue(sslobj.cipher()) - self.assertIsNotNone(sslobj.shared_ciphers()) - self.assertTrue(sslobj.getpeercert()) - if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: - self.assertTrue(sslobj.get_channel_binding('tls-unique')) + ctx.load_verify_locations(sha256_cert) + s = ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="sha256.tbs-internet.com") try: - self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) - except ssl.SSLSyscallError: - # self-signed.pythontest.net probably shuts down the TCP - # connection without sending a secure shutdown message, and - # this is reported as SSL_ERROR_SYSCALL - pass - self.assertRaises(ssl.SSLError, sslobj.write, b'foo') - sock.close() + s.connect(remote) + if support.verbose: + sys.stdout.write("\nCipher with %r is %r\n" % + (remote, s.cipher())) + sys.stdout.write("Certificate is:\n%s\n" % + pprint.pformat(s.getpeercert())) + finally: + s.close() - def test_read_write_data(self): - with support.transient_internet(REMOTE_HOST): - sock = socket.socket(socket.AF_INET) - sock.connect((REMOTE_HOST, 443)) - incoming = ssl.MemoryBIO() - outgoing = ssl.MemoryBIO() - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_NONE - sslobj = ctx.wrap_bio(incoming, outgoing, False) - self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) - req = b'GET / HTTP/1.0\r\n\r\n' - self.ssl_io_loop(sock, incoming, outgoing, sslobj.write, req) - buf = self.ssl_io_loop(sock, incoming, outgoing, sslobj.read, 1024) - self.assertEqual(buf[:5], b'HTTP/') - self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) - sock.close() +def _test_get_server_certificate(test, host, port, cert=None): + pem = ssl.get_server_certificate((host, port)) + if not pem: + test.fail("No server certificate on %s:%s!" % (host, port)) + + pem = ssl.get_server_certificate((host, port), ca_certs=cert) + if not pem: + test.fail("No server certificate on %s:%s!" % (host, port)) + if support.verbose: + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) + +def _test_get_server_certificate_fail(test, host, port): + try: + pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) -try: - import threading -except ImportError: - _have_threads = False -else: - _have_threads = True +if _have_threads: from test.ssl_servers import make_https_server class ThreadedEchoServer(threading.Thread): @@ -1910,6 +1932,15 @@ else: if not stripped: # eof, so quit this handler self.running = False + try: + self.sock = self.sslconn.unwrap() + except OSError: + # Many tests shut the TCP connection down + # without an SSL shutdown. This causes + # unwrap() to raise OSError with errno=0! + pass + else: + self.sslconn = None self.close() elif stripped == b'over': if support.verbose and self.server.connectionchatty: @@ -2037,7 +2068,7 @@ else: class ConnectionHandler (asyncore.dispatcher_with_send): def __init__(self, conn, certfile): - self.socket = ssl.wrap_socket(conn, server_side=True, + self.socket = test_wrap_socket(conn, server_side=True, certfile=certfile, do_handshake_on_connect=False) asyncore.dispatcher_with_send.__init__(self, self.socket) @@ -2145,7 +2176,8 @@ else: self.server.close() def server_params_test(client_context, server_context, indata=b"FOO\n", - chatty=True, connectionchatty=False, sni_name=None): + chatty=True, connectionchatty=False, sni_name=None, + session=None): """ Launch a server, connect a client to it and try various reads and writes. @@ -2156,7 +2188,7 @@ else: connectionchatty=False) with server: with client_context.wrap_socket(socket.socket(), - server_hostname=sni_name) as s: + server_hostname=sni_name, session=session) as s: s.connect((HOST, server.port)) for arg in [indata, bytearray(indata), memoryview(indata)]: if connectionchatty: @@ -2184,6 +2216,8 @@ else: 'client_alpn_protocol': s.selected_alpn_protocol(), 'client_npn_protocol': s.selected_npn_protocol(), 'version': s.version(), + 'session_reused': s.session_reused, + 'session': s.session, }) s.close() stats['server_alpn_protocols'] = server.selected_alpn_protocols @@ -2259,12 +2293,53 @@ else: if support.verbose: sys.stdout.write("\n") for protocol in PROTOCOLS: + if protocol in {ssl.PROTOCOL_TLS_CLIENT, ssl.PROTOCOL_TLS_SERVER}: + continue with self.subTest(protocol=ssl._PROTOCOL_NAMES[protocol]): context = ssl.SSLContext(protocol) context.load_cert_chain(CERTFILE) server_params_test(context, context, chatty=True, connectionchatty=True) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.load_verify_locations(SIGNING_CA) + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # server_context.load_verify_locations(SIGNING_CA) + server_context.load_cert_chain(SIGNED_CERTFILE2) + + with self.subTest(client=ssl.PROTOCOL_TLS_CLIENT, server=ssl.PROTOCOL_TLS_SERVER): + server_params_test(client_context=client_context, + server_context=server_context, + chatty=True, connectionchatty=True, + sni_name='fakehostname') + + client_context.check_hostname = False + with self.subTest(client=ssl.PROTOCOL_TLS_SERVER, server=ssl.PROTOCOL_TLS_CLIENT): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=client_context, + chatty=True, connectionchatty=True, + sni_name='fakehostname') + self.assertIn('called a function you should not call', + str(e.exception)) + + with self.subTest(client=ssl.PROTOCOL_TLS_SERVER, server=ssl.PROTOCOL_TLS_SERVER): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=server_context, + chatty=True, connectionchatty=True) + self.assertIn('called a function you should not call', + str(e.exception)) + + with self.subTest(client=ssl.PROTOCOL_TLS_CLIENT, server=ssl.PROTOCOL_TLS_CLIENT): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=client_context, + chatty=True, connectionchatty=True) + self.assertIn('called a function you should not call', + str(e.exception)) + + def test_getpeercert(self): if support.verbose: sys.stdout.write("\n") @@ -2398,7 +2473,7 @@ else: connectionchatty=False) with server, \ socket.socket() as sock, \ - ssl.wrap_socket(sock, + test_wrap_socket(sock, certfile=certfile, ssl_version=ssl.PROTOCOL_TLSv1) as s: try: @@ -2445,7 +2520,7 @@ else: c.connect((HOST, port)) listener_gone.wait() try: - ssl_sock = ssl.wrap_socket(c) + ssl_sock = test_wrap_socket(c) except OSError: pass else: @@ -2635,7 +2710,7 @@ else: sys.stdout.write( " client: read %r from server, starting TLS...\n" % msg) - conn = ssl.wrap_socket(s, ssl_version=ssl.PROTOCOL_TLSv1) + conn = test_wrap_socket(s, ssl_version=ssl.PROTOCOL_TLSv1) wrapped = True elif indata == b"ENDTLS" and msg.startswith(b"ok"): # ENDTLS ok, switch back to clear text @@ -2694,7 +2769,7 @@ else: indata = b"FOO\n" server = AsyncoreEchoServer(CERTFILE) with server: - s = ssl.wrap_socket(socket.socket()) + s = test_wrap_socket(socket.socket()) s.connect(('127.0.0.1', server.port)) if support.verbose: sys.stdout.write( @@ -2727,7 +2802,7 @@ else: chatty=True, connectionchatty=False) with server: - s = ssl.wrap_socket(socket.socket(), + s = test_wrap_socket(socket.socket(), server_side=False, certfile=CERTFILE, ca_certs=CERTFILE, @@ -2745,12 +2820,13 @@ else: count, addr = s.recvfrom_into(b) return b[:count] - # (name, method, whether to expect success, *args) + # (name, method, expect success?, *args, return value func) send_methods = [ - ('send', s.send, True, []), - ('sendto', s.sendto, False, ["some.address"]), - ('sendall', s.sendall, True, []), + ('send', s.send, True, [], len), + ('sendto', s.sendto, False, ["some.address"], len), + ('sendall', s.sendall, True, [], lambda x: None), ] + # (name, method, whether to expect success, *args) recv_methods = [ ('recv', s.recv, True, []), ('recvfrom', s.recvfrom, False, ["some.address"]), @@ -2759,10 +2835,13 @@ else: ] data_prefix = "PREFIX_" - for meth_name, send_meth, expect_success, args in send_methods: + for (meth_name, send_meth, expect_success, args, + ret_val_meth) in send_methods: indata = (data_prefix + meth_name).encode('ascii') try: - send_meth(indata, *args) + ret = send_meth(indata, *args) + msg = "sending with {}".format(meth_name) + self.assertEqual(ret, ret_val_meth(indata), msg=msg) outdata = s.read() if outdata != indata.lower(): self.fail( @@ -2847,7 +2926,7 @@ else: self.addCleanup(server.__exit__, None, None) s = socket.create_connection((HOST, server.port)) self.addCleanup(s.close) - s = ssl.wrap_socket(s, suppress_ragged_eofs=False) + s = test_wrap_socket(s, suppress_ragged_eofs=False) self.addCleanup(s.close) # recv/read(0) should return no data @@ -2869,7 +2948,7 @@ else: chatty=True, connectionchatty=False) with server: - s = ssl.wrap_socket(socket.socket(), + s = test_wrap_socket(socket.socket(), server_side=False, certfile=CERTFILE, ca_certs=CERTFILE, @@ -2923,12 +3002,12 @@ else: c.connect((host, port)) # Will attempt handshake and time out self.assertRaisesRegex(socket.timeout, "timed out", - ssl.wrap_socket, c) + test_wrap_socket, c) finally: c.close() try: c = socket.socket(socket.AF_INET) - c = ssl.wrap_socket(c) + c = test_wrap_socket(c) c.settimeout(0.2) # Will attempt handshake and time out self.assertRaisesRegex(socket.timeout, "timed out", @@ -2951,6 +3030,7 @@ else: host = "127.0.0.1" port = support.bind_port(server) server = context.wrap_socket(server, server_side=True) + self.assertTrue(server.server_side) evt = threading.Event() remote = None @@ -3053,7 +3133,7 @@ else: chatty=True, connectionchatty=False) with server: - s = ssl.wrap_socket(socket.socket(), + s = test_wrap_socket(socket.socket(), server_side=False, certfile=CERTFILE, ca_certs=CERTFILE, @@ -3078,7 +3158,7 @@ else: s.close() # now, again - s = ssl.wrap_socket(socket.socket(), + s = test_wrap_socket(socket.socket(), server_side=False, certfile=CERTFILE, ca_certs=CERTFILE, @@ -3388,6 +3468,111 @@ else: s.sendfile(file) self.assertEqual(s.recv(1024), TEST_DATA) + def test_session(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.verify_mode = ssl.CERT_REQUIRED + client_context.load_verify_locations(SIGNING_CA) + + # first connection without session + stats = server_params_test(client_context, server_context) + session = stats['session'] + self.assertTrue(session.id) + self.assertGreater(session.time, 0) + self.assertGreater(session.timeout, 0) + self.assertTrue(session.has_ticket) + if ssl.OPENSSL_VERSION_INFO > (1, 0, 1): + self.assertGreater(session.ticket_lifetime_hint, 0) + self.assertFalse(stats['session_reused']) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 1) + self.assertEqual(sess_stat['hits'], 0) + + # reuse session + stats = server_params_test(client_context, server_context, session=session) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 2) + self.assertEqual(sess_stat['hits'], 1) + self.assertTrue(stats['session_reused']) + session2 = stats['session'] + self.assertEqual(session2.id, session.id) + self.assertEqual(session2, session) + self.assertIsNot(session2, session) + self.assertGreaterEqual(session2.time, session.time) + self.assertGreaterEqual(session2.timeout, session.timeout) + + # another one without session + stats = server_params_test(client_context, server_context) + self.assertFalse(stats['session_reused']) + session3 = stats['session'] + self.assertNotEqual(session3.id, session.id) + self.assertNotEqual(session3, session) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 3) + self.assertEqual(sess_stat['hits'], 1) + + # reuse session again + stats = server_params_test(client_context, server_context, session=session) + self.assertTrue(stats['session_reused']) + session4 = stats['session'] + self.assertEqual(session4.id, session.id) + self.assertEqual(session4, session) + self.assertGreaterEqual(session4.time, session.time) + self.assertGreaterEqual(session4.timeout, session.timeout) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 4) + self.assertEqual(sess_stat['hits'], 2) + + def test_session_handling(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + + context2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context2.verify_mode = ssl.CERT_REQUIRED + context2.load_verify_locations(CERTFILE) + context2.load_cert_chain(CERTFILE) + + server = ThreadedEchoServer(context=context, chatty=False) + with server: + with context.wrap_socket(socket.socket()) as s: + # session is None before handshake + self.assertEqual(s.session, None) + self.assertEqual(s.session_reused, None) + s.connect((HOST, server.port)) + session = s.session + self.assertTrue(session) + with self.assertRaises(TypeError) as e: + s.session = object + self.assertEqual(str(e.exception), 'Value is not a SSLSession.') + + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + # cannot set session after handshake + with self.assertRaises(ValueError) as e: + s.session = session + self.assertEqual(str(e.exception), + 'Cannot set session after handshake.') + + with context.wrap_socket(socket.socket()) as s: + # can set session before handshake and before the + # connection was established + s.session = session + s.connect((HOST, server.port)) + self.assertEqual(s.session.id, session.id) + self.assertEqual(s.session, session) + self.assertEqual(s.session_reused, True) + + with context2.wrap_socket(socket.socket()) as s: + # cannot re-use session with a different SSLContext + with self.assertRaises(ValueError) as e: + s.session = session + s.connect((HOST, server.port)) + self.assertEqual(str(e.exception), + 'Session refers to a different SSLContext.') + def test_main(verbose=False): if support.verbose: @@ -3400,7 +3585,7 @@ def test_main(verbose=False): with warnings.catch_warnings(): warnings.filterwarnings( 'ignore', - 'dist\(\) and linux_distribution\(\) ' + r'dist\(\) and linux_distribution\(\) ' 'functions are deprecated .*', PendingDeprecationWarning, ) @@ -3422,18 +3607,20 @@ def test_main(verbose=False): pass for filename in [ - CERTFILE, REMOTE_ROOT_CERT, BYTES_CERTFILE, + CERTFILE, BYTES_CERTFILE, ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY, SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA, BADCERT, BADKEY, EMPTYCERT]: if not os.path.exists(filename): raise support.TestFailed("Can't read certificate file %r" % filename) - tests = [ContextTests, BasicSocketTests, SSLErrorTests, MemoryBIOTests] + tests = [ + ContextTests, BasicSocketTests, SSLErrorTests, MemoryBIOTests, + SimpleBackgroundTests, + ] if support.is_resource_enabled('network'): tests.append(NetworkedTests) - tests.append(NetworkedBIOTests) if _have_threads: thread_info = support.threading_setup() diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index f1a5938a39..cd02a6ee37 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -1,7 +1,7 @@ import unittest import os import sys -from test.support import TESTFN, import_fresh_module +from test.support import TESTFN, import_fresh_module, android_not_root c_stat = import_fresh_module('stat', fresh=['_stat']) py_stat = import_fresh_module('stat', blocked=['_stat']) @@ -168,6 +168,7 @@ class TestFilemode: self.assertS_IS("LNK", st_mode) @unittest.skipUnless(hasattr(os, 'mkfifo'), 'os.mkfifo not available') + @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") def test_fifo(self): os.mkfifo(TESTFN, 0o700) st_mode, modestr = self.get_mode() diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 0089ae8dc6..4b3fd364a7 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -21,6 +21,10 @@ import statistics # === Helper functions and class === +def sign(x): + """Return -1.0 for negatives, including -0.0, otherwise +1.0.""" + return math.copysign(1, x) + def _nan_equal(a, b): """Return True if a and b are both the same kind of NAN. @@ -264,6 +268,13 @@ class NumericTestCase(unittest.TestCase): # === Test the helpers === # ======================== +class TestSign(unittest.TestCase): + """Test that the helper function sign() works correctly.""" + def testZeroes(self): + # Test that signed zeroes report their sign correctly. + self.assertEqual(sign(0.0), +1) + self.assertEqual(sign(-0.0), -1) + # --- Tests for approx_equal --- @@ -659,7 +670,7 @@ class DocTests(unittest.TestCase): @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -OO and above") def test_doc_tests(self): - failed, tried = doctest.testmod(statistics) + failed, tried = doctest.testmod(statistics, optionflags=doctest.ELLIPSIS) self.assertGreater(tried, 0) self.assertEqual(failed, 0) @@ -702,9 +713,9 @@ class ExactRatioTest(unittest.TestCase): def test_decimal(self): D = Decimal _exact_ratio = statistics._exact_ratio - self.assertEqual(_exact_ratio(D("0.125")), (125, 1000)) - self.assertEqual(_exact_ratio(D("12.345")), (12345, 1000)) - self.assertEqual(_exact_ratio(D("-1.98")), (-198, 100)) + self.assertEqual(_exact_ratio(D("0.125")), (1, 8)) + self.assertEqual(_exact_ratio(D("12.345")), (2469, 200)) + self.assertEqual(_exact_ratio(D("-1.98")), (-99, 50)) def test_inf(self): INF = float("INF") @@ -743,18 +754,18 @@ class ExactRatioTest(unittest.TestCase): class DecimalToRatioTest(unittest.TestCase): - # Test _decimal_to_ratio private function. + # Test _exact_ratio private function. def test_infinity(self): # Test that INFs are handled correctly. inf = Decimal('INF') - self.assertEqual(statistics._decimal_to_ratio(inf), (inf, None)) - self.assertEqual(statistics._decimal_to_ratio(-inf), (-inf, None)) + self.assertEqual(statistics._exact_ratio(inf), (inf, None)) + self.assertEqual(statistics._exact_ratio(-inf), (-inf, None)) def test_nan(self): # Test that NANs are handled correctly. for nan in (Decimal('NAN'), Decimal('sNAN')): - num, den = statistics._decimal_to_ratio(nan) + num, den = statistics._exact_ratio(nan) # Because NANs always compare non-equal, we cannot use assertEqual. # Nor can we use an identity test, as we don't guarantee anything # about the object identity. @@ -767,30 +778,30 @@ class DecimalToRatioTest(unittest.TestCase): for d in numbers: # First test positive decimals. assert d > 0 - num, den = statistics._decimal_to_ratio(d) + num, den = statistics._exact_ratio(d) self.assertGreaterEqual(num, 0) self.assertGreater(den, 0) # Then test negative decimals. - num, den = statistics._decimal_to_ratio(-d) + num, den = statistics._exact_ratio(-d) self.assertLessEqual(num, 0) self.assertGreater(den, 0) def test_negative_exponent(self): # Test result when the exponent is negative. - t = statistics._decimal_to_ratio(Decimal("0.1234")) - self.assertEqual(t, (1234, 10000)) + t = statistics._exact_ratio(Decimal("0.1234")) + self.assertEqual(t, (617, 5000)) def test_positive_exponent(self): # Test results when the exponent is positive. - t = statistics._decimal_to_ratio(Decimal("1.234e7")) + t = statistics._exact_ratio(Decimal("1.234e7")) self.assertEqual(t, (12340000, 1)) def test_regression_20536(self): # Regression test for issue 20536. # See http://bugs.python.org/issue20536 - t = statistics._decimal_to_ratio(Decimal("1e2")) + t = statistics._exact_ratio(Decimal("1e2")) self.assertEqual(t, (100, 1)) - t = statistics._decimal_to_ratio(Decimal("1.47e5")) + t = statistics._exact_ratio(Decimal("1.47e5")) self.assertEqual(t, (147000, 1)) @@ -971,6 +982,34 @@ class ConvertTest(unittest.TestCase): self.assertTrue(_nan_equal(x, nan)) +class FailNegTest(unittest.TestCase): + """Test _fail_neg private function.""" + + def test_pass_through(self): + # Test that values are passed through unchanged. + values = [1, 2.0, Fraction(3), Decimal(4)] + new = list(statistics._fail_neg(values)) + self.assertEqual(values, new) + + def test_negatives_raise(self): + # Test that negatives raise an exception. + for x in [1, 2.0, Fraction(3), Decimal(4)]: + seq = [-x] + it = statistics._fail_neg(seq) + self.assertRaises(statistics.StatisticsError, next, it) + + def test_error_msg(self): + # Test that a given error message is used. + msg = "badness #%d" % random.randint(10000, 99999) + try: + next(statistics._fail_neg([-1], msg)) + except statistics.StatisticsError as e: + errmsg = e.args[0] + else: + self.fail("expected exception, but it didn't happen") + self.assertEqual(errmsg, msg) + + # === Tests for public functions === class UnivariateCommonMixin: @@ -1082,13 +1121,13 @@ class UnivariateTypeMixin: Not all tests to do with types need go in this class. Only those that rely on the function returning the same type as its input data. """ - def test_types_conserved(self): - # Test that functions keeps the same type as their data points. - # (Excludes mixed data types.) This only tests the type of the return - # result, not the value. + def prepare_types_for_conservation_test(self): + """Return the types which are expected to be conserved.""" class MyFloat(float): def __truediv__(self, other): return type(self)(super().__truediv__(other)) + def __rtruediv__(self, other): + return type(self)(super().__rtruediv__(other)) def __sub__(self, other): return type(self)(super().__sub__(other)) def __rsub__(self, other): @@ -1098,9 +1137,14 @@ class UnivariateTypeMixin: def __add__(self, other): return type(self)(super().__add__(other)) __radd__ = __add__ + return (float, Decimal, Fraction, MyFloat) + def test_types_conserved(self): + # Test that functions keeps the same type as their data points. + # (Excludes mixed data types.) This only tests the type of the return + # result, not the value. data = self.prepare_data() - for kind in (float, Decimal, Fraction, MyFloat): + for kind in self.prepare_types_for_conservation_test(): d = [kind(x) for x in data] result = self.func(d) self.assertIs(type(result), kind) @@ -1275,12 +1319,16 @@ class AverageMixin(UnivariateCommonMixin): for x in (23, 42.5, 1.3e15, Fraction(15, 19), Decimal('0.28')): self.assertEqual(self.func([x]), x) + def prepare_values_for_repeated_single_test(self): + return (3.5, 17, 2.5e15, Fraction(61, 67), Decimal('4.9712')) + def test_repeated_single_value(self): # The average of a single repeated value is the value itself. - for x in (3.5, 17, 2.5e15, Fraction(61, 67), Decimal('4.9712')): + for x in self.prepare_values_for_repeated_single_test(): for count in (2, 5, 10, 20): - data = [x]*count - self.assertEqual(self.func(data), x) + with self.subTest(x=x, count=count): + data = [x]*count + self.assertEqual(self.func(data), x) class TestMean(NumericTestCase, AverageMixin, UnivariateTypeMixin): @@ -1304,7 +1352,7 @@ class TestMean(NumericTestCase, AverageMixin, UnivariateTypeMixin): self.assertEqual(self.func(data), 22.015625) def test_decimals(self): - # Test mean with ints. + # Test mean with Decimals. D = Decimal data = [D("1.634"), D("2.517"), D("3.912"), D("4.072"), D("5.813")] random.shuffle(data) @@ -1379,6 +1427,94 @@ class TestMean(NumericTestCase, AverageMixin, UnivariateTypeMixin): self.assertEqual(statistics.mean([tiny]*n), tiny) +class TestHarmonicMean(NumericTestCase, AverageMixin, UnivariateTypeMixin): + def setUp(self): + self.func = statistics.harmonic_mean + + def prepare_data(self): + # Override mixin method. + values = super().prepare_data() + values.remove(0) + return values + + def prepare_values_for_repeated_single_test(self): + # Override mixin method. + return (3.5, 17, 2.5e15, Fraction(61, 67), Decimal('4.125')) + + def test_zero(self): + # Test that harmonic mean returns zero when given zero. + values = [1, 0, 2] + self.assertEqual(self.func(values), 0) + + def test_negative_error(self): + # Test that harmonic mean raises when given a negative value. + exc = statistics.StatisticsError + for values in ([-1], [1, -2, 3]): + with self.subTest(values=values): + self.assertRaises(exc, self.func, values) + + def test_ints(self): + # Test harmonic mean with ints. + data = [2, 4, 4, 8, 16, 16] + random.shuffle(data) + self.assertEqual(self.func(data), 6*4/5) + + def test_floats_exact(self): + # Test harmonic mean with some carefully chosen floats. + data = [1/8, 1/4, 1/4, 1/2, 1/2] + random.shuffle(data) + self.assertEqual(self.func(data), 1/4) + self.assertEqual(self.func([0.25, 0.5, 1.0, 1.0]), 0.5) + + def test_singleton_lists(self): + # Test that harmonic mean([x]) returns (approximately) x. + for x in range(1, 101): + self.assertEqual(self.func([x]), x) + + def test_decimals_exact(self): + # Test harmonic mean with some carefully chosen Decimals. + D = Decimal + self.assertEqual(self.func([D(15), D(30), D(60), D(60)]), D(30)) + data = [D("0.05"), D("0.10"), D("0.20"), D("0.20")] + random.shuffle(data) + self.assertEqual(self.func(data), D("0.10")) + data = [D("1.68"), D("0.32"), D("5.94"), D("2.75")] + random.shuffle(data) + self.assertEqual(self.func(data), D(66528)/70723) + + def test_fractions(self): + # Test harmonic mean with Fractions. + F = Fraction + data = [F(1, 2), F(2, 3), F(3, 4), F(4, 5), F(5, 6), F(6, 7), F(7, 8)] + random.shuffle(data) + self.assertEqual(self.func(data), F(7*420, 4029)) + + def test_inf(self): + # Test harmonic mean with infinity. + values = [2.0, float('inf'), 1.0] + self.assertEqual(self.func(values), 2.0) + + def test_nan(self): + # Test harmonic mean with NANs. + values = [2.0, float('nan'), 1.0] + self.assertTrue(math.isnan(self.func(values))) + + def test_multiply_data_points(self): + # Test multiplying every data point by a constant. + c = 111 + data = [3.4, 4.5, 4.9, 6.7, 6.8, 7.2, 8.0, 8.1, 9.4] + expected = self.func(data)*c + result = self.func([x*c for x in data]) + self.assertEqual(result, expected) + + def test_doubled_data(self): + # Harmonic mean of [a,b...z] should be same as for [a,a,b,b...z,z]. + data = [random.uniform(1, 5) for _ in range(1000)] + expected = self.func(data) + actual = self.func(data*2) + self.assertApproxEqual(actual, expected) + + class TestMedian(NumericTestCase, AverageMixin): # Common tests for median and all median.* functions. def setUp(self): @@ -1600,6 +1736,22 @@ class TestMedianGrouped(TestMedian): data = [220, 220, 240, 260, 260, 260, 260, 280, 280, 300, 320, 340] self.assertEqual(self.func(data, 20), 265.0) + def test_data_type_error(self): + # Test median_grouped with str, bytes data types for data and interval + data = ["", "", ""] + self.assertRaises(TypeError, self.func, data) + #--- + data = [b"", b"", b""] + self.assertRaises(TypeError, self.func, data) + #--- + data = [1, 2, 3] + interval = "" + self.assertRaises(TypeError, self.func, data, interval) + #--- + data = [1, 2, 3] + interval = b"" + self.assertRaises(TypeError, self.func, data, interval) + class TestMode(NumericTestCase, AverageMixin, UnivariateTypeMixin): # Test cases for the discrete version of mode. diff --git a/Lib/test/test_strftime.py b/Lib/test/test_strftime.py index 772cd0688c..72b1910c38 100644 --- a/Lib/test/test_strftime.py +++ b/Lib/test/test_strftime.py @@ -23,9 +23,9 @@ def escapestr(text, ampm): """ new_text = re.escape(text) new_text = new_text.replace(re.escape(ampm), ampm) - new_text = new_text.replace('\%', '%') - new_text = new_text.replace('\:', ':') - new_text = new_text.replace('\?', '?') + new_text = new_text.replace(r'\%', '%') + new_text = new_text.replace(r'\:', ':') + new_text = new_text.replace(r'\?', '?') return new_text diff --git a/Lib/test/test_strlit.py b/Lib/test/test_string_literals.py index 87cffe843a..aba4fc4667 100644 --- a/Lib/test/test_strlit.py +++ b/Lib/test/test_string_literals.py @@ -31,6 +31,7 @@ import os import sys import shutil import tempfile +import warnings import unittest @@ -104,6 +105,29 @@ class TestLiterals(unittest.TestCase): self.assertRaises(SyntaxError, eval, r""" '\U000000' """) self.assertRaises(SyntaxError, eval, r""" '\U0000000' """) + def test_eval_str_invalid_escape(self): + for b in range(1, 128): + if b in b"""\n\r"'01234567NU\\abfnrtuvx""": + continue + with self.assertWarns(DeprecationWarning): + self.assertEqual(eval(r"'\%c'" % b), '\\' + chr(b)) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always', category=DeprecationWarning) + eval("'''\n\\z'''") + self.assertEqual(len(w), 1) + self.assertEqual(w[0].filename, '<string>') + self.assertEqual(w[0].lineno, 2) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('error', category=DeprecationWarning) + with self.assertRaises(SyntaxError) as cm: + eval("'''\n\\z'''") + exc = cm.exception + self.assertEqual(w, []) + self.assertEqual(exc.filename, '<string>') + self.assertEqual(exc.lineno, 2) + def test_eval_str_raw(self): self.assertEqual(eval(""" r'x' """), 'x') self.assertEqual(eval(r""" r'\x01' """), '\\' + 'x01') @@ -121,15 +145,38 @@ class TestLiterals(unittest.TestCase): self.assertEqual(eval(""" b'\x01' """), byte(1)) self.assertEqual(eval(r""" b'\x81' """), byte(0x81)) self.assertRaises(SyntaxError, eval, """ b'\x81' """) - self.assertEqual(eval(r""" b'\u1881' """), b'\\' + b'u1881') + self.assertEqual(eval(r""" br'\u1881' """), b'\\' + b'u1881') self.assertRaises(SyntaxError, eval, """ b'\u1881' """) - self.assertEqual(eval(r""" b'\U0001d120' """), b'\\' + b'U0001d120') + self.assertEqual(eval(r""" br'\U0001d120' """), b'\\' + b'U0001d120') self.assertRaises(SyntaxError, eval, """ b'\U0001d120' """) def test_eval_bytes_incomplete(self): self.assertRaises(SyntaxError, eval, r""" b'\x' """) self.assertRaises(SyntaxError, eval, r""" b'\x0' """) + def test_eval_bytes_invalid_escape(self): + for b in range(1, 128): + if b in b"""\n\r"'01234567\\abfnrtvx""": + continue + with self.assertWarns(DeprecationWarning): + self.assertEqual(eval(r"b'\%c'" % b), b'\\' + bytes([b])) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always', category=DeprecationWarning) + eval("b'''\n\\z'''") + self.assertEqual(len(w), 1) + self.assertEqual(w[0].filename, '<string>') + self.assertEqual(w[0].lineno, 2) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('error', category=DeprecationWarning) + with self.assertRaises(SyntaxError) as cm: + eval("b'''\n\\z'''") + exc = cm.exception + self.assertEqual(w, []) + self.assertEqual(exc.filename, '<string>') + self.assertEqual(exc.lineno, 2) + def test_eval_bytes_raw(self): self.assertEqual(eval(""" br'x' """), b'x') self.assertEqual(eval(""" rb'x' """), b'x') diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 85126e6e53..2cf0926239 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -5,7 +5,6 @@ import time import locale import re import os -import sys from test import support from datetime import date as datetime_date @@ -130,7 +129,7 @@ class TimeRETests(unittest.TestCase): def test_pattern_escaping(self): # Make sure any characters in the format string that might be taken as # regex syntax is escaped. - pattern_string = self.time_re.pattern("\d+") + pattern_string = self.time_re.pattern(r"\d+") self.assertIn(r"\\d\+", pattern_string, "%s does not have re characters escaped properly" % pattern_string) @@ -153,8 +152,8 @@ class TimeRETests(unittest.TestCase): "'%s' using '%s'; group 'a' = '%s', group 'b' = %s'" % (found.string, found.re.pattern, found.group('a'), found.group('b'))) - for directive in ('a','A','b','B','c','d','H','I','j','m','M','p','S', - 'U','w','W','x','X','y','Y','Z','%'): + for directive in ('a','A','b','B','c','d','G','H','I','j','m','M','p', + 'S','u','U','V','w','W','x','X','y','Y','Z','%'): compiled = self.time_re.compile("%" + directive) found = compiled.match(time.strftime("%" + directive)) self.assertTrue(found, "Matching failed on '%s' using '%s' regex" % @@ -171,9 +170,9 @@ class TimeRETests(unittest.TestCase): def test_matching_with_escapes(self): # Make sure a format that requires escaping of characters works - compiled_re = self.time_re.compile("\w+ %m") - found = compiled_re.match("\w+ 10") - self.assertTrue(found, "Escaping failed of format '\w+ 10'") + compiled_re = self.time_re.compile(r"\w+ %m") + found = compiled_re.match(r"\w+ 10") + self.assertTrue(found, r"Escaping failed of format '\w+ 10'") def test_locale_data_w_regex_metacharacters(self): # Check that if locale data contains regex metacharacters they are @@ -219,6 +218,26 @@ class StrptimeTests(unittest.TestCase): else: self.fail("'%s' did not raise ValueError" % bad_format) + # Ambiguous or incomplete cases using ISO year/week/weekday directives + # 1. ISO week (%V) is specified, but the year is specified with %Y + # instead of %G + with self.assertRaises(ValueError): + _strptime._strptime("1999 50", "%Y %V") + # 2. ISO year (%G) and ISO week (%V) are specified, but weekday is not + with self.assertRaises(ValueError): + _strptime._strptime("1999 51", "%G %V") + # 3. ISO year (%G) and weekday are specified, but ISO week (%V) is not + for w in ('A', 'a', 'w', 'u'): + with self.assertRaises(ValueError): + _strptime._strptime("1999 51","%G %{}".format(w)) + # 4. ISO year is specified alone (e.g. time.strptime('2015', '%G')) + with self.assertRaises(ValueError): + _strptime._strptime("2015", "%G") + # 5. Julian/ordinal day (%j) is specified with %G, but not %Y + with self.assertRaises(ValueError): + _strptime._strptime("1999 256", "%G %j") + + def test_strptime_exception_context(self): # check that this doesn't chain exceptions needlessly (see #17572) with self.assertRaises(ValueError) as e: @@ -290,7 +309,7 @@ class StrptimeTests(unittest.TestCase): def test_weekday(self): # Test weekday directives - for directive in ('A', 'a', 'w'): + for directive in ('A', 'a', 'w', 'u'): self.helper(directive,6) def test_julian(self): @@ -384,7 +403,7 @@ class StrptimeTests(unittest.TestCase): # unbalanced parentheses when the regex is compiled if they are not # escaped. # Test instigated by bug #796149 . - need_escaping = ".^$*+?{}\[]|)(" + need_escaping = r".^$*+?{}\[]|)(" self.assertTrue(_strptime._strptime_time(need_escaping, need_escaping)) def test_feb29_on_leap_year_without_year(self): @@ -453,20 +472,37 @@ class CalculationTests(unittest.TestCase): "Calculation of day of the week failed;" "%s != %s" % (result.tm_wday, self.time_tuple.tm_wday)) + if support.is_android: + # Issue #26929: strftime() on Android incorrectly formats %V or %G for + # the last or the first incomplete week in a year. + _ymd_excluded = ((1905, 1, 1), (1906, 12, 31), (2008, 12, 29), + (1917, 12, 31)) + _formats_excluded = ('%G %V',) + else: + _ymd_excluded = () + _formats_excluded = () + def test_week_of_year_and_day_of_week_calculation(self): # Should be able to infer date if given year, week of year (%U or %W) # and day of the week def test_helper(ymd_tuple, test_reason): - for directive in ('W', 'U'): - format_string = "%%Y %%%s %%w" % directive - dt_date = datetime_date(*ymd_tuple) - strp_input = dt_date.strftime(format_string) - strp_output = _strptime._strptime_time(strp_input, format_string) - self.assertTrue(strp_output[:3] == ymd_tuple, - "%s(%s) test failed w/ '%s': %s != %s (%s != %s)" % - (test_reason, directive, strp_input, - strp_output[:3], ymd_tuple, - strp_output[7], dt_date.timetuple()[7])) + for year_week_format in ('%Y %W', '%Y %U', '%G %V'): + if (year_week_format in self._formats_excluded and + ymd_tuple in self._ymd_excluded): + return + for weekday_format in ('%w', '%u', '%a', '%A'): + format_string = year_week_format + ' ' + weekday_format + with self.subTest(test_reason, + date=ymd_tuple, + format=format_string): + dt_date = datetime_date(*ymd_tuple) + strp_input = dt_date.strftime(format_string) + strp_output = _strptime._strptime_time(strp_input, + format_string) + msg = "%r: %s != %s" % (strp_input, + strp_output[7], + dt_date.timetuple()[7]) + self.assertEqual(strp_output[:3], ymd_tuple, msg) test_helper((1901, 1, 3), "week 0") test_helper((1901, 1, 8), "common case") test_helper((1901, 1, 13), "day on Sunday") @@ -498,33 +534,48 @@ class CalculationTests(unittest.TestCase): self.assertEqual(_strptime._strptime_time(value, format)[:-1], expected) check('2015 0 0', '%Y %U %w', 2014, 12, 28, 0, 0, 0, 6, 362) check('2015 0 0', '%Y %W %w', 2015, 1, 4, 0, 0, 0, 6, 4) + check('2015 1 1', '%G %V %u', 2014, 12, 29, 0, 0, 0, 0, 363) check('2015 0 1', '%Y %U %w', 2014, 12, 29, 0, 0, 0, 0, 363) check('2015 0 1', '%Y %W %w', 2014, 12, 29, 0, 0, 0, 0, 363) + check('2015 1 2', '%G %V %u', 2014, 12, 30, 0, 0, 0, 1, 364) check('2015 0 2', '%Y %U %w', 2014, 12, 30, 0, 0, 0, 1, 364) check('2015 0 2', '%Y %W %w', 2014, 12, 30, 0, 0, 0, 1, 364) + check('2015 1 3', '%G %V %u', 2014, 12, 31, 0, 0, 0, 2, 365) check('2015 0 3', '%Y %U %w', 2014, 12, 31, 0, 0, 0, 2, 365) check('2015 0 3', '%Y %W %w', 2014, 12, 31, 0, 0, 0, 2, 365) + check('2015 1 4', '%G %V %u', 2015, 1, 1, 0, 0, 0, 3, 1) check('2015 0 4', '%Y %U %w', 2015, 1, 1, 0, 0, 0, 3, 1) check('2015 0 4', '%Y %W %w', 2015, 1, 1, 0, 0, 0, 3, 1) + check('2015 1 5', '%G %V %u', 2015, 1, 2, 0, 0, 0, 4, 2) check('2015 0 5', '%Y %U %w', 2015, 1, 2, 0, 0, 0, 4, 2) check('2015 0 5', '%Y %W %w', 2015, 1, 2, 0, 0, 0, 4, 2) + check('2015 1 6', '%G %V %u', 2015, 1, 3, 0, 0, 0, 5, 3) check('2015 0 6', '%Y %U %w', 2015, 1, 3, 0, 0, 0, 5, 3) check('2015 0 6', '%Y %W %w', 2015, 1, 3, 0, 0, 0, 5, 3) + check('2015 1 7', '%G %V %u', 2015, 1, 4, 0, 0, 0, 6, 4) check('2009 0 0', '%Y %U %w', 2008, 12, 28, 0, 0, 0, 6, 363) check('2009 0 0', '%Y %W %w', 2009, 1, 4, 0, 0, 0, 6, 4) + check('2009 1 1', '%G %V %u', 2008, 12, 29, 0, 0, 0, 0, 364) check('2009 0 1', '%Y %U %w', 2008, 12, 29, 0, 0, 0, 0, 364) check('2009 0 1', '%Y %W %w', 2008, 12, 29, 0, 0, 0, 0, 364) + check('2009 1 2', '%G %V %u', 2008, 12, 30, 0, 0, 0, 1, 365) check('2009 0 2', '%Y %U %w', 2008, 12, 30, 0, 0, 0, 1, 365) check('2009 0 2', '%Y %W %w', 2008, 12, 30, 0, 0, 0, 1, 365) + check('2009 1 3', '%G %V %u', 2008, 12, 31, 0, 0, 0, 2, 366) check('2009 0 3', '%Y %U %w', 2008, 12, 31, 0, 0, 0, 2, 366) check('2009 0 3', '%Y %W %w', 2008, 12, 31, 0, 0, 0, 2, 366) + check('2009 1 4', '%G %V %u', 2009, 1, 1, 0, 0, 0, 3, 1) check('2009 0 4', '%Y %U %w', 2009, 1, 1, 0, 0, 0, 3, 1) check('2009 0 4', '%Y %W %w', 2009, 1, 1, 0, 0, 0, 3, 1) + check('2009 1 5', '%G %V %u', 2009, 1, 2, 0, 0, 0, 4, 2) check('2009 0 5', '%Y %U %w', 2009, 1, 2, 0, 0, 0, 4, 2) check('2009 0 5', '%Y %W %w', 2009, 1, 2, 0, 0, 0, 4, 2) + check('2009 1 6', '%G %V %u', 2009, 1, 3, 0, 0, 0, 5, 3) check('2009 0 6', '%Y %U %w', 2009, 1, 3, 0, 0, 0, 5, 3) check('2009 0 6', '%Y %W %w', 2009, 1, 3, 0, 0, 0, 5, 3) + check('2009 1 7', '%G %V %u', 2009, 1, 4, 0, 0, 0, 6, 4) + class CacheTests(unittest.TestCase): """Test that caching works properly.""" diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 9b8178f84a..be0047589c 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -1,5 +1,6 @@ from collections import abc import array +import math import operator import unittest import struct @@ -15,22 +16,10 @@ byteorders = '', '@', '=', '<', '>', '!' def iter_integer_formats(byteorders=byteorders): for code in integer_codes: for byteorder in byteorders: - if (byteorder in ('', '@') and code in ('q', 'Q') and - not HAVE_LONG_LONG): - continue if (byteorder not in ('', '@') and code in ('n', 'N')): continue yield code, byteorder -# Native 'q' packing isn't available on systems that don't have the C -# long long type. -try: - struct.pack('q', 5) -except struct.error: - HAVE_LONG_LONG = False -else: - HAVE_LONG_LONG = True - def string_reverse(s): return s[::-1] @@ -158,9 +147,7 @@ class StructTest(unittest.TestCase): self.assertEqual(size, expected_size[code]) # native integer sizes - native_pairs = 'bB', 'hH', 'iI', 'lL', 'nN' - if HAVE_LONG_LONG: - native_pairs += 'qQ', + native_pairs = 'bB', 'hH', 'iI', 'lL', 'nN', 'qQ' for format_pair in native_pairs: for byteorder in '', '@': signed_size = struct.calcsize(byteorder + format_pair[0]) @@ -173,9 +160,8 @@ class StructTest(unittest.TestCase): self.assertLessEqual(4, struct.calcsize('l')) self.assertLessEqual(struct.calcsize('h'), struct.calcsize('i')) self.assertLessEqual(struct.calcsize('i'), struct.calcsize('l')) - if HAVE_LONG_LONG: - self.assertLessEqual(8, struct.calcsize('q')) - self.assertLessEqual(struct.calcsize('l'), struct.calcsize('q')) + self.assertLessEqual(8, struct.calcsize('q')) + self.assertLessEqual(struct.calcsize('l'), struct.calcsize('q')) self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('i')) self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('P')) @@ -366,8 +352,6 @@ class StructTest(unittest.TestCase): # SF bug 705836. "<f" and ">f" had a severe rounding bug, where a carry # from the low-order discarded bits could propagate into the exponent # field, causing the result to be wrong by a factor of 2. - import math - for base in range(1, 33): # smaller <- largest representable float less than base. delta = 0.5 @@ -663,6 +647,110 @@ class UnpackIteratorTest(unittest.TestCase): self.assertRaises(StopIteration, next, it) self.assertRaises(StopIteration, next, it) + def test_half_float(self): + # Little-endian examples from: + # http://en.wikipedia.org/wiki/Half_precision_floating-point_format + format_bits_float__cleanRoundtrip_list = [ + (b'\x00\x3c', 1.0), + (b'\x00\xc0', -2.0), + (b'\xff\x7b', 65504.0), # (max half precision) + (b'\x00\x04', 2**-14), # ~= 6.10352 * 10**-5 (min pos normal) + (b'\x01\x00', 2**-24), # ~= 5.96046 * 10**-8 (min pos subnormal) + (b'\x00\x00', 0.0), + (b'\x00\x80', -0.0), + (b'\x00\x7c', float('+inf')), + (b'\x00\xfc', float('-inf')), + (b'\x55\x35', 0.333251953125), # ~= 1/3 + ] + + for le_bits, f in format_bits_float__cleanRoundtrip_list: + be_bits = le_bits[::-1] + self.assertEqual(f, struct.unpack('<e', le_bits)[0]) + self.assertEqual(le_bits, struct.pack('<e', f)) + self.assertEqual(f, struct.unpack('>e', be_bits)[0]) + self.assertEqual(be_bits, struct.pack('>e', f)) + if sys.byteorder == 'little': + self.assertEqual(f, struct.unpack('e', le_bits)[0]) + self.assertEqual(le_bits, struct.pack('e', f)) + else: + self.assertEqual(f, struct.unpack('e', be_bits)[0]) + self.assertEqual(be_bits, struct.pack('e', f)) + + # Check for NaN handling: + format_bits__nan_list = [ + ('<e', b'\x01\xfc'), + ('<e', b'\x00\xfe'), + ('<e', b'\xff\xff'), + ('<e', b'\x01\x7c'), + ('<e', b'\x00\x7e'), + ('<e', b'\xff\x7f'), + ] + + for formatcode, bits in format_bits__nan_list: + self.assertTrue(math.isnan(struct.unpack('<e', bits)[0])) + self.assertTrue(math.isnan(struct.unpack('>e', bits[::-1])[0])) + + # Check that packing produces a bit pattern representing a quiet NaN: + # all exponent bits and the msb of the fraction should all be 1. + packed = struct.pack('<e', math.nan) + self.assertEqual(packed[1] & 0x7e, 0x7e) + packed = struct.pack('<e', -math.nan) + self.assertEqual(packed[1] & 0x7e, 0x7e) + + # Checks for round-to-even behavior + format_bits_float__rounding_list = [ + ('>e', b'\x00\x01', 2.0**-25 + 2.0**-35), # Rounds to minimum subnormal + ('>e', b'\x00\x00', 2.0**-25), # Underflows to zero (nearest even mode) + ('>e', b'\x00\x00', 2.0**-26), # Underflows to zero + ('>e', b'\x03\xff', 2.0**-14 - 2.0**-24), # Largest subnormal. + ('>e', b'\x03\xff', 2.0**-14 - 2.0**-25 - 2.0**-65), + ('>e', b'\x04\x00', 2.0**-14 - 2.0**-25), + ('>e', b'\x04\x00', 2.0**-14), # Smallest normal. + ('>e', b'\x3c\x01', 1.0+2.0**-11 + 2.0**-16), # rounds to 1.0+2**(-10) + ('>e', b'\x3c\x00', 1.0+2.0**-11), # rounds to 1.0 (nearest even mode) + ('>e', b'\x3c\x00', 1.0+2.0**-12), # rounds to 1.0 + ('>e', b'\x7b\xff', 65504), # largest normal + ('>e', b'\x7b\xff', 65519), # rounds to 65504 + ('>e', b'\x80\x01', -2.0**-25 - 2.0**-35), # Rounds to minimum subnormal + ('>e', b'\x80\x00', -2.0**-25), # Underflows to zero (nearest even mode) + ('>e', b'\x80\x00', -2.0**-26), # Underflows to zero + ('>e', b'\xbc\x01', -1.0-2.0**-11 - 2.0**-16), # rounds to 1.0+2**(-10) + ('>e', b'\xbc\x00', -1.0-2.0**-11), # rounds to 1.0 (nearest even mode) + ('>e', b'\xbc\x00', -1.0-2.0**-12), # rounds to 1.0 + ('>e', b'\xfb\xff', -65519), # rounds to 65504 + ] + + for formatcode, bits, f in format_bits_float__rounding_list: + self.assertEqual(bits, struct.pack(formatcode, f)) + + # This overflows, and so raises an error + format_bits_float__roundingError_list = [ + # Values that round to infinity. + ('>e', 65520.0), + ('>e', 65536.0), + ('>e', 1e300), + ('>e', -65520.0), + ('>e', -65536.0), + ('>e', -1e300), + ('<e', 65520.0), + ('<e', 65536.0), + ('<e', 1e300), + ('<e', -65520.0), + ('<e', -65536.0), + ('<e', -1e300), + ] + + for formatcode, f in format_bits_float__roundingError_list: + self.assertRaises(OverflowError, struct.pack, formatcode, f) + + # Double rounding + format_bits_float__doubleRoundingError_list = [ + ('>e', b'\x67\xff', 0x1ffdffffff * 2**-26), # should be 2047, if double-rounded 64>32>16, becomes 2048 + ] + + for formatcode, bits, f in format_bits_float__doubleRoundingError_list: + self.assertEqual(bits, struct.pack(formatcode, f)) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_subclassinit.py b/Lib/test/test_subclassinit.py new file mode 100644 index 0000000000..3c331bb98f --- /dev/null +++ b/Lib/test/test_subclassinit.py @@ -0,0 +1,284 @@ +import sys +import types +import unittest + + +class Test(unittest.TestCase): + def test_init_subclass(self): + class A: + initialized = False + + def __init_subclass__(cls): + super().__init_subclass__() + cls.initialized = True + + class B(A): + pass + + self.assertFalse(A.initialized) + self.assertTrue(B.initialized) + + def test_init_subclass_dict(self): + class A(dict): + initialized = False + + def __init_subclass__(cls): + super().__init_subclass__() + cls.initialized = True + + class B(A): + pass + + self.assertFalse(A.initialized) + self.assertTrue(B.initialized) + + def test_init_subclass_kwargs(self): + class A: + def __init_subclass__(cls, **kwargs): + cls.kwargs = kwargs + + class B(A, x=3): + pass + + self.assertEqual(B.kwargs, dict(x=3)) + + def test_init_subclass_error(self): + class A: + def __init_subclass__(cls): + raise RuntimeError + + with self.assertRaises(RuntimeError): + class B(A): + pass + + def test_init_subclass_wrong(self): + class A: + def __init_subclass__(cls, whatever): + pass + + with self.assertRaises(TypeError): + class B(A): + pass + + def test_init_subclass_skipped(self): + class BaseWithInit: + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls.initialized = cls + + class BaseWithoutInit(BaseWithInit): + pass + + class A(BaseWithoutInit): + pass + + self.assertIs(A.initialized, A) + self.assertIs(BaseWithoutInit.initialized, BaseWithoutInit) + + def test_init_subclass_diamond(self): + class Base: + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls.calls = [] + + class Left(Base): + pass + + class Middle: + def __init_subclass__(cls, middle, **kwargs): + super().__init_subclass__(**kwargs) + cls.calls += [middle] + + class Right(Base): + def __init_subclass__(cls, right="right", **kwargs): + super().__init_subclass__(**kwargs) + cls.calls += [right] + + class A(Left, Middle, Right, middle="middle"): + pass + + self.assertEqual(A.calls, ["right", "middle"]) + self.assertEqual(Left.calls, []) + self.assertEqual(Right.calls, []) + + def test_set_name(self): + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class A: + d = Descriptor() + + self.assertEqual(A.d.name, "d") + self.assertIs(A.d.owner, A) + + def test_set_name_metaclass(self): + class Meta(type): + def __new__(cls, name, bases, ns): + ret = super().__new__(cls, name, bases, ns) + self.assertEqual(ret.d.name, "d") + self.assertIs(ret.d.owner, ret) + return 0 + + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class A(metaclass=Meta): + d = Descriptor() + self.assertEqual(A, 0) + + def test_set_name_error(self): + class Descriptor: + def __set_name__(self, owner, name): + 1/0 + + with self.assertRaises(RuntimeError) as cm: + class NotGoingToWork: + attr = Descriptor() + + exc = cm.exception + self.assertRegex(str(exc), r'\bNotGoingToWork\b') + self.assertRegex(str(exc), r'\battr\b') + self.assertRegex(str(exc), r'\bDescriptor\b') + self.assertIsInstance(exc.__cause__, ZeroDivisionError) + + def test_set_name_wrong(self): + class Descriptor: + def __set_name__(self): + pass + + with self.assertRaises(RuntimeError) as cm: + class NotGoingToWork: + attr = Descriptor() + + exc = cm.exception + self.assertRegex(str(exc), r'\bNotGoingToWork\b') + self.assertRegex(str(exc), r'\battr\b') + self.assertRegex(str(exc), r'\bDescriptor\b') + self.assertIsInstance(exc.__cause__, TypeError) + + def test_set_name_lookup(self): + resolved = [] + class NonDescriptor: + def __getattr__(self, name): + resolved.append(name) + + class A: + d = NonDescriptor() + + self.assertNotIn('__set_name__', resolved, + '__set_name__ is looked up in instance dict') + + def test_set_name_init_subclass(self): + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class Meta(type): + def __new__(cls, name, bases, ns): + self = super().__new__(cls, name, bases, ns) + self.meta_owner = self.owner + self.meta_name = self.name + return self + + class A: + def __init_subclass__(cls): + cls.owner = cls.d.owner + cls.name = cls.d.name + + class B(A, metaclass=Meta): + d = Descriptor() + + self.assertIs(B.owner, B) + self.assertEqual(B.name, 'd') + self.assertIs(B.meta_owner, B) + self.assertEqual(B.name, 'd') + + def test_set_name_modifying_dict(self): + notified = [] + class Descriptor: + def __set_name__(self, owner, name): + setattr(owner, name + 'x', None) + notified.append(name) + + class A: + a = Descriptor() + b = Descriptor() + c = Descriptor() + d = Descriptor() + e = Descriptor() + + self.assertCountEqual(notified, ['a', 'b', 'c', 'd', 'e']) + + def test_errors(self): + class MyMeta(type): + pass + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + with self.assertRaises(TypeError): + types.new_class("MyClass", (object,), + dict(metaclass=MyMeta, otherarg=1)) + types.prepare_class("MyClass", (object,), + dict(metaclass=MyMeta, otherarg=1)) + + class MyMeta(type): + def __init__(self, name, bases, namespace, otherarg): + super().__init__(name, bases, namespace) + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + class MyMeta(type): + def __new__(cls, name, bases, namespace, otherarg): + return super().__new__(cls, name, bases, namespace) + + def __init__(self, name, bases, namespace, otherarg): + super().__init__(name, bases, namespace) + self.otherarg = otherarg + + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + self.assertEqual(MyClass.otherarg, 1) + + def test_errors_changed_pep487(self): + # These tests failed before Python 3.6, PEP 487 + class MyMeta(type): + def __new__(cls, name, bases, namespace): + return super().__new__(cls, name=name, bases=bases, + dict=namespace) + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta): + pass + + class MyMeta(type): + def __new__(cls, name, bases, namespace, otherarg): + self = super().__new__(cls, name, bases, namespace) + self.otherarg = otherarg + return self + + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + self.assertEqual(MyClass.otherarg, 1) + + def test_type(self): + t = type('NewClass', (object,), {}) + self.assertIsInstance(t, type) + self.assertEqual(t.__name__, 'NewClass') + + with self.assertRaises(TypeError): + type(name='NewClass', bases=(object,), dict={}) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 2dc03ee700..e63f9f254c 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1,21 +1,17 @@ import unittest from unittest import mock -from test.support import script_helper from test import support import subprocess import sys import platform import signal import io -import locale import os import errno import tempfile import time -import re import selectors import sysconfig -import warnings import select import shutil import gc @@ -31,6 +27,9 @@ try: except ImportError: threading = None +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + mswindows = (sys.platform == "win32") # @@ -300,7 +299,8 @@ class ProcessTestCase(BaseTestCase): # Verify first that the call succeeds without the executable arg. pre_args = [sys.executable, "-c"] self._assert_python(pre_args) - self.assertRaises(FileNotFoundError, self._assert_python, pre_args, + self.assertRaises((FileNotFoundError, PermissionError), + self._assert_python, pre_args, executable="doesnotexist") @unittest.skipIf(mswindows, "executable argument replaces shell") @@ -454,8 +454,8 @@ class ProcessTestCase(BaseTestCase): p = subprocess.Popen([sys.executable, "-c", 'import sys; sys.stdout.write("orange")'], stdout=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read(), b"orange") + with p: + self.assertEqual(p.stdout.read(), b"orange") def test_stdout_filedes(self): # stdout is set to open file descriptor @@ -485,8 +485,8 @@ class ProcessTestCase(BaseTestCase): p = subprocess.Popen([sys.executable, "-c", 'import sys; sys.stderr.write("strawberry")'], stderr=subprocess.PIPE) - self.addCleanup(p.stderr.close) - self.assertStderrEqual(p.stderr.read(), b"strawberry") + with p: + self.assertStderrEqual(p.stderr.read(), b"strawberry") def test_stderr_filedes(self): # stderr is set to open file descriptor @@ -541,8 +541,8 @@ class ProcessTestCase(BaseTestCase): 'sys.stderr.write("orange")'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - self.addCleanup(p.stdout.close) - self.assertStderrEqual(p.stdout.read(), b"appleorange") + with p: + self.assertStderrEqual(p.stdout.read(), b"appleorange") def test_stdout_stderr_file(self): # capture stdout and stderr to the same open file @@ -800,18 +800,19 @@ class ProcessTestCase(BaseTestCase): stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=1) - p.stdin.write("line1\n") - p.stdin.flush() - self.assertEqual(p.stdout.readline(), "line1\n") - p.stdin.write("line3\n") - p.stdin.close() - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.readline(), - "line2\n") - self.assertEqual(p.stdout.read(6), - "line3\n") - self.assertEqual(p.stdout.read(), - "line4\nline5\nline6\nline7\nline8") + with p: + p.stdin.write("line1\n") + p.stdin.flush() + self.assertEqual(p.stdout.readline(), "line1\n") + p.stdin.write("line3\n") + p.stdin.close() + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.readline(), + "line2\n") + self.assertEqual(p.stdout.read(6), + "line3\n") + self.assertEqual(p.stdout.read(), + "line4\nline5\nline6\nline7\nline8") def test_universal_newlines_communicate(self): # universal newlines through communicate() @@ -900,31 +901,42 @@ class ProcessTestCase(BaseTestCase): # # UTF-16 and UTF-32-BE are sufficient to check both with BOM and # without, and UTF-16 and UTF-32. - import _bootlocale for encoding in ['utf-16', 'utf-32-be']: - old_getpreferredencoding = _bootlocale.getpreferredencoding - # Indirectly via io.TextIOWrapper, Popen() defaults to - # locale.getpreferredencoding(False) and earlier in Python 3.2 to - # locale.getpreferredencoding(). - def getpreferredencoding(do_setlocale=True): - return encoding code = ("import sys; " r"sys.stdout.buffer.write('1\r\n2\r3\n4'.encode('%s'))" % encoding) args = [sys.executable, '-c', code] - try: - _bootlocale.getpreferredencoding = getpreferredencoding - # We set stdin to be non-None because, as of this writing, - # a different code path is used when the number of pipes is - # zero or one. - popen = subprocess.Popen(args, universal_newlines=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - stdout, stderr = popen.communicate(input='') - finally: - _bootlocale.getpreferredencoding = old_getpreferredencoding + # We set stdin to be non-None because, as of this writing, + # a different code path is used when the number of pipes is + # zero or one. + popen = subprocess.Popen(args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding=encoding) + stdout, stderr = popen.communicate(input='') self.assertEqual(stdout, '1\n2\n3\n4') + def test_communicate_errors(self): + for errors, expected in [ + ('ignore', ''), + ('replace', '\ufffd\ufffd'), + ('surrogateescape', '\udc80\udc80'), + ('backslashreplace', '\\x80\\x80'), + ]: + code = ("import sys; " + r"sys.stdout.buffer.write(b'[\x80\x80]')") + args = [sys.executable, '-c', code] + # We set stdin to be non-None because, as of this writing, + # a different code path is used when the number of pipes is + # zero or one. + popen = subprocess.Popen(args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding='utf-8', + errors=errors) + stdout, stderr = popen.communicate(input='') + self.assertEqual(stdout, '[{}]'.format(expected)) + def test_no_leaking(self): # Make sure we leak no resources if not mswindows: @@ -1009,6 +1021,19 @@ class ProcessTestCase(BaseTestCase): # time to start. self.assertEqual(p.wait(timeout=3), 0) + def test_wait_endtime(self): + """Confirm that the deprecated endtime parameter warns.""" + p = subprocess.Popen([sys.executable, "-c", "pass"]) + try: + with self.assertWarns(DeprecationWarning) as warn_cm: + p.wait(endtime=time.time()+0.01) + except subprocess.TimeoutExpired: + pass # We're not testing endtime timeout behavior. + finally: + p.kill() + self.assertIn('test_subprocess.py', warn_cm.filename) + self.assertIn('endtime', str(warn_cm.warning)) + def test_invalid_bufsize(self): # an invalid type of the bufsize argument should raise # TypeError. @@ -1437,6 +1462,27 @@ class POSIXProcessTestCase(BaseTestCase): p.wait() self.assertEqual(-p.returncode, signal.SIGABRT) + def test_CalledProcessError_str_signal(self): + err = subprocess.CalledProcessError(-int(signal.SIGABRT), "fake cmd") + error_string = str(err) + # We're relying on the repr() of the signal.Signals intenum to provide + # the word signal, the signal name and the numeric value. + self.assertIn("signal", error_string.lower()) + # We're not being specific about the signal name as some signals have + # multiple names and which name is revealed can vary. + self.assertIn("SIG", error_string) + self.assertIn(str(signal.SIGABRT), error_string) + + def test_CalledProcessError_str_unknown_signal(self): + err = subprocess.CalledProcessError(-9876543, "fake cmd") + error_string = str(err) + self.assertIn("unknown signal 9876543.", error_string) + + def test_CalledProcessError_str_non_zero(self): + err = subprocess.CalledProcessError(2, "fake cmd") + error_string = str(err) + self.assertIn("non-zero exit status 2.", error_string) + def test_preexec(self): # DISCLAIMER: Setting environment variables is *not* a good use # of a preexec_fn. This is merely a test. @@ -1445,8 +1491,8 @@ class POSIXProcessTestCase(BaseTestCase): 'sys.stdout.write(os.getenv("FRUIT"))'], stdout=subprocess.PIPE, preexec_fn=lambda: os.putenv("FRUIT", "apple")) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read(), b"apple") + with p: + self.assertEqual(p.stdout.read(), b"apple") def test_preexec_exception(self): def raise_it(): @@ -1567,7 +1613,7 @@ class POSIXProcessTestCase(BaseTestCase): fd, fname = tempfile.mkstemp() # reopen in text mode with open(fd, "w", errors="surrogateescape") as fobj: - fobj.write("#!/bin/sh\n") + fobj.write("#!%s\n" % support.unix_shell) fobj.write("exec '%s' -c 'import sys; sys.exit(47)'\n" % sys.executable) os.chmod(fname, 0o700) @@ -1594,8 +1640,8 @@ class POSIXProcessTestCase(BaseTestCase): p = subprocess.Popen(["echo $FRUIT"], shell=1, stdout=subprocess.PIPE, env=newenv) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") + with p: + self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") def test_shell_string(self): # Run command through the shell (string) @@ -1604,15 +1650,15 @@ class POSIXProcessTestCase(BaseTestCase): p = subprocess.Popen("echo $FRUIT", shell=1, stdout=subprocess.PIPE, env=newenv) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") + with p: + self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") def test_call_string(self): # call() function with string argument on UNIX fd, fname = tempfile.mkstemp() # reopen in text mode with open(fd, "w", errors="surrogateescape") as fobj: - fobj.write("#!/bin/sh\n") + fobj.write("#!%s\n" % support.unix_shell) fobj.write("exec '%s' -c 'import sys; sys.exit(47)'\n" % sys.executable) os.chmod(fname, 0o700) @@ -1637,8 +1683,8 @@ class POSIXProcessTestCase(BaseTestCase): for sh in shells: p = subprocess.Popen("echo $0", executable=sh, shell=True, stdout=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.assertEqual(p.stdout.read().strip(), bytes(sh, 'ascii')) + with p: + self.assertEqual(p.stdout.read().strip(), bytes(sh, 'ascii')) def _kill_process(self, method, *args): # Do not inherit file handles from the parent. @@ -2296,7 +2342,9 @@ class POSIXProcessTestCase(BaseTestCase): self.addCleanup(p.stderr.close) ident = id(p) pid = p.pid - del p + with support.check_warnings(('', ResourceWarning)): + p = None + # check that p is in the active processes list self.assertIn(ident, [id(o) for o in subprocess._active]) @@ -2315,7 +2363,9 @@ class POSIXProcessTestCase(BaseTestCase): self.addCleanup(p.stderr.close) ident = id(p) pid = p.pid - del p + with support.check_warnings(('', ResourceWarning)): + p = None + os.kill(pid, signal.SIGKILL) # check that p is in the active processes list self.assertIn(ident, [id(o) for o in subprocess._active]) @@ -2547,8 +2597,8 @@ class Win32ProcessTestCase(BaseTestCase): p = subprocess.Popen(["set"], shell=1, stdout=subprocess.PIPE, env=newenv) - self.addCleanup(p.stdout.close) - self.assertIn(b"physalis", p.stdout.read()) + with p: + self.assertIn(b"physalis", p.stdout.read()) def test_shell_string(self): # Run command through the shell (string) @@ -2557,8 +2607,20 @@ class Win32ProcessTestCase(BaseTestCase): p = subprocess.Popen("set", shell=1, stdout=subprocess.PIPE, env=newenv) - self.addCleanup(p.stdout.close) - self.assertIn(b"physalis", p.stdout.read()) + with p: + self.assertIn(b"physalis", p.stdout.read()) + + def test_shell_encodings(self): + # Run command through the shell (string) + for enc in ['ansi', 'oem']: + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen("set", shell=1, + stdout=subprocess.PIPE, + env=newenv, + encoding=enc) + with p: + self.assertIn("physalis", p.stdout.read(), enc) def test_call_string(self): # call() function with string argument on Windows @@ -2577,16 +2639,14 @@ class Win32ProcessTestCase(BaseTestCase): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stderr.close) - self.addCleanup(p.stdin.close) - # Wait for the interpreter to be completely initialized before - # sending any signal. - p.stdout.read(1) - getattr(p, method)(*args) - _, stderr = p.communicate() - self.assertStderrEqual(stderr, b'') - returncode = p.wait() + with p: + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + returncode = p.wait() self.assertNotEqual(returncode, 0) def _kill_dead_process(self, method, *args): @@ -2599,19 +2659,17 @@ class Win32ProcessTestCase(BaseTestCase): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stderr.close) - self.addCleanup(p.stdin.close) - # Wait for the interpreter to be completely initialized before - # sending any signal. - p.stdout.read(1) - # The process should end after this - time.sleep(1) - # This shouldn't raise even though the child is now dead - getattr(p, method)(*args) - _, stderr = p.communicate() - self.assertStderrEqual(stderr, b'') - rc = p.wait() + with p: + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + rc = p.wait() self.assertEqual(rc, 42) def test_send_signal(self): @@ -2654,8 +2712,7 @@ class MiscTests(unittest.TestCase): def test__all__(self): """Ensure that __all__ is populated properly.""" - # STARTUPINFO added to __all__ in 3.6 - intentionally_excluded = {"list2cmdline", "STARTUPINFO", "Handle"} + intentionally_excluded = {"list2cmdline", "Handle"} exported = set(subprocess.__all__) possible_exports = set() import types @@ -2700,11 +2757,11 @@ class CommandsWithSpaces (BaseTestCase): def with_spaces(self, *args, **kwargs): kwargs['stdout'] = subprocess.PIPE p = subprocess.Popen(*args, **kwargs) - self.addCleanup(p.stdout.close) - self.assertEqual( - p.stdout.read ().decode("mbcs"), - "2 [%r, 'ab cd']" % self.fname - ) + with p: + self.assertEqual( + p.stdout.read ().decode("mbcs"), + "2 [%r, 'ab cd']" % self.fname + ) def test_shell_string_with_spaces(self): # call() function with string argument with spaces on Windows @@ -2756,7 +2813,7 @@ class ContextManagerTests(BaseTestCase): self.assertEqual(proc.returncode, 1) def test_invalid_args(self): - with self.assertRaises(FileNotFoundError) as c: + with self.assertRaises((FileNotFoundError, PermissionError)) as c: with subprocess.Popen(['nonexisting_i_hope'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc: @@ -2779,19 +2836,5 @@ class ContextManagerTests(BaseTestCase): self.assertTrue(proc.stdin.closed) -def test_main(): - unit_tests = (ProcessTestCase, - POSIXProcessTestCase, - Win32ProcessTestCase, - MiscTests, - ProcessTestCaseNoPoll, - CommandsWithSpaces, - ContextManagerTests, - RunFuncTestCase, - ) - - support.run_unittest(*unit_tests) - support.reap_children() - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sunau.py b/Lib/test/test_sunau.py index 0f4134e982..bc1f46c0eb 100644 --- a/Lib/test/test_sunau.py +++ b/Lib/test/test_sunau.py @@ -1,4 +1,3 @@ -from test.support import TESTFN import unittest from test import audiotests from audioop import byteswap diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index b84863fe53..447dec9130 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -1,7 +1,9 @@ -"""Unit tests for new super() implementation.""" +"""Unit tests for zero-argument super() & related machinery.""" import sys import unittest +import warnings +from test.support import check_warnings class A: @@ -143,6 +145,166 @@ class TestSuper(unittest.TestCase): return __class__ self.assertIs(X.f(), X) + def test___class___new(self): + # See issue #23722 + # Ensure zero-arg super() works as soon as type.__new__() is completed + test_class = None + + class Meta(type): + def __new__(cls, name, bases, namespace): + nonlocal test_class + self = super().__new__(cls, name, bases, namespace) + test_class = self.f() + return self + + class A(metaclass=Meta): + @staticmethod + def f(): + return __class__ + + self.assertIs(test_class, A) + + def test___class___delayed(self): + # See issue #23722 + test_namespace = None + + class Meta(type): + def __new__(cls, name, bases, namespace): + nonlocal test_namespace + test_namespace = namespace + return None + + # This case shouldn't trigger the __classcell__ deprecation warning + with check_warnings() as w: + warnings.simplefilter("always", DeprecationWarning) + class A(metaclass=Meta): + @staticmethod + def f(): + return __class__ + self.assertEqual(w.warnings, []) + + self.assertIs(A, None) + + B = type("B", (), test_namespace) + self.assertIs(B.f(), B) + + def test___class___mro(self): + # See issue #23722 + test_class = None + + class Meta(type): + def mro(self): + # self.f() doesn't work yet... + self.__dict__["f"]() + return super().mro() + + class A(metaclass=Meta): + def f(): + nonlocal test_class + test_class = __class__ + + self.assertIs(test_class, A) + + def test___classcell___expected_behaviour(self): + # See issue #23722 + class Meta(type): + def __new__(cls, name, bases, namespace): + nonlocal namespace_snapshot + namespace_snapshot = namespace.copy() + return super().__new__(cls, name, bases, namespace) + + # __classcell__ is injected into the class namespace by the compiler + # when at least one method needs it, and should be omitted otherwise + namespace_snapshot = None + class WithoutClassRef(metaclass=Meta): + pass + self.assertNotIn("__classcell__", namespace_snapshot) + + # With zero-arg super() or an explicit __class__ reference, + # __classcell__ is the exact cell reference to be populated by + # type.__new__ + namespace_snapshot = None + class WithClassRef(metaclass=Meta): + def f(self): + return __class__ + + class_cell = namespace_snapshot["__classcell__"] + method_closure = WithClassRef.f.__closure__ + self.assertEqual(len(method_closure), 1) + self.assertIs(class_cell, method_closure[0]) + # Ensure the cell reference *doesn't* get turned into an attribute + with self.assertRaises(AttributeError): + WithClassRef.__classcell__ + + def test___classcell___missing(self): + # See issue #23722 + # Some metaclasses may not pass the original namespace to type.__new__ + # We test that case here by forcibly deleting __classcell__ + class Meta(type): + def __new__(cls, name, bases, namespace): + namespace.pop('__classcell__', None) + return super().__new__(cls, name, bases, namespace) + + # The default case should continue to work without any warnings + with check_warnings() as w: + warnings.simplefilter("always", DeprecationWarning) + class WithoutClassRef(metaclass=Meta): + pass + self.assertEqual(w.warnings, []) + + # With zero-arg super() or an explicit __class__ reference, we expect + # __build_class__ to emit a DeprecationWarning complaining that + # __class__ was not set, and asking if __classcell__ was propagated + # to type.__new__. + # In Python 3.7, that warning will become a RuntimeError. + expected_warning = ( + '__class__ not set.*__classcell__ propagated', + DeprecationWarning + ) + with check_warnings(expected_warning): + warnings.simplefilter("always", DeprecationWarning) + class WithClassRef(metaclass=Meta): + def f(self): + return __class__ + # Check __class__ still gets set despite the warning + self.assertIs(WithClassRef().f(), WithClassRef) + + # Check the warning is turned into an error as expected + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + with self.assertRaises(DeprecationWarning): + class WithClassRef(metaclass=Meta): + def f(self): + return __class__ + + def test___classcell___overwrite(self): + # See issue #23722 + # Overwriting __classcell__ with nonsense is explicitly prohibited + class Meta(type): + def __new__(cls, name, bases, namespace, cell): + namespace['__classcell__'] = cell + return super().__new__(cls, name, bases, namespace) + + for bad_cell in (None, 0, "", object()): + with self.subTest(bad_cell=bad_cell): + with self.assertRaises(TypeError): + class A(metaclass=Meta, cell=bad_cell): + pass + + def test___classcell___wrong_cell(self): + # See issue #23722 + # Pointing the cell reference at the wrong class is also prohibited + class Meta(type): + def __new__(cls, name, bases, namespace): + cls = super().__new__(cls, name, bases, namespace) + B = type("B", (), namespace) + return cls + + with self.assertRaises(TypeError): + class A(metaclass=Meta): + def f(self): + return __class__ + def test_obscure_super_errors(self): def f(): super() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 5e0f990e40..e83a4d6426 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -241,9 +241,8 @@ class TestSupport(unittest.TestCase): def test_check_syntax_error(self): support.check_syntax_error(self, "def class", lineno=1, offset=9) - self.assertRaises(AssertionError, support.check_syntax_error, self, "1") - #with self.assertRaises(AssertionError): - #support.check_syntax_error(self, "x=1") + with self.assertRaises(AssertionError): + support.check_syntax_error(self, "x=1") def test_CleanImport(self): import importlib @@ -327,6 +326,28 @@ class TestSupport(unittest.TestCase): self.OtherClass, self.RefClass, ignore=ignore) self.assertEqual(set(), missing_items) + def test_check__all__(self): + extra = {'tempdir'} + blacklist = {'template'} + support.check__all__(self, + tempfile, + extra=extra, + blacklist=blacklist) + + extra = {'TextTestResult', 'installHandler'} + blacklist = {'load_tests', "TestProgram", "BaseTestSuite"} + + support.check__all__(self, + unittest, + ("unittest.result", "unittest.case", + "unittest.suite", "unittest.loader", + "unittest.main", "unittest.runner", + "unittest.signals"), + extra=extra, + blacklist=blacklist) + + self.assertRaises(AssertionError, support.check__all__, self, unittest) + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled diff --git a/Lib/test/test_symbol.py b/Lib/test/test_symbol.py new file mode 100644 index 0000000000..c1306f5432 --- /dev/null +++ b/Lib/test/test_symbol.py @@ -0,0 +1,54 @@ +import unittest +from test import support +import os +import sys +import subprocess + + +SYMBOL_FILE = support.findfile('symbol.py') +GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), + '..', '..', 'Include', 'graminit.h') +TEST_PY_FILE = 'symbol_test.py' + + +class TestSymbolGeneration(unittest.TestCase): + + def _copy_file_without_generated_symbols(self, source_file, dest_file): + with open(source_file) as fp: + lines = fp.readlines() + with open(dest_file, 'w') as fp: + fp.writelines(lines[:lines.index("#--start constants--\n") + 1]) + fp.writelines(lines[lines.index("#--end constants--\n"):]) + + def _generate_symbols(self, grammar_file, target_symbol_py_file): + proc = subprocess.Popen([sys.executable, + SYMBOL_FILE, + grammar_file, + target_symbol_py_file], stderr=subprocess.PIPE) + stderr = proc.communicate()[1] + return proc.returncode, stderr + + def compare_files(self, file1, file2): + with open(file1) as fp: + lines1 = fp.readlines() + with open(file2) as fp: + lines2 = fp.readlines() + self.assertEqual(lines1, lines2) + + @unittest.skipIf(not os.path.exists(GRAMMAR_FILE), + 'test only works from source build directory') + def test_real_grammar_and_symbol_file(self): + output = support.TESTFN + self.addCleanup(support.unlink, output) + + self._copy_file_without_generated_symbols(SYMBOL_FILE, output) + + exitcode, stderr = self._generate_symbols(GRAMMAR_FILE, output) + self.assertEqual(b'', stderr) + self.assertEqual(0, exitcode) + + self.compare_files(SYMBOL_FILE, output) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index fbb1bdc9de..dfaee173ef 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -133,6 +133,17 @@ class SymtableTest(unittest.TestCase): self.assertTrue(self.Mine.lookup("a_method").is_assigned()) self.assertFalse(self.internal.lookup("x").is_assigned()) + def test_annotated(self): + st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec') + st2 = st1.get_children()[0] + self.assertTrue(st2.lookup('x').is_local()) + self.assertTrue(st2.lookup('x').is_annotated()) + self.assertFalse(st2.lookup('x').is_global()) + st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec') + st4 = st3.get_children()[0] + self.assertTrue(st4.lookup('x').is_local()) + self.assertFalse(st4.lookup('x').is_annotated()) + def test_imported(self): self.assertTrue(self.top.lookup("sys").is_imported()) @@ -160,9 +171,11 @@ class SymtableTest(unittest.TestCase): checkfilename("def f(x): foo)(", 14) # parse-time checkfilename("def f(x): global x", 10) # symtable-build-time symtable.symtable("pass", b"spam", "exec") - with self.assertRaises(TypeError): + with self.assertWarns(DeprecationWarning), \ + self.assertRaises(TypeError): symtable.symtable("pass", bytearray(b"spam"), "exec") - symtable.symtable("pass", memoryview(b"spam"), "exec") + with self.assertWarns(DeprecationWarning): + symtable.symtable("pass", memoryview(b"spam"), "exec") with self.assertRaises(TypeError): symtable.symtable("pass", list(b"spam"), "exec") diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 301c142d75..7f7e6dafcf 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -35,14 +35,6 @@ SyntaxError: invalid syntax Traceback (most recent call last): SyntaxError: can't assign to keyword -It's a syntax error to assign to the empty tuple. Why isn't it an -error to assign to the empty list? It will always raise some error at -runtime. - ->>> () = 1 -Traceback (most recent call last): -SyntaxError: can't assign to () - >>> f() = 1 Traceback (most recent call last): SyntaxError: can't assign to function call @@ -374,7 +366,23 @@ build. The number of blocks must be greater than CO_MAXBLOCKS. SF #1565514 ... SyntaxError: too many statically nested blocks -Misuse of the nonlocal statement can lead to a few unique syntax errors. +Misuse of the nonlocal and global statement can lead to a few unique syntax errors. + + >>> def f(): + ... x = 1 + ... global x + Traceback (most recent call last): + ... + SyntaxError: name 'x' is assigned to before global declaration + + >>> def f(): + ... x = 1 + ... def g(): + ... print(x) + ... nonlocal x + Traceback (most recent call last): + ... + SyntaxError: name 'x' is used prior to nonlocal declaration >>> def f(x): ... nonlocal x @@ -493,10 +501,6 @@ Traceback (most recent call last): ... SyntaxError: keyword argument repeated ->>> del () -Traceback (most recent call last): -SyntaxError: can't delete () - >>> {1, 2, 3} = 42 Traceback (most recent call last): SyntaxError: can't assign to literal @@ -580,7 +584,7 @@ class SyntaxTestCase(unittest.TestCase): global b # SyntaxWarning """ warnings.filterwarnings(action='ignore', category=SyntaxWarning) - self._check_error(source, "global", lineno=3, offset=16) + self._check_error(source, "global") warnings.filters.pop(0) def test_break_outside_loop(self): diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 4435d6995b..df9ebd4085 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -10,6 +10,7 @@ import codecs import gc import sysconfig import platform +import locale # count the number of test runs, used to create unique # strings to intern in test_intern() @@ -627,6 +628,8 @@ class SysModuleTest(unittest.TestCase): @unittest.skipUnless(test.support.FS_NONASCII, 'requires OS support of non-ASCII encodings') + @unittest.skipUnless(sys.getfilesystemencoding() == locale.getpreferredencoding(False), + 'requires FS encoding to match locale') def test_ioencoding_nonascii(self): env = dict(os.environ) @@ -669,8 +672,6 @@ class SysModuleTest(unittest.TestCase): fs_encoding = sys.getfilesystemencoding() if sys.platform == 'darwin': expected = 'utf-8' - elif sys.platform == 'win32': - expected = 'mbcs' else: expected = None self.check_fsencoding(fs_encoding, expected) @@ -912,13 +913,13 @@ class SizeofTest(unittest.TestCase): return inner check(get_cell().__closure__[0], size('P')) # code - check(get_cell().__code__, size('5i9Pi3P')) - check(get_cell.__code__, size('5i9Pi3P')) + check(get_cell().__code__, size('6i13P')) + check(get_cell.__code__, size('6i13P')) def get_cell2(x): def inner(): return x return inner - check(get_cell2.__code__, size('5i9Pi3P') + 1) + check(get_cell2.__code__, size('6i13P') + 1) # complex check(complex(0,1), size('2d')) # method_descriptor (descriptor object) @@ -936,9 +937,9 @@ class SizeofTest(unittest.TestCase): # method-wrapper (descriptor object) check({}.__iter__, size('2P')) # dict - check({}, size('n2P') + calcsize('2nPn') + 8*calcsize('n2P')) + check({}, size('nQ2P') + calcsize('2nP2n') + 8 + (8*2//3)*calcsize('n2P')) longdict = {1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8} - check(longdict, size('n2P') + calcsize('2nPn') + 16*calcsize('n2P')) + check(longdict, size('nQ2P') + calcsize('2nP2n') + 16 + (16*2//3)*calcsize('n2P')) # dictionary-keyview check({}.keys(), size('P')) # dictionary-valueview @@ -1096,13 +1097,13 @@ class SizeofTest(unittest.TestCase): '10P' # PySequenceMethods '2P' # PyBufferProcs '4P') - # Separate block for PyDictKeysObject with 4 entries - s += calcsize("2nPn") + 4*calcsize("n2P") + # Separate block for PyDictKeysObject with 8 keys and 5 entries + s += calcsize("2nP2n") + 8 + 5*calcsize("n2P") # class class newstyleclass(object): pass check(newstyleclass, s) # dict with shared keys - check(newstyleclass().__dict__, size('n2P' + '2nPn')) + check(newstyleclass().__dict__, size('nQ2P' + '2nP2n')) # unicode # each tuple contains a string and its expected character size # don't put any static strings here, as they may contain @@ -1191,6 +1192,32 @@ class SizeofTest(unittest.TestCase): # sys.flags check(sys.flags, vsize('') + self.P * len(sys.flags)) + def test_asyncgen_hooks(self): + old = sys.get_asyncgen_hooks() + self.assertIsNone(old.firstiter) + self.assertIsNone(old.finalizer) + + firstiter = lambda *a: None + sys.set_asyncgen_hooks(firstiter=firstiter) + hooks = sys.get_asyncgen_hooks() + self.assertIs(hooks.firstiter, firstiter) + self.assertIs(hooks[0], firstiter) + self.assertIs(hooks.finalizer, None) + self.assertIs(hooks[1], None) + + finalizer = lambda *a: None + sys.set_asyncgen_hooks(finalizer=finalizer) + hooks = sys.get_asyncgen_hooks() + self.assertIs(hooks.firstiter, firstiter) + self.assertIs(hooks[0], firstiter) + self.assertIs(hooks.finalizer, finalizer) + self.assertIs(hooks[1], finalizer) + + sys.set_asyncgen_hooks(*old) + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + def test_main(): test.support.run_unittest(SysModuleTest, SizeofTest) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 509bc3e505..25c5835fd6 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -338,8 +338,8 @@ class TraceTestCase(unittest.TestCase): def test_14_onliner_if(self): def onliners(): - if True: False - else: True + if True: x=False + else: x=True return 0 self.run_and_compare( onliners, diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 38306790e4..a29ca96d1f 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -386,7 +386,9 @@ class TestSysConfig(unittest.TestCase): self.assertIsNotNone(vars['SO']) self.assertEqual(vars['SO'], vars['EXT_SUFFIX']) - @unittest.skipUnless(sys.platform == 'linux', 'Linux-specific test') + @unittest.skipUnless(sys.platform == 'linux' and + hasattr(sys.implementation, '_multiarch'), + 'multiarch-specific test') def test_triplet_in_ext_suffix(self): ctypes = import_module('ctypes') import platform, re @@ -396,8 +398,8 @@ class TestSysConfig(unittest.TestCase): self.assertTrue('linux' in suffix, suffix) if re.match('(i[3-6]86|x86_64)$', machine): if ctypes.sizeof(ctypes.c_char_p()) == 4: - self.assertTrue(suffix.endswith('i386-linux-gnu.so') \ - or suffix.endswith('x86_64-linux-gnux32.so'), + self.assertTrue(suffix.endswith('i386-linux-gnu.so') or + suffix.endswith('x86_64-linux-gnux32.so'), suffix) else: # 8 byte pointer size self.assertTrue(suffix.endswith('x86_64-linux-gnu.so'), suffix) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 1efb841bb2..619cbc03b2 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -2080,6 +2080,24 @@ class MiscTest(unittest.TestCase): with self.assertRaises(ValueError): tarfile.itn(0x10000000000, 6, tarfile.GNU_FORMAT) + def test__all__(self): + blacklist = {'version', 'grp', 'pwd', 'symlink_exception', + 'NUL', 'BLOCKSIZE', 'RECORDSIZE', 'GNU_MAGIC', + 'POSIX_MAGIC', 'LENGTH_NAME', 'LENGTH_LINK', + 'LENGTH_PREFIX', 'REGTYPE', 'AREGTYPE', 'LNKTYPE', + 'SYMTYPE', 'CHRTYPE', 'BLKTYPE', 'DIRTYPE', 'FIFOTYPE', + 'CONTTYPE', 'GNUTYPE_LONGNAME', 'GNUTYPE_LONGLINK', + 'GNUTYPE_SPARSE', 'XHDTYPE', 'XGLTYPE', 'SOLARIS_XHDTYPE', + 'SUPPORTED_TYPES', 'REGULAR_TYPES', 'GNU_TYPES', + 'PAX_FIELDS', 'PAX_NAME_FIELDS', 'PAX_NUMBER_FIELDS', + 'stn', 'nts', 'nti', 'itn', 'calc_chksums', 'copyfileobj', + 'filemode', + 'EmptyHeaderError', 'TruncatedHeaderError', + 'EOFHeaderError', 'InvalidHeaderError', + 'SubsequentHeaderError', 'ExFileObject', + 'main'} + support.check__all__(self, tarfile, blacklist=blacklist) + class CommandLineTest(unittest.TestCase): diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py index 8e219f4217..51d82e1158 100644 --- a/Lib/test/test_telnetlib.py +++ b/Lib/test/test_telnetlib.py @@ -1,7 +1,6 @@ import socket import selectors import telnetlib -import time import contextlib from test import support @@ -42,6 +41,11 @@ class GeneralTests(unittest.TestCase): telnet = telnetlib.Telnet(HOST, self.port) telnet.sock.close() + def testContextManager(self): + with telnetlib.Telnet(HOST, self.port) as tn: + self.assertIsNotNone(tn.get_socket()) + self.assertIsNone(tn.get_socket()) + def testTimeoutDefault(self): self.assertTrue(socket.getdefaulttimeout() is None) socket.setdefaulttimeout(30) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index b63050982a..2c2914fd6d 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -8,7 +8,6 @@ from test.support import (verbose, import_module, cpython_only, from test.support.script_helper import assert_python_ok, assert_python_failure import random -import re import sys _thread = import_module('_thread') threading = import_module('threading') @@ -19,6 +18,7 @@ import os import subprocess from test import lock_tests +from test import support # Between fork() and exec(), only async-safe functions are allowed (issues @@ -462,7 +462,7 @@ class ThreadTests(BaseTestCase): self.addCleanup(sys.setswitchinterval, old_interval) # Make the bug more likely to manifest. - sys.setswitchinterval(1e-6) + test.support.setswitchinterval(1e-6) for i in range(20): t = threading.Thread(target=lambda: None) @@ -1117,5 +1117,12 @@ class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): class BarrierTests(lock_tests.BarrierTests): barriertype = staticmethod(threading.Barrier) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + extra = {"ThreadError"} + blacklist = {'currentThread', 'activeCount'} + support.check__all__(self, threading, ('threading', '_thread'), + extra=extra, blacklist=blacklist) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 76b894eece..f2242126d1 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1,6 +1,8 @@ from test import support +import decimal import enum import locale +import math import platform import sys import sysconfig @@ -21,17 +23,27 @@ SIZEOF_INT = sysconfig.get_config_var('SIZEOF_INT') or 4 TIME_MAXYEAR = (1 << 8 * SIZEOF_INT - 1) - 1 TIME_MINYEAR = -TIME_MAXYEAR - 1 +SEC_TO_US = 10 ** 6 US_TO_NS = 10 ** 3 MS_TO_NS = 10 ** 6 SEC_TO_NS = 10 ** 9 +NS_TO_SEC = 10 ** 9 class _PyTime(enum.IntEnum): # Round towards minus infinity (-inf) ROUND_FLOOR = 0 # Round towards infinity (+inf) ROUND_CEILING = 1 + # Round to nearest with ties going to nearest even integer + ROUND_HALF_EVEN = 2 -ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING) +# Rounding modes supported by PyTime +ROUNDING_MODES = ( + # (PyTime rounding method, decimal rounding method) + (_PyTime.ROUND_FLOOR, decimal.ROUND_FLOOR), + (_PyTime.ROUND_CEILING, decimal.ROUND_CEILING), + (_PyTime.ROUND_HALF_EVEN, decimal.ROUND_HALF_EVEN), +) class TimeTestCase(unittest.TestCase): @@ -607,79 +619,6 @@ class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear, unittest.TestCase): class TestPytime(unittest.TestCase): - def setUp(self): - self.invalid_values = ( - -(2 ** 100), 2 ** 100, - -(2.0 ** 100.0), 2.0 ** 100.0, - ) - - @support.cpython_only - def test_time_t(self): - from _testcapi import pytime_object_to_time_t - for obj, time_t, rnd in ( - # Round towards minus infinity (-inf) - (0, 0, _PyTime.ROUND_FLOOR), - (-1, -1, _PyTime.ROUND_FLOOR), - (-1.0, -1, _PyTime.ROUND_FLOOR), - (-1.9, -2, _PyTime.ROUND_FLOOR), - (1.0, 1, _PyTime.ROUND_FLOOR), - (1.9, 1, _PyTime.ROUND_FLOOR), - # Round towards infinity (+inf) - (0, 0, _PyTime.ROUND_CEILING), - (-1, -1, _PyTime.ROUND_CEILING), - (-1.0, -1, _PyTime.ROUND_CEILING), - (-1.9, -1, _PyTime.ROUND_CEILING), - (1.0, 1, _PyTime.ROUND_CEILING), - (1.9, 2, _PyTime.ROUND_CEILING), - ): - self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) - - rnd = _PyTime.ROUND_FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_time_t, invalid, rnd) - - @support.cpython_only - def test_timespec(self): - from _testcapi import pytime_object_to_timespec - for obj, timespec, rnd in ( - # Round towards minus infinity (-inf) - (0, (0, 0), _PyTime.ROUND_FLOOR), - (-1, (-1, 0), _PyTime.ROUND_FLOOR), - (-1.0, (-1, 0), _PyTime.ROUND_FLOOR), - (1e-9, (0, 1), _PyTime.ROUND_FLOOR), - (1e-10, (0, 0), _PyTime.ROUND_FLOOR), - (-1e-9, (-1, 999999999), _PyTime.ROUND_FLOOR), - (-1e-10, (-1, 999999999), _PyTime.ROUND_FLOOR), - (-1.2, (-2, 800000000), _PyTime.ROUND_FLOOR), - (0.9999999999, (0, 999999999), _PyTime.ROUND_FLOOR), - (1.1234567890, (1, 123456789), _PyTime.ROUND_FLOOR), - (1.1234567899, (1, 123456789), _PyTime.ROUND_FLOOR), - (-1.1234567890, (-2, 876543211), _PyTime.ROUND_FLOOR), - (-1.1234567891, (-2, 876543210), _PyTime.ROUND_FLOOR), - # Round towards infinity (+inf) - (0, (0, 0), _PyTime.ROUND_CEILING), - (-1, (-1, 0), _PyTime.ROUND_CEILING), - (-1.0, (-1, 0), _PyTime.ROUND_CEILING), - (1e-9, (0, 1), _PyTime.ROUND_CEILING), - (1e-10, (0, 1), _PyTime.ROUND_CEILING), - (-1e-9, (-1, 999999999), _PyTime.ROUND_CEILING), - (-1e-10, (0, 0), _PyTime.ROUND_CEILING), - (-1.2, (-2, 800000000), _PyTime.ROUND_CEILING), - (0.9999999999, (1, 0), _PyTime.ROUND_CEILING), - (1.1234567890, (1, 123456790), _PyTime.ROUND_CEILING), - (1.1234567899, (1, 123456790), _PyTime.ROUND_CEILING), - (-1.1234567890, (-2, 876543211), _PyTime.ROUND_CEILING), - (-1.1234567891, (-2, 876543211), _PyTime.ROUND_CEILING), - ): - with self.subTest(obj=obj, round=rnd, timespec=timespec): - self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) - - rnd = _PyTime.ROUND_FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_timespec, invalid, rnd) - @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_localtime_timezone(self): @@ -734,266 +673,291 @@ class TestPytime(unittest.TestCase): self.assertIs(lt.tm_zone, None) -@unittest.skipUnless(_testcapi is not None, - 'need the _testcapi module') -class TestPyTime_t(unittest.TestCase): +@unittest.skipIf(_testcapi is None, 'need the _testcapi module') +class CPyTimeTestCase: + """ + Base class to test the C _PyTime_t API. + """ + OVERFLOW_SECONDS = None + + def setUp(self): + from _testcapi import SIZEOF_TIME_T + bits = SIZEOF_TIME_T * 8 - 1 + self.time_t_min = -2 ** bits + self.time_t_max = 2 ** bits - 1 + + def time_t_filter(self, seconds): + return (self.time_t_min <= seconds <= self.time_t_max) + + def _rounding_values(self, use_float): + "Build timestamps used to test rounding." + + units = [1, US_TO_NS, MS_TO_NS, SEC_TO_NS] + if use_float: + # picoseconds are only tested to pytime_converter accepting floats + units.append(1e-3) + + values = ( + # small values + 1, 2, 5, 7, 123, 456, 1234, + # 10^k - 1 + 9, + 99, + 999, + 9999, + 99999, + 999999, + # test half even rounding near 0.5, 1.5, 2.5, 3.5, 4.5 + 499, 500, 501, + 1499, 1500, 1501, + 2500, + 3500, + 4500, + ) + + ns_timestamps = [0] + for unit in units: + for value in values: + ns = value * unit + ns_timestamps.extend((-ns, ns)) + for pow2 in (0, 5, 10, 15, 22, 23, 24, 30, 33): + ns = (2 ** pow2) * SEC_TO_NS + ns_timestamps.extend(( + -ns-1, -ns, -ns+1, + ns-1, ns, ns+1 + )) + for seconds in (_testcapi.INT_MIN, _testcapi.INT_MAX): + ns_timestamps.append(seconds * SEC_TO_NS) + if use_float: + # numbers with an exact representation in IEEE 754 (base 2) + for pow2 in (3, 7, 10, 15): + ns = 2.0 ** (-pow2) + ns_timestamps.extend((-ns, ns)) + + # seconds close to _PyTime_t type limit + ns = (2 ** 63 // SEC_TO_NS) * SEC_TO_NS + ns_timestamps.extend((-ns, ns)) + + return ns_timestamps + + def _check_rounding(self, pytime_converter, expected_func, + use_float, unit_to_sec, value_filter=None): + + def convert_values(ns_timestamps): + if use_float: + unit_to_ns = SEC_TO_NS / float(unit_to_sec) + values = [ns / unit_to_ns for ns in ns_timestamps] + else: + unit_to_ns = SEC_TO_NS // unit_to_sec + values = [ns // unit_to_ns for ns in ns_timestamps] + + if value_filter: + values = filter(value_filter, values) + + # remove duplicates and sort + return sorted(set(values)) + + # test rounding + ns_timestamps = self._rounding_values(use_float) + valid_values = convert_values(ns_timestamps) + for time_rnd, decimal_rnd in ROUNDING_MODES : + context = decimal.getcontext() + context.rounding = decimal_rnd + + for value in valid_values: + debug_info = {'value': value, 'rounding': decimal_rnd} + try: + result = pytime_converter(value, time_rnd) + expected = expected_func(value) + except Exception as exc: + self.fail("Error on timestamp conversion: %s" % debug_info) + self.assertEqual(result, + expected, + debug_info) + + # test overflow + ns = self.OVERFLOW_SECONDS * SEC_TO_NS + ns_timestamps = (-ns, ns) + overflow_values = convert_values(ns_timestamps) + for time_rnd, _ in ROUNDING_MODES : + for value in overflow_values: + debug_info = {'value': value, 'rounding': time_rnd} + with self.assertRaises(OverflowError, msg=debug_info): + pytime_converter(value, time_rnd) + + def check_int_rounding(self, pytime_converter, expected_func, + unit_to_sec=1, value_filter=None): + self._check_rounding(pytime_converter, expected_func, + False, unit_to_sec, value_filter) + + def check_float_rounding(self, pytime_converter, expected_func, + unit_to_sec=1, value_filter=None): + self._check_rounding(pytime_converter, expected_func, + True, unit_to_sec, value_filter) + + def decimal_round(self, x): + d = decimal.Decimal(x) + d = d.quantize(1) + return int(d) + + +class TestCPyTime(CPyTimeTestCase, unittest.TestCase): + """ + Test the C _PyTime_t API. + """ + # _PyTime_t is a 64-bit signed integer + OVERFLOW_SECONDS = math.ceil((2**63 + 1) / SEC_TO_NS) + def test_FromSeconds(self): from _testcapi import PyTime_FromSeconds - for seconds in (0, 3, -456, _testcapi.INT_MAX, _testcapi.INT_MIN): - with self.subTest(seconds=seconds): - self.assertEqual(PyTime_FromSeconds(seconds), - seconds * SEC_TO_NS) + + # PyTime_FromSeconds() expects a C int, reject values out of range + def c_int_filter(secs): + return (_testcapi.INT_MIN <= secs <= _testcapi.INT_MAX) + + self.check_int_rounding(lambda secs, rnd: PyTime_FromSeconds(secs), + lambda secs: secs * SEC_TO_NS, + value_filter=c_int_filter) def test_FromSecondsObject(self): from _testcapi import PyTime_FromSecondsObject - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, ts in ( - # integers - (0, 0), - (1, SEC_TO_NS), - (-3, -3 * SEC_TO_NS), - - # float: subseconds - (0.0, 0), - (1e-9, 1), - (1e-6, 10 ** 3), - (1e-3, 10 ** 6), - - # float: seconds - (2.0, 2 * SEC_TO_NS), - (123.0, 123 * SEC_TO_NS), - (-7.0, -7 * SEC_TO_NS), - - # nanosecond are kept for value <= 2^23 seconds - (2**22 - 1e-9, 4194303999999999), - (2**22, 4194304000000000), - (2**22 + 1e-9, 4194304000000001), - (2**23 - 1e-9, 8388607999999999), - (2**23, 8388608000000000), - - # start losing precision for value > 2^23 seconds - (2**23 + 1e-9, 8388608000000002), - - # nanoseconds are lost for value > 2^23 seconds - (2**24 - 1e-9, 16777215999999998), - (2**24, 16777216000000000), - (2**24 + 1e-9, 16777216000000000), - (2**25 - 1e-9, 33554432000000000), - (2**25 , 33554432000000000), - (2**25 + 1e-9, 33554432000000000), - - # close to 2^63 nanoseconds (_PyTime_t limit) - (9223372036, 9223372036 * SEC_TO_NS), - (9223372036.0, 9223372036 * SEC_TO_NS), - (-9223372036, -9223372036 * SEC_TO_NS), - (-9223372036.0, -9223372036 * SEC_TO_NS), - ): - with self.subTest(obj=obj, round=rnd, timestamp=ts): - self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) - - with self.subTest(round=rnd): - with self.assertRaises(OverflowError): - PyTime_FromSecondsObject(9223372037, rnd) - PyTime_FromSecondsObject(9223372037.0, rnd) - PyTime_FromSecondsObject(-9223372037, rnd) - PyTime_FromSecondsObject(-9223372037.0, rnd) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - for obj, ts, rnd in ( - # close to zero - ( 1e-10, 0, FLOOR), - ( 1e-10, 1, CEILING), - (-1e-10, -1, FLOOR), - (-1e-10, 0, CEILING), - - # test rounding of the last nanosecond - ( 1.1234567899, 1123456789, FLOOR), - ( 1.1234567899, 1123456790, CEILING), - (-1.1234567899, -1123456790, FLOOR), - (-1.1234567899, -1123456789, CEILING), - - # close to 1 second - ( 0.9999999999, 999999999, FLOOR), - ( 0.9999999999, 1000000000, CEILING), - (-0.9999999999, -1000000000, FLOOR), - (-0.9999999999, -999999999, CEILING), - ): - with self.subTest(obj=obj, round=rnd, timestamp=ts): - self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) + self.check_int_rounding( + PyTime_FromSecondsObject, + lambda secs: secs * SEC_TO_NS) + + self.check_float_rounding( + PyTime_FromSecondsObject, + lambda ns: self.decimal_round(ns * SEC_TO_NS)) def test_AsSecondsDouble(self): from _testcapi import PyTime_AsSecondsDouble - for nanoseconds, seconds in ( - # near 1 nanosecond - ( 0, 0.0), - ( 1, 1e-9), - (-1, -1e-9), - - # near 1 second - (SEC_TO_NS + 1, 1.0 + 1e-9), - (SEC_TO_NS, 1.0), - (SEC_TO_NS - 1, 1.0 - 1e-9), - - # a few seconds - (123 * SEC_TO_NS, 123.0), - (-567 * SEC_TO_NS, -567.0), - - # nanosecond are kept for value <= 2^23 seconds - (4194303999999999, 2**22 - 1e-9), - (4194304000000000, 2**22), - (4194304000000001, 2**22 + 1e-9), - - # start losing precision for value > 2^23 seconds - (8388608000000002, 2**23 + 1e-9), - - # nanoseconds are lost for value > 2^23 seconds - (16777215999999998, 2**24 - 1e-9), - (16777215999999999, 2**24 - 1e-9), - (16777216000000000, 2**24 ), - (16777216000000001, 2**24 ), - (16777216000000002, 2**24 + 2e-9), - - (33554432000000000, 2**25 ), - (33554432000000002, 2**25 ), - (33554432000000004, 2**25 + 4e-9), - - # close to 2^63 nanoseconds (_PyTime_t limit) - (9223372036 * SEC_TO_NS, 9223372036.0), - (-9223372036 * SEC_TO_NS, -9223372036.0), - ): - with self.subTest(nanoseconds=nanoseconds, seconds=seconds): - self.assertEqual(PyTime_AsSecondsDouble(nanoseconds), - seconds) - - def test_timeval(self): + def float_converter(ns): + if abs(ns) % SEC_TO_NS == 0: + return float(ns // SEC_TO_NS) + else: + return float(ns) / SEC_TO_NS + + self.check_int_rounding(lambda ns, rnd: PyTime_AsSecondsDouble(ns), + float_converter, + NS_TO_SEC) + + def create_decimal_converter(self, denominator): + denom = decimal.Decimal(denominator) + + def converter(value): + d = decimal.Decimal(value) / denom + return self.decimal_round(d) + + return converter + + def test_AsTimeval(self): from _testcapi import PyTime_AsTimeval - for rnd in ALL_ROUNDING_METHODS: - for ns, tv in ( - # microseconds - (0, (0, 0)), - (1000, (0, 1)), - (-1000, (-1, 999999)), - - # seconds - (2 * SEC_TO_NS, (2, 0)), - (-3 * SEC_TO_NS, (-3, 0)), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) - - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - for ns, tv, rnd in ( - # nanoseconds - (1, (0, 0), FLOOR), - (1, (0, 1), CEILING), - (-1, (-1, 999999), FLOOR), - (-1, (0, 0), CEILING), - - # seconds + nanoseconds - (1234567001, (1, 234567), FLOOR), - (1234567001, (1, 234568), CEILING), - (-1234567001, (-2, 765432), FLOOR), - (-1234567001, (-2, 765433), CEILING), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) + + us_converter = self.create_decimal_converter(US_TO_NS) + + def timeval_converter(ns): + us = us_converter(ns) + return divmod(us, SEC_TO_US) + + if sys.platform == 'win32': + from _testcapi import LONG_MIN, LONG_MAX + + # On Windows, timeval.tv_sec type is a C long + def seconds_filter(secs): + return LONG_MIN <= secs <= LONG_MAX + else: + seconds_filter = self.time_t_filter + + self.check_int_rounding(PyTime_AsTimeval, + timeval_converter, + NS_TO_SEC, + value_filter=seconds_filter) @unittest.skipUnless(hasattr(_testcapi, 'PyTime_AsTimespec'), 'need _testcapi.PyTime_AsTimespec') - def test_timespec(self): + def test_AsTimespec(self): from _testcapi import PyTime_AsTimespec - for ns, ts in ( - # nanoseconds - (0, (0, 0)), - (1, (0, 1)), - (-1, (-1, 999999999)), - - # seconds - (2 * SEC_TO_NS, (2, 0)), - (-3 * SEC_TO_NS, (-3, 0)), - - # seconds + nanoseconds - (1234567890, (1, 234567890)), - (-1234567890, (-2, 765432110)), - ): - with self.subTest(nanoseconds=ns, timespec=ts): - self.assertEqual(PyTime_AsTimespec(ns), ts) - - def test_milliseconds(self): + + def timespec_converter(ns): + return divmod(ns, SEC_TO_NS) + + self.check_int_rounding(lambda ns, rnd: PyTime_AsTimespec(ns), + timespec_converter, + NS_TO_SEC, + value_filter=self.time_t_filter) + + def test_AsMilliseconds(self): from _testcapi import PyTime_AsMilliseconds - for rnd in ALL_ROUNDING_METHODS: - for ns, tv in ( - # milliseconds - (1 * MS_TO_NS, 1), - (-2 * MS_TO_NS, -2), - - # seconds - (2 * SEC_TO_NS, 2000), - (-3 * SEC_TO_NS, -3000), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsMilliseconds(ns, rnd), tv) - - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - for ns, ms, rnd in ( - # nanoseconds - (1, 0, FLOOR), - (1, 1, CEILING), - (-1, -1, FLOOR), - (-1, 0, CEILING), - - # seconds + nanoseconds - (1234 * MS_TO_NS + 1, 1234, FLOOR), - (1234 * MS_TO_NS + 1, 1235, CEILING), - (-1234 * MS_TO_NS - 1, -1235, FLOOR), - (-1234 * MS_TO_NS - 1, -1234, CEILING), - ): - with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): - self.assertEqual(PyTime_AsMilliseconds(ns, rnd), ms) - - def test_microseconds(self): + + self.check_int_rounding(PyTime_AsMilliseconds, + self.create_decimal_converter(MS_TO_NS), + NS_TO_SEC) + + def test_AsMicroseconds(self): from _testcapi import PyTime_AsMicroseconds - for rnd in ALL_ROUNDING_METHODS: - for ns, tv in ( - # microseconds - (1 * US_TO_NS, 1), - (-2 * US_TO_NS, -2), - - # milliseconds - (1 * MS_TO_NS, 1000), - (-2 * MS_TO_NS, -2000), - - # seconds - (2 * SEC_TO_NS, 2000000), - (-3 * SEC_TO_NS, -3000000), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsMicroseconds(ns, rnd), tv) - - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - for ns, ms, rnd in ( - # nanoseconds - (1, 0, FLOOR), - (1, 1, CEILING), - (-1, -1, FLOOR), - (-1, 0, CEILING), - - # seconds + nanoseconds - (1234 * US_TO_NS + 1, 1234, FLOOR), - (1234 * US_TO_NS + 1, 1235, CEILING), - (-1234 * US_TO_NS - 1, -1235, FLOOR), - (-1234 * US_TO_NS - 1, -1234, CEILING), - ): - with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): - self.assertEqual(PyTime_AsMicroseconds(ns, rnd), ms) + + self.check_int_rounding(PyTime_AsMicroseconds, + self.create_decimal_converter(US_TO_NS), + NS_TO_SEC) + + +class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): + """ + Test the old C _PyTime_t API: _PyTime_ObjectToXXX() functions. + """ + + # time_t is a 32-bit or 64-bit signed integer + OVERFLOW_SECONDS = 2 ** 64 + + def test_object_to_time_t(self): + from _testcapi import pytime_object_to_time_t + + self.check_int_rounding(pytime_object_to_time_t, + lambda secs: secs, + value_filter=self.time_t_filter) + + self.check_float_rounding(pytime_object_to_time_t, + self.decimal_round, + value_filter=self.time_t_filter) + + def create_converter(self, sec_to_unit): + def converter(secs): + floatpart, intpart = math.modf(secs) + intpart = int(intpart) + floatpart *= sec_to_unit + floatpart = self.decimal_round(floatpart) + if floatpart < 0: + floatpart += sec_to_unit + intpart -= 1 + elif floatpart >= sec_to_unit: + floatpart -= sec_to_unit + intpart += 1 + return (intpart, floatpart) + return converter + + def test_object_to_timeval(self): + from _testcapi import pytime_object_to_timeval + + self.check_int_rounding(pytime_object_to_timeval, + lambda secs: (secs, 0), + value_filter=self.time_t_filter) + + self.check_float_rounding(pytime_object_to_timeval, + self.create_converter(SEC_TO_US), + value_filter=self.time_t_filter) + + def test_object_to_timespec(self): + from _testcapi import pytime_object_to_timespec + + self.check_int_rounding(pytime_object_to_timespec, + lambda secs: (secs, 0), + value_filter=self.time_t_filter) + + self.check_float_rounding(pytime_object_to_timespec, + self.create_converter(SEC_TO_NS), + value_filter=self.time_t_filter) if __name__ == "__main__": diff --git a/Lib/test/test_timeit.py b/Lib/test/test_timeit.py index 2db3c1bed2..1a95e2979c 100644 --- a/Lib/test/test_timeit.py +++ b/Lib/test/test_timeit.py @@ -354,6 +354,28 @@ class TestTimeit(unittest.TestCase): s = self.run_main(switches=['-n1', '1/0']) self.assert_exc_string(error_stringio.getvalue(), 'ZeroDivisionError') + def autorange(self, callback=None): + timer = FakeTimer(seconds_per_increment=0.001) + t = timeit.Timer(stmt=self.fake_stmt, setup=self.fake_setup, timer=timer) + return t.autorange(callback) + + def test_autorange(self): + num_loops, time_taken = self.autorange() + self.assertEqual(num_loops, 1000) + self.assertEqual(time_taken, 1.0) + + def test_autorange_with_callback(self): + def callback(a, b): + print("{} {:.3f}".format(a, b)) + with captured_stdout() as s: + num_loops, time_taken = self.autorange(callback) + self.assertEqual(num_loops, 1000) + self.assertEqual(time_taken, 1.0) + expected = ('10 0.010\n' + '100 0.100\n' + '1000 1.000\n') + self.assertEqual(s.getvalue(), expected) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 3b17ca6329..5a81a5f11a 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -4,6 +4,8 @@ from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP, open as tokenize_open, Untokenizer) from io import BytesIO from unittest import TestCase, mock +from test.test_grammar import (VALID_UNDERSCORE_LITERALS, + INVALID_UNDERSCORE_LITERALS) import os import token @@ -24,8 +26,7 @@ class TokenizeTest(TestCase): if type == ENDMARKER: break type = tok_name[type] - result.append(" %(type)-10.10s %(token)-13.13r %(start)s %(end)s" % - locals()) + result.append(f" {type:10} {token!r:13} {start} {end}") self.assertEqual(result, [" ENCODING 'utf-8' (0, 0) (0, 0)"] + expected.rstrip().splitlines()) @@ -132,18 +133,18 @@ def k(x): self.check_tokenize("x = 0xfffffffffff", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) - NUMBER '0xffffffffff (1, 4) (1, 17) + NUMBER '0xfffffffffff' (1, 4) (1, 17) """) self.check_tokenize("x = 123141242151251616110", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) - NUMBER '123141242151 (1, 4) (1, 25) + NUMBER '123141242151251616110' (1, 4) (1, 25) """) self.check_tokenize("x = -15921590215012591", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '-' (1, 4) (1, 5) - NUMBER '159215902150 (1, 5) (1, 22) + NUMBER '15921590215012591' (1, 5) (1, 22) """) def test_float(self): @@ -186,6 +187,21 @@ def k(x): NUMBER '3.14e159' (1, 4) (1, 12) """) + def test_underscore_literals(self): + def number_token(s): + f = BytesIO(s.encode('utf-8')) + for toktype, token, start, end, line in tokenize(f.readline): + if toktype == NUMBER: + return token + return 'invalid token' + for lit in VALID_UNDERSCORE_LITERALS: + if '(' in lit: + # this won't work with compound complex inputs + continue + self.assertEqual(number_token(lit), lit) + for lit in INVALID_UNDERSCORE_LITERALS: + self.assertNotEqual(number_token(lit), lit) + def test_string(self): # String literals self.check_tokenize("x = ''; y = \"\"", """\ @@ -307,6 +323,50 @@ def k(x): OP '+' (1, 28) (1, 29) STRING 'RB"abc"' (1, 30) (1, 37) """) + # Check 0, 1, and 2 character string prefixes. + self.check_tokenize(r'"a\ +de\ +fg"', """\ + STRING '"a\\\\\\nde\\\\\\nfg"\' (1, 0) (3, 3) + """) + self.check_tokenize(r'u"a\ +de"', """\ + STRING 'u"a\\\\\\nde"\' (1, 0) (2, 3) + """) + self.check_tokenize(r'rb"a\ +d"', """\ + STRING 'rb"a\\\\\\nd"\' (1, 0) (2, 2) + """) + self.check_tokenize(r'"""a\ +b"""', """\ + STRING '\"\""a\\\\\\nb\"\""' (1, 0) (2, 4) + """) + self.check_tokenize(r'u"""a\ +b"""', """\ + STRING 'u\"\""a\\\\\\nb\"\""' (1, 0) (2, 4) + """) + self.check_tokenize(r'rb"""a\ +b\ +c"""', """\ + STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4) + """) + self.check_tokenize('f"abc"', """\ + STRING 'f"abc"' (1, 0) (1, 6) + """) + self.check_tokenize('fR"a{b}c"', """\ + STRING 'fR"a{b}c"' (1, 0) (1, 9) + """) + self.check_tokenize('f"""abc"""', """\ + STRING 'f\"\"\"abc\"\"\"' (1, 0) (1, 10) + """) + self.check_tokenize(r'f"abc\ +def"', """\ + STRING 'f"abc\\\\\\ndef"' (1, 0) (2, 4) + """) + self.check_tokenize(r'Rf"abc\ +def"', """\ + STRING 'Rf"abc\\\\\\ndef"' (1, 0) (2, 4) + """) def test_function(self): self.check_tokenize("def d22(a, b, c=2, d=2, *k): pass", """\ @@ -505,7 +565,7 @@ def k(x): # Methods self.check_tokenize("@staticmethod\ndef foo(x,y): pass", """\ OP '@' (1, 0) (1, 1) - NAME 'staticmethod (1, 1) (1, 13) + NAME 'staticmethod' (1, 1) (1, 13) NEWLINE '\\n' (1, 13) (1, 14) NAME 'def' (2, 0) (2, 3) NAME 'foo' (2, 4) (2, 7) @@ -1488,10 +1548,10 @@ class TestRoundtrip(TestCase): # Tokenize is broken on test_pep3131.py because regular expressions are # broken on the obscure unicode identifiers in it. *sigh* - # With roundtrip extended to test the 5-tuple mode of untokenize, + # With roundtrip extended to test the 5-tuple mode of untokenize, # 7 more testfiles fail. Remove them also until the failure is diagnosed. - testfiles.remove(os.path.join(tempdir, "test_pep3131.py")) + testfiles.remove(os.path.join(tempdir, "test_unicode_identifiers.py")) for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'): testfiles.remove(os.path.join(tempdir, "test_%s.py") % f) diff --git a/Lib/test/test_tools/__init__.py b/Lib/test/test_tools/__init__.py index 04c8726e11..4d0fca330a 100644 --- a/Lib/test/test_tools/__init__.py +++ b/Lib/test/test_tools/__init__.py @@ -3,7 +3,6 @@ import os import unittest import importlib from test import support -from fnmatch import fnmatch basepath = os.path.dirname( # <src/install dir> os.path.dirname( # Lib diff --git a/Lib/test/test_tools/test_gprof2html.py b/Lib/test/test_tools/test_gprof2html.py index 0c294ec0c4..9489ed91f9 100644 --- a/Lib/test/test_tools/test_gprof2html.py +++ b/Lib/test/test_tools/test_gprof2html.py @@ -2,12 +2,11 @@ import os import sys -import importlib import unittest from unittest import mock import tempfile -from test.test_tools import scriptsdir, skip_if_missing, import_tool +from test.test_tools import skip_if_missing, import_tool skip_if_missing() diff --git a/Lib/test/test_tools/test_md5sum.py b/Lib/test/test_tools/test_md5sum.py index 1305295ef1..fb565b7377 100644 --- a/Lib/test/test_tools/test_md5sum.py +++ b/Lib/test/test_tools/test_md5sum.py @@ -1,12 +1,11 @@ """Tests for the md5sum script in the Tools directory.""" import os -import sys import unittest from test import support from test.support.script_helper import assert_python_ok, assert_python_failure -from test.test_tools import scriptsdir, import_tool, skip_if_missing +from test.test_tools import scriptsdir, skip_if_missing skip_if_missing() diff --git a/Lib/test/test_tools/test_pdeps.py b/Lib/test/test_tools/test_pdeps.py index 091fa6a06d..27cbfe215d 100644 --- a/Lib/test/test_tools/test_pdeps.py +++ b/Lib/test/test_tools/test_pdeps.py @@ -1,12 +1,10 @@ """Tests for the pdeps script in the Tools directory.""" import os -import sys import unittest import tempfile -from test import support -from test.test_tools import scriptsdir, skip_if_missing, import_tool +from test.test_tools import skip_if_missing, import_tool skip_if_missing() diff --git a/Lib/test/test_tools/test_unparse.py b/Lib/test/test_tools/test_unparse.py index 734bbc215a..65dee1b5ae 100644 --- a/Lib/test/test_tools/test_unparse.py +++ b/Lib/test/test_tools/test_unparse.py @@ -134,6 +134,11 @@ class ASTTestCase(unittest.TestCase): class UnparseTestCase(ASTTestCase): # Tests for specific bugs found in earlier versions of unparse + def test_fstrings(self): + # See issue 25180 + self.check_roundtrip(r"""f'{f"{0}"*3}'""") + self.check_roundtrip(r"""f'{f"{y}"*3}'""") + def test_del_statement(self): self.check_roundtrip("del x, y, z") @@ -279,8 +284,18 @@ class DirectoryTestCase(ASTTestCase): for filename in names: if test.support.verbose: print('Testing %s' % filename) - source = read_pyfile(filename) - self.check_roundtrip(source) + + # Some f-strings are not correctly round-tripped by + # Tools/parser/unparse.py. See issue 28002 for details. + # We need to skip files that contain such f-strings. + if os.path.basename(filename) in ('test_fstring.py', ): + if test.support.verbose: + print(f'Skipping {filename}: see issue 28002') + continue + + with self.subTest(filename=filename): + source = read_pyfile(filename) + self.check_roundtrip(source) if __name__ == '__main__': diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index f66a3bc3e4..1d87aea5a3 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,11 +1,11 @@ import os -import io import sys from test.support import TESTFN, rmtree, unlink, captured_stdout +from test.support.script_helper import assert_python_ok, assert_python_failure import unittest import trace -from trace import CoverageResults, Trace +from trace import Trace from test.tracedmodules import testmod @@ -365,51 +365,27 @@ class Test_Ignore(unittest.TestCase): # Matched before. self.assertTrue(ignore.names(jn('bar', 'baz.py'), 'baz')) - -class TestDeprecatedMethods(unittest.TestCase): - - def test_deprecated_usage(self): - sio = io.StringIO() - with self.assertWarns(DeprecationWarning): - trace.usage(sio) - self.assertIn('Usage:', sio.getvalue()) - - def test_deprecated_Ignore(self): - with self.assertWarns(DeprecationWarning): - trace.Ignore() - - def test_deprecated_modname(self): - with self.assertWarns(DeprecationWarning): - self.assertEqual("spam", trace.modname("spam")) - - def test_deprecated_fullmodname(self): - with self.assertWarns(DeprecationWarning): - self.assertEqual("spam", trace.fullmodname("spam")) - - def test_deprecated_find_lines_from_code(self): - with self.assertWarns(DeprecationWarning): - def foo(): - pass - trace.find_lines_from_code(foo.__code__, ["eggs"]) - - def test_deprecated_find_lines(self): - with self.assertWarns(DeprecationWarning): - def foo(): - pass - trace.find_lines(foo.__code__, ["eggs"]) - - def test_deprecated_find_strings(self): - with open(TESTFN, 'w') as fd: - self.addCleanup(unlink, TESTFN) - with self.assertWarns(DeprecationWarning): - trace.find_strings(fd.name) - - def test_deprecated_find_executable_linenos(self): +class TestCommandLine(unittest.TestCase): + + def test_failures(self): + _errors = ( + (b'filename is missing: required with the main options', '-l', '-T'), + (b'cannot specify both --listfuncs and (--trace or --count)', '-lc'), + (b'argument -R/--no-report: not allowed with argument -r/--report', '-rR'), + (b'must specify one of --trace, --count, --report, --listfuncs, or --trackcalls', '-g'), + (b'-r/--report requires -f/--file', '-r'), + (b'--summary can only be used with --count or --report', '-sT'), + (b'unrecognized arguments: -y', '-y')) + for message, *args in _errors: + *_, stderr = assert_python_failure('-m', 'trace', *args) + self.assertIn(message, stderr) + + def test_listfuncs_flag_success(self): with open(TESTFN, 'w') as fd: self.addCleanup(unlink, TESTFN) - with self.assertWarns(DeprecationWarning): - trace.find_executable_linenos(fd.name) - + fd.write("a = 1\n") + status, stdout, stderr = assert_python_ok('-m', 'trace', '-l', TESTFN) + self.assertIn(b'functions called:', stdout) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 49cedda550..7276bc7ee7 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -125,12 +125,12 @@ class TracebackCases(unittest.TestCase): def do_test(firstlines, message, charset, lineno): # Raise the message in a subprocess, and catch the output try: - output = open(TESTFN, "w", encoding=charset) - output.write("""{0}if 1: - import traceback; - raise RuntimeError('{1}') - """.format(firstlines, message)) - output.close() + with open(TESTFN, "w", encoding=charset) as output: + output.write("""{0}if 1: + import traceback; + raise RuntimeError('{1}') + """.format(firstlines, message)) + process = subprocess.Popen([sys.executable, TESTFN], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout, stderr = process.communicate() @@ -171,8 +171,8 @@ class TracebackCases(unittest.TestCase): text, charset, 5) do_test(" \t\f\n# coding: {0}\n".format(charset), text, charset, 5) - # Issue #18960: coding spec should has no effect - do_test("0\n# coding: GBK\n", "h\xe9 ho", 'utf-8', 5) + # Issue #18960: coding spec should have no effect + do_test("x=0\n# coding: GBK\n", "h\xe9 ho", 'utf-8', 5) @support.requires_type_collecting def test_print_traceback_at_exit(self): @@ -299,6 +299,137 @@ class TracebackFormatTests(unittest.TestCase): ' traceback.print_stack()', ]) + # issue 26823 - Shrink recursive tracebacks + def _check_recursive_traceback_display(self, render_exc): + # Always show full diffs when this test fails + # Note that rearranging things may require adjusting + # the relative line numbers in the expected tracebacks + self.maxDiff = None + + # Check hitting the recursion limit + def f(): + f() + + with captured_output("stderr") as stderr_f: + try: + f() + except RecursionError as exc: + render_exc() + else: + self.fail("no recursion occurred") + + lineno_f = f.__code__.co_firstlineno + result_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {lineno_f+5}, in _check_recursive_traceback_display\n' + ' f()\n' + f' File "{__file__}", line {lineno_f+1}, in f\n' + ' f()\n' + f' File "{__file__}", line {lineno_f+1}, in f\n' + ' f()\n' + f' File "{__file__}", line {lineno_f+1}, in f\n' + ' f()\n' + # XXX: The following line changes depending on whether the tests + # are run through the interactive interpreter or with -m + # It also varies depending on the platform (stack size) + # Fortunately, we don't care about exactness here, so we use regex + r' \[Previous line repeated (\d+) more times\]' '\n' + 'RecursionError: maximum recursion depth exceeded\n' + ) + + expected = result_f.splitlines() + actual = stderr_f.getvalue().splitlines() + + # Check the output text matches expectations + # 2nd last line contains the repetition count + self.assertEqual(actual[:-2], expected[:-2]) + self.assertRegex(actual[-2], expected[-2]) + self.assertEqual(actual[-1], expected[-1]) + + # Check the recursion count is roughly as expected + rec_limit = sys.getrecursionlimit() + self.assertIn(int(re.search(r"\d+", actual[-2]).group()), range(rec_limit-60, rec_limit)) + + # Check a known (limited) number of recursive invocations + def g(count=10): + if count: + return g(count-1) + raise ValueError + + with captured_output("stderr") as stderr_g: + try: + g() + except ValueError as exc: + render_exc() + else: + self.fail("no value error was raised") + + lineno_g = g.__code__.co_firstlineno + result_g = ( + f' File "{__file__}", line {lineno_g+2}, in g\n' + ' return g(count-1)\n' + f' File "{__file__}", line {lineno_g+2}, in g\n' + ' return g(count-1)\n' + f' File "{__file__}", line {lineno_g+2}, in g\n' + ' return g(count-1)\n' + ' [Previous line repeated 6 more times]\n' + f' File "{__file__}", line {lineno_g+3}, in g\n' + ' raise ValueError\n' + 'ValueError\n' + ) + tb_line = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {lineno_g+7}, in _check_recursive_traceback_display\n' + ' g()\n' + ) + expected = (tb_line + result_g).splitlines() + actual = stderr_g.getvalue().splitlines() + self.assertEqual(actual, expected) + + # Check 2 different repetitive sections + def h(count=10): + if count: + return h(count-1) + g() + + with captured_output("stderr") as stderr_h: + try: + h() + except ValueError as exc: + render_exc() + else: + self.fail("no value error was raised") + + lineno_h = h.__code__.co_firstlineno + result_h = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {lineno_h+7}, in _check_recursive_traceback_display\n' + ' h()\n' + f' File "{__file__}", line {lineno_h+2}, in h\n' + ' return h(count-1)\n' + f' File "{__file__}", line {lineno_h+2}, in h\n' + ' return h(count-1)\n' + f' File "{__file__}", line {lineno_h+2}, in h\n' + ' return h(count-1)\n' + ' [Previous line repeated 6 more times]\n' + f' File "{__file__}", line {lineno_h+3}, in h\n' + ' g()\n' + ) + expected = (result_h + result_g).splitlines() + actual = stderr_h.getvalue().splitlines() + self.assertEqual(actual, expected) + + def test_recursive_traceback_python(self): + self._check_recursive_traceback_display(traceback.print_exc) + + @cpython_only + def test_recursive_traceback_cpython_internal(self): + from _testcapi import exception_print + def render_exc(): + exc_type, exc_value, exc_tb = sys.exc_info() + exception_print(exc_value) + self._check_recursive_traceback_display(render_exc) + def test_format_stack(self): def fmt(): return traceback.format_stack() diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index da89a9a871..790ab7ee60 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -11,9 +11,15 @@ try: import threading except ImportError: threading = None +try: + import _testcapi +except ImportError: + _testcapi = None + EMPTY_STRING_SIZE = sys.getsizeof(b'') + def get_frames(nframe, lineno_delta): frames = [] frame = sys._getframe(1) @@ -37,28 +43,31 @@ def allocate_bytes(size): def create_snapshots(): traceback_limit = 2 + # _tracemalloc._get_traces() returns a list of (domain, size, + # traceback_frames) tuples. traceback_frames is a tuple of (filename, + # line_number) tuples. raw_traces = [ - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), - (2, (('a.py', 5), ('b.py', 4))), + (1, 2, (('a.py', 5), ('b.py', 4))), - (66, (('b.py', 1),)), + (2, 66, (('b.py', 1),)), - (7, (('<unknown>', 0),)), + (3, 7, (('<unknown>', 0),)), ] snapshot = tracemalloc.Snapshot(raw_traces, traceback_limit) raw_traces2 = [ - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), - (2, (('a.py', 5), ('b.py', 4))), - (5000, (('a.py', 5), ('b.py', 4))), + (2, 2, (('a.py', 5), ('b.py', 4))), + (2, 5000, (('a.py', 5), ('b.py', 4))), - (400, (('c.py', 578),)), + (4, 400, (('c.py', 578),)), ] snapshot2 = tracemalloc.Snapshot(raw_traces2, traceback_limit) @@ -126,7 +135,7 @@ class TestTracemallocEnabled(unittest.TestCase): def find_trace(self, traces, traceback): for trace in traces: - if trace[1] == traceback._frames: + if trace[2] == traceback._frames: return trace self.fail("trace not found") @@ -140,7 +149,7 @@ class TestTracemallocEnabled(unittest.TestCase): trace = self.find_trace(traces, obj_traceback) self.assertIsInstance(trace, tuple) - size, traceback = trace + domain, size, traceback = trace self.assertEqual(size, obj_size) self.assertEqual(traceback, obj_traceback._frames) @@ -167,9 +176,8 @@ class TestTracemallocEnabled(unittest.TestCase): trace1 = self.find_trace(traces, obj1_traceback) trace2 = self.find_trace(traces, obj2_traceback) - size1, traceback1 = trace1 - size2, traceback2 = trace2 - self.assertEqual(traceback2, traceback1) + domain1, size1, traceback1 = trace1 + domain2, size2, traceback2 = trace2 self.assertIs(traceback2, traceback1) def test_get_traced_memory(self): @@ -292,7 +300,7 @@ class TestSnapshot(unittest.TestCase): maxDiff = 4000 def test_create_snapshot(self): - raw_traces = [(5, (('a.py', 2),))] + raw_traces = [(0, 5, (('a.py', 2),))] with contextlib.ExitStack() as stack: stack.enter_context(patch.object(tracemalloc, 'is_tracing', @@ -322,11 +330,11 @@ class TestSnapshot(unittest.TestCase): # exclude b.py snapshot3 = snapshot.filter_traces((filter1,)) self.assertEqual(snapshot3.traces._traces, [ - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (2, (('a.py', 5), ('b.py', 4))), - (7, (('<unknown>', 0),)), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (1, 2, (('a.py', 5), ('b.py', 4))), + (3, 7, (('<unknown>', 0),)), ]) # filter_traces() must not touch the original snapshot @@ -335,10 +343,10 @@ class TestSnapshot(unittest.TestCase): # only include two lines of a.py snapshot4 = snapshot3.filter_traces((filter2, filter3)) self.assertEqual(snapshot4.traces._traces, [ - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (2, (('a.py', 5), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (1, 2, (('a.py', 5), ('b.py', 4))), ]) # No filter: just duplicate the snapshot @@ -349,6 +357,54 @@ class TestSnapshot(unittest.TestCase): self.assertRaises(TypeError, snapshot.filter_traces, filter1) + def test_filter_traces_domain(self): + snapshot, snapshot2 = create_snapshots() + filter1 = tracemalloc.Filter(False, "a.py", domain=1) + filter2 = tracemalloc.Filter(True, "a.py", domain=1) + + original_traces = list(snapshot.traces._traces) + + # exclude a.py of domain 1 + snapshot3 = snapshot.filter_traces((filter1,)) + self.assertEqual(snapshot3.traces._traces, [ + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (2, 66, (('b.py', 1),)), + (3, 7, (('<unknown>', 0),)), + ]) + + # include domain 1 + snapshot3 = snapshot.filter_traces((filter1,)) + self.assertEqual(snapshot3.traces._traces, [ + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (2, 66, (('b.py', 1),)), + (3, 7, (('<unknown>', 0),)), + ]) + + def test_filter_traces_domain_filter(self): + snapshot, snapshot2 = create_snapshots() + filter1 = tracemalloc.DomainFilter(False, domain=3) + filter2 = tracemalloc.DomainFilter(True, domain=3) + + # exclude domain 2 + snapshot3 = snapshot.filter_traces((filter1,)) + self.assertEqual(snapshot3.traces._traces, [ + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (1, 2, (('a.py', 5), ('b.py', 4))), + (2, 66, (('b.py', 1),)), + ]) + + # include domain 2 + snapshot3 = snapshot.filter_traces((filter2,)) + self.assertEqual(snapshot3.traces._traces, [ + (3, 7, (('<unknown>', 0),)), + ]) + def test_snapshot_group_by_line(self): snapshot, snapshot2 = create_snapshots() tb_0 = traceback_lineno('<unknown>', 0) @@ -816,12 +872,121 @@ class TestCommandLine(unittest.TestCase): assert_python_ok('-X', 'tracemalloc', '-c', code) +@unittest.skipIf(_testcapi is None, 'need _testcapi') +class TestCAPI(unittest.TestCase): + maxDiff = 80 * 20 + + def setUp(self): + if tracemalloc.is_tracing(): + self.skipTest("tracemalloc must be stopped before the test") + + self.domain = 5 + self.size = 123 + self.obj = allocate_bytes(self.size)[0] + + # for the type "object", id(obj) is the address of its memory block. + # This type is not tracked by the garbage collector + self.ptr = id(self.obj) + + def tearDown(self): + tracemalloc.stop() + + def get_traceback(self): + frames = _testcapi.tracemalloc_get_traceback(self.domain, self.ptr) + if frames is not None: + return tracemalloc.Traceback(frames) + else: + return None + + def track(self, release_gil=False, nframe=1): + frames = get_frames(nframe, 2) + _testcapi.tracemalloc_track(self.domain, self.ptr, self.size, + release_gil) + return frames + + def untrack(self): + _testcapi.tracemalloc_untrack(self.domain, self.ptr) + + def get_traced_memory(self): + # Get the traced size in the domain + snapshot = tracemalloc.take_snapshot() + domain_filter = tracemalloc.DomainFilter(True, self.domain) + snapshot = snapshot.filter_traces([domain_filter]) + return sum(trace.size for trace in snapshot.traces) + + def check_track(self, release_gil): + nframe = 5 + tracemalloc.start(nframe) + + size = tracemalloc.get_traced_memory()[0] + + frames = self.track(release_gil, nframe) + self.assertEqual(self.get_traceback(), + tracemalloc.Traceback(frames)) + + self.assertEqual(self.get_traced_memory(), self.size) + + def test_track(self): + self.check_track(False) + + def test_track_without_gil(self): + # check that calling _PyTraceMalloc_Track() without holding the GIL + # works too + self.check_track(True) + + def test_track_already_tracked(self): + nframe = 5 + tracemalloc.start(nframe) + + # track a first time + self.track() + + # calling _PyTraceMalloc_Track() must remove the old trace and add + # a new trace with the new traceback + frames = self.track(nframe=nframe) + self.assertEqual(self.get_traceback(), + tracemalloc.Traceback(frames)) + + def test_untrack(self): + tracemalloc.start() + + self.track() + self.assertIsNotNone(self.get_traceback()) + self.assertEqual(self.get_traced_memory(), self.size) + + # untrack must remove the trace + self.untrack() + self.assertIsNone(self.get_traceback()) + self.assertEqual(self.get_traced_memory(), 0) + + # calling _PyTraceMalloc_Untrack() multiple times must not crash + self.untrack() + self.untrack() + + def test_stop_track(self): + tracemalloc.start() + tracemalloc.stop() + + with self.assertRaises(RuntimeError): + self.track() + self.assertIsNone(self.get_traceback()) + + def test_stop_untrack(self): + tracemalloc.start() + self.track() + + tracemalloc.stop() + with self.assertRaises(RuntimeError): + self.untrack() + + def test_main(): support.run_unittest( TestTracemallocEnabled, TestSnapshot, TestFilters, TestCommandLine, + TestCAPI, ) if __name__ == "__main__": diff --git a/Lib/test/test_ttk_guionly.py b/Lib/test/test_ttk_guionly.py index 490e72337f..462665db5f 100644 --- a/Lib/test/test_ttk_guionly.py +++ b/Lib/test/test_ttk_guionly.py @@ -1,4 +1,3 @@ -import os import unittest from test import support diff --git a/Lib/test/test_ttk_textonly.py b/Lib/test/test_ttk_textonly.py index 566fc9d09a..7540a43103 100644 --- a/Lib/test/test_ttk_textonly.py +++ b/Lib/test/test_ttk_textonly.py @@ -1,4 +1,3 @@ -import os from test import support # Skip this test if _tkinter does not exist. diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 5e741153f4..382ca03e5a 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -48,6 +48,7 @@ class TypesTests(unittest.TestCase): def test_float_constructor(self): self.assertRaises(ValueError, float, '') self.assertRaises(ValueError, float, '5\0') + self.assertRaises(ValueError, float, '5_5\0') def test_zero_division(self): try: 5.0 / 0.0 @@ -1000,6 +1001,24 @@ class ClassCreationTests(unittest.TestCase): with self.assertRaises(TypeError): X = types.new_class("X", (int(), C)) + def test_one_argument_type(self): + expected_message = 'type.__new__() takes exactly 3 arguments (1 given)' + + # Only type itself can use the one-argument form (#27157) + self.assertIs(type(5), int) + + class M(type): + pass + with self.assertRaises(TypeError) as cm: + M(5) + self.assertEqual(str(cm.exception), expected_message) + + class N(type, metaclass=M): + pass + with self.assertRaises(TypeError) as cm: + N(5) + self.assertEqual(str(cm.exception), expected_message) + class SimpleNamespaceTests(unittest.TestCase): diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 3136ea1a1b..86ebd45e7b 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -10,6 +10,7 @@ import codecs import itertools import operator import struct +import string import sys import unittest import warnings @@ -993,6 +994,19 @@ class UnicodeTest(string_tests.CommonTest, def __format__(self, format_spec): return int.__format__(self * 2, format_spec) + class M: + def __init__(self, x): + self.x = x + def __repr__(self): + return 'M(' + self.x + ')' + __str__ = None + + class N: + def __init__(self, x): + self.x = x + def __repr__(self): + return 'N(' + self.x + ')' + __format__ = None self.assertEqual(''.format(), '') self.assertEqual('abc'.format(), 'abc') @@ -1207,6 +1221,16 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual("0x{:0{:d}X}".format(0x0,16), "0x0000000000000000") + # Blocking fallback + m = M('data') + self.assertEqual("{!r}".format(m), 'M(data)') + self.assertRaises(TypeError, "{!s}".format, m) + self.assertRaises(TypeError, "{}".format, m) + n = N('data') + self.assertEqual("{!r}".format(n), 'N(data)') + self.assertEqual("{!s}".format(n), 'N(data)') + self.assertRaises(TypeError, "{}".format, n) + def test_format_map(self): self.assertEqual(''.format_map({}), '') self.assertEqual('a'.format_map({}), 'a') @@ -1548,7 +1572,7 @@ class UnicodeTest(string_tests.CommonTest, ('+', b'+-'), ('+-', b'+--'), ('+?', b'+-?'), - ('\?', b'+AFw?'), + (r'\?', b'+AFw?'), ('+?', b'+-?'), (r'\\?', b'+AFwAXA?'), (r'\\\?', b'+AFwAXABc?'), @@ -2427,7 +2451,7 @@ class CAPITest(unittest.TestCase): # non-ascii format, ascii argument: ensure that PyUnicode_FromFormatV() # raises an error self.assertRaisesRegex(ValueError, - '^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format ' + r'^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format ' 'string, got a non-ASCII byte: 0xe9$', PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii') @@ -2799,7 +2823,6 @@ class CAPITest(unittest.TestCase): # Check that the second call returns the same result self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) - class StringModuleTest(unittest.TestCase): def test_formatter_parser(self): def parse(format): diff --git a/Lib/test/test_pep277.py b/Lib/test/test_unicode_file_functions.py index 98c716b4c7..98c716b4c7 100644 --- a/Lib/test/test_pep277.py +++ b/Lib/test/test_unicode_file_functions.py diff --git a/Lib/test/test_pep3131.py b/Lib/test/test_unicode_identifiers.py index 0679845238..0679845238 100644 --- a/Lib/test/test_pep3131.py +++ b/Lib/test/test_unicode_identifiers.py diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 6ecc91362d..4fc11ec32f 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -20,7 +20,7 @@ errors = 'surrogatepass' class UnicodeMethodsTest(unittest.TestCase): # update this, if the database changes - expectedchecksum = '5971760872b2f98bb9c701e6c0db3273d756b3ec' + expectedchecksum = 'c1fa98674a683aa8a8d8dee0c84494f8d36346e6' def test_method_checksum(self): h = hashlib.sha1() @@ -80,7 +80,7 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest): # Update this if the database changes. Make sure to do a full rebuild # (e.g. 'make distclean && make') to get the correct checksum. - expectedchecksum = '5e74827cd07f9e546a30f34b7bcf6cc2eac38c8c' + expectedchecksum = 'f891b1e6430c712531b9bc935a38e22d78ba1bf3' def test_function_checksum(self): data = [] h = hashlib.sha1() @@ -222,6 +222,10 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest): self.assertEqual(eaw('\u2010'), 'A') self.assertEqual(eaw('\U00020000'), 'W') + def test_east_asian_width_9_0_changes(self): + self.assertEqual(self.db.ucd_3_2_0.east_asian_width('\u231a'), 'N') + self.assertEqual(self.db.east_asian_width('\u231a'), 'W') + class UnicodeMiscTest(UnicodeDatabaseTest): def test_failed_import_during_compiling(self): diff --git a/Lib/test/test_unpack.py b/Lib/test/test_unpack.py index d1ccb38937..3fcb18fb43 100644 --- a/Lib/test/test_unpack.py +++ b/Lib/test/test_unpack.py @@ -117,6 +117,27 @@ error) ... test.test_unpack.BozoError +Allow unpacking empty iterables + + >>> () = [] + >>> [] = () + >>> [] = [] + >>> () = () + +Unpacking non-iterables should raise TypeError + + >>> () = 42 + Traceback (most recent call last): + ... + TypeError: 'int' object is not iterable + +Unpacking to an empty iterable should raise ValueError + + >>> () = [42] + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 0) + """ __test__ = {'doctests' : doctests} diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index 74346b4205..6be8f551fc 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -357,7 +357,6 @@ Some size constraints (all fail.) __test__ = {'doctests' : doctests} def test_main(verbose=False): - import sys from test import support from test import test_unpack_ex support.run_doctest(test_unpack_ex, verbose) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 1772399803..5084486e5a 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -470,10 +470,11 @@ Connection: close @unittest.skipUnless(ssl, "ssl module required") def test_cafile_and_context(self): context = ssl.create_default_context() - with self.assertRaises(ValueError): - urllib.request.urlopen( - "https://localhost", cafile="/nonexistent/path", context=context - ) + with support.check_warnings(('', DeprecationWarning)): + with self.assertRaises(ValueError): + urllib.request.urlopen( + "https://localhost", cafile="/nonexistent/path", context=context + ) class urlopen_DataTests(unittest.TestCase): """Test urlopen() opening a data URL.""" @@ -730,7 +731,7 @@ FF class QuotingTests(unittest.TestCase): - """Tests for urllib.quote() and urllib.quote_plus() + r"""Tests for urllib.quote() and urllib.quote_plus() According to RFC 2396 (Uniform Resource Identifiers), to escape a character you write it as '%' + <2 character US-ASCII hex value>. @@ -805,7 +806,7 @@ class QuotingTests(unittest.TestCase): # Make sure all characters that should be quoted are by default sans # space (separate test for that). should_quote = [chr(num) for num in range(32)] # For 0x00 - 0x1F - should_quote.append('<>#%"{}|\^[]`') + should_quote.append(r'<>#%"{}|\^[]`') should_quote.append(chr(127)) # For 0x7F should_quote = ''.join(should_quote) for char in should_quote: diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index eda7cccc60..34329f8716 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -7,6 +7,8 @@ import io import socket import array import sys +import tempfile +import subprocess import urllib.request # The proxy bypass method imported below has logic specific to the OSX @@ -335,7 +337,8 @@ class MockHTTPClass: else: self._tunnel_headers.clear() - def request(self, method, url, body=None, headers=None): + def request(self, method, url, body=None, headers=None, *, + encode_chunked=False): self.method = method self.selector = url if headers is not None: @@ -343,6 +346,7 @@ class MockHTTPClass: self.req_headers.sort() if body: self.data = body + self.encode_chunked = encode_chunked if self.raise_on_endheaders: raise OSError() @@ -908,41 +912,110 @@ class HandlerTests(unittest.TestCase): self.assertEqual(req.unredirected_hdrs["Host"], "baz") self.assertEqual(req.unredirected_hdrs["Spam"], "foo") - # Check iterable body support - def iterable_body(): - yield b"one" - yield b"two" - yield b"three" + def test_http_body_file(self): + # A regular file - chunked encoding is used unless Content Length is + # already set. - for headers in {}, {"Content-Length": 11}: - req = Request("http://example.com/", iterable_body(), headers) - if not headers: - # Having an iterable body without a Content-Length should - # raise an exception - self.assertRaises(ValueError, h.do_request_, req) - else: - newreq = h.do_request_(req) + h = urllib.request.AbstractHTTPHandler() + o = h.parent = MockOpener() - # A file object. - # Test only Content-Length attribute of request. + file_obj = tempfile.NamedTemporaryFile(mode='w+b', delete=False) + file_path = file_obj.name + file_obj.close() + self.addCleanup(os.unlink, file_path) + + with open(file_path, "rb") as f: + req = Request("http://example.com/", f, {}) + newreq = h.do_request_(req) + te = newreq.get_header('Transfer-encoding') + self.assertEqual(te, "chunked") + self.assertFalse(newreq.has_header('Content-length')) + with open(file_path, "rb") as f: + req = Request("http://example.com/", f, {"Content-Length": 30}) + newreq = h.do_request_(req) + self.assertEqual(int(newreq.get_header('Content-length')), 30) + self.assertFalse(newreq.has_header("Transfer-encoding")) + + def test_http_body_fileobj(self): + # A file object - chunked encoding is used + # unless Content Length is already set. + # (Note that there are some subtle differences to a regular + # file, that is why we are testing both cases.) + + h = urllib.request.AbstractHTTPHandler() + o = h.parent = MockOpener() file_obj = io.BytesIO() - file_obj.write(b"Something\nSomething\nSomething\n") + req = Request("http://example.com/", file_obj, {}) + newreq = h.do_request_(req) + self.assertEqual(newreq.get_header('Transfer-encoding'), 'chunked') + self.assertFalse(newreq.has_header('Content-length')) + + headers = {"Content-Length": 30} + req = Request("http://example.com/", file_obj, headers) + newreq = h.do_request_(req) + self.assertEqual(int(newreq.get_header('Content-length')), 30) + self.assertFalse(newreq.has_header("Transfer-encoding")) + + file_obj.close() + + def test_http_body_pipe(self): + # A file reading from a pipe. + # A pipe cannot be seek'ed. There is no way to determine the + # content length up front. Thus, do_request_() should fall + # back to Transfer-encoding chunked. + + h = urllib.request.AbstractHTTPHandler() + o = h.parent = MockOpener() + + cmd = [sys.executable, "-c", r"pass"] for headers in {}, {"Content-Length": 30}: - req = Request("http://example.com/", file_obj, headers) + with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc: + req = Request("http://example.com/", proc.stdout, headers) + newreq = h.do_request_(req) + if not headers: + self.assertEqual(newreq.get_header('Content-length'), None) + self.assertEqual(newreq.get_header('Transfer-encoding'), + 'chunked') + else: + self.assertEqual(int(newreq.get_header('Content-length')), + 30) + + def test_http_body_iterable(self): + # Generic iterable. There is no way to determine the content + # length up front. Fall back to Transfer-encoding chunked. + + h = urllib.request.AbstractHTTPHandler() + o = h.parent = MockOpener() + + def iterable_body(): + yield b"one" + + for headers in {}, {"Content-Length": 11}: + req = Request("http://example.com/", iterable_body(), headers) + newreq = h.do_request_(req) if not headers: - # Having an iterable body without a Content-Length should - # raise an exception - self.assertRaises(ValueError, h.do_request_, req) + self.assertEqual(newreq.get_header('Content-length'), None) + self.assertEqual(newreq.get_header('Transfer-encoding'), + 'chunked') else: - newreq = h.do_request_(req) - self.assertEqual(int(newreq.get_header('Content-length')), 30) + self.assertEqual(int(newreq.get_header('Content-length')), 11) - file_obj.close() + def test_http_body_empty_seq(self): + # Zero-length iterable body should be treated like any other iterable + h = urllib.request.AbstractHTTPHandler() + h.parent = MockOpener() + req = h.do_request_(Request("http://example.com/", ())) + self.assertEqual(req.get_header("Transfer-encoding"), "chunked") + self.assertFalse(req.has_header("Content-length")) + def test_http_body_array(self): # array.array Iterable - Content Length is calculated + h = urllib.request.AbstractHTTPHandler() + o = h.parent = MockOpener() + iterable_array = array.array("I",[1,2,3,4]) for headers in {}, {"Content-Length": 16}: diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index 68c523abbc..70c4c013e9 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -557,26 +557,28 @@ class TestUrlopen(unittest.TestCase): def test_https_with_cafile(self): handler = self.start_https_server(certfile=CERT_localhost) - # Good cert - data = self.urlopen("https://localhost:%s/bizarre" % handler.port, - cafile=CERT_localhost) - self.assertEqual(data, b"we care a bit") - # Bad cert - with self.assertRaises(urllib.error.URLError) as cm: - self.urlopen("https://localhost:%s/bizarre" % handler.port, - cafile=CERT_fakehostname) - # Good cert, but mismatching hostname - handler = self.start_https_server(certfile=CERT_fakehostname) - with self.assertRaises(ssl.CertificateError) as cm: - self.urlopen("https://localhost:%s/bizarre" % handler.port, - cafile=CERT_fakehostname) + with support.check_warnings(('', DeprecationWarning)): + # Good cert + data = self.urlopen("https://localhost:%s/bizarre" % handler.port, + cafile=CERT_localhost) + self.assertEqual(data, b"we care a bit") + # Bad cert + with self.assertRaises(urllib.error.URLError) as cm: + self.urlopen("https://localhost:%s/bizarre" % handler.port, + cafile=CERT_fakehostname) + # Good cert, but mismatching hostname + handler = self.start_https_server(certfile=CERT_fakehostname) + with self.assertRaises(ssl.CertificateError) as cm: + self.urlopen("https://localhost:%s/bizarre" % handler.port, + cafile=CERT_fakehostname) def test_https_with_cadefault(self): handler = self.start_https_server(certfile=CERT_localhost) # Self-signed cert should fail verification with system certificate store - with self.assertRaises(urllib.error.URLError) as cm: - self.urlopen("https://localhost:%s/bizarre" % handler.port, - cadefault=True) + with support.check_warnings(('', DeprecationWarning)): + with self.assertRaises(urllib.error.URLError) as cm: + self.urlopen("https://localhost:%s/bizarre" % handler.port, + cadefault=True) def test_https_sni(self): if ssl is None: diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index b811930bbc..949716c2b5 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -4,7 +4,6 @@ from test import support import contextlib import socket import urllib.request -import sys import os import email.message import time diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 71abc147a6..99c5c033e3 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -607,29 +607,27 @@ class UrlParseTestCase(unittest.TestCase): self.assertEqual(p.port, 80) self.assertEqual(p.geturl(), url) - # Verify an illegal port is returned as None + # Verify an illegal port raises ValueError url = b"HTTP://WWW.PYTHON.ORG:65536/doc/#frag" p = urllib.parse.urlsplit(url) - self.assertEqual(p.port, None) + with self.assertRaisesRegex(ValueError, "out of range"): + p.port def test_attributes_bad_port(self): - """Check handling of non-integer ports.""" - p = urllib.parse.urlsplit("http://www.example.net:foo") - self.assertEqual(p.netloc, "www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) - - p = urllib.parse.urlparse("http://www.example.net:foo") - self.assertEqual(p.netloc, "www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) - - # Once again, repeat ourselves to test bytes - p = urllib.parse.urlsplit(b"http://www.example.net:foo") - self.assertEqual(p.netloc, b"www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) - - p = urllib.parse.urlparse(b"http://www.example.net:foo") - self.assertEqual(p.netloc, b"www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) + """Check handling of invalid ports.""" + for bytes in (False, True): + for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): + for port in ("foo", "1.5", "-1", "0x10"): + with self.subTest(bytes=bytes, parse=parse, port=port): + netloc = "www.example.net:" + port + url = "http://" + netloc + if bytes: + netloc = netloc.encode("ascii") + url = url.encode("ascii") + p = parse(url) + self.assertEqual(p.netloc, netloc) + with self.assertRaises(ValueError): + p.port def test_attributes_without_netloc(self): # This example is straight from RFC 3261. It looks like it diff --git a/Lib/test/test_userdict.py b/Lib/test/test_userdict.py index 8357f8bcd1..662c7f641a 100644 --- a/Lib/test/test_userdict.py +++ b/Lib/test/test_userdict.py @@ -1,6 +1,6 @@ # Check every path through every method of UserDict -from test import support, mapping_tests +from test import mapping_tests import unittest import collections @@ -30,7 +30,7 @@ class UserDictTest(mapping_tests.TestHashMappingProtocol): self.assertEqual(collections.UserDict(one=1, two=2), d2) # item sequence constructor self.assertEqual(collections.UserDict([('one',1), ('two',2)]), d2) - with self.assertWarnsRegex(PendingDeprecationWarning, "'dict'"): + with self.assertWarnsRegex(DeprecationWarning, "'dict'"): self.assertEqual(collections.UserDict(dict=[('one',1), ('two',2)]), d2) # both together self.assertEqual(collections.UserDict([('one',1), ('two',2)], two=3, three=5), d3) @@ -149,7 +149,7 @@ class UserDictTest(mapping_tests.TestHashMappingProtocol): [('dict', 42)]) self.assertEqual(list(collections.UserDict({}, dict=None).items()), [('dict', None)]) - with self.assertWarnsRegex(PendingDeprecationWarning, "'dict'"): + with self.assertWarnsRegex(DeprecationWarning, "'dict'"): self.assertEqual(list(collections.UserDict(dict={'a': 42}).items()), [('a', 42)]) self.assertRaises(TypeError, collections.UserDict, 42) diff --git a/Lib/test/test_userlist.py b/Lib/test/test_userlist.py index f92e4d385e..8de6c14e39 100644 --- a/Lib/test/test_userlist.py +++ b/Lib/test/test_userlist.py @@ -1,7 +1,7 @@ # Check every path through every method of UserList from collections import UserList -from test import support, list_tests +from test import list_tests import unittest class UserListTest(list_tests.CommonTest): diff --git a/Lib/test/test_userstring.py b/Lib/test/test_userstring.py index 9bc8edd99d..71528223d3 100644 --- a/Lib/test/test_userstring.py +++ b/Lib/test/test_userstring.py @@ -1,9 +1,8 @@ # UserString is a wrapper around the native builtin string type. # UserString instances should behave similar to builtin string objects. -import string import unittest -from test import support, string_tests +from test import string_tests from collections import UserString diff --git a/Lib/test/test_pep3120.py b/Lib/test/test_utf8source.py index 97dced8a62..97dced8a62 100644 --- a/Lib/test/test_pep3120.py +++ b/Lib/test/test_utf8source.py diff --git a/Lib/test/test_uu.py b/Lib/test/test_uu.py index 25fffbf993..ad2f2c59c1 100644 --- a/Lib/test/test_uu.py +++ b/Lib/test/test_uu.py @@ -8,7 +8,6 @@ from test import support import sys, os import uu -from io import BytesIO import io plaintext = b"The smooth-scaled python crept over the sleeping dog\n" diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index c1e656613b..2691632ff4 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -15,16 +15,9 @@ import sys import tempfile from test.support import (captured_stdout, captured_stderr, can_symlink, EnvironmentVarGuard, rmtree) -import textwrap import unittest import venv -# pip currently requires ssl support, so we ensure we handle -# it being missing (http://bugs.python.org/issue19744) -try: - import ssl -except ImportError: - ssl = None try: import threading @@ -39,15 +32,9 @@ except ImportError: skipInVenv = unittest.skipIf(sys.prefix != sys.base_prefix, 'Test not appropriate in a venv') -# os.path.exists('nul') is False: http://bugs.python.org/issue20541 -if os.devnull.lower() == 'nul': - failsOnWindows = unittest.expectedFailure -else: - def failsOnWindows(f): - return f - class BaseTest(unittest.TestCase): """Base class for venv tests.""" + maxDiff = 80 * 50 def setUp(self): self.env_dir = os.path.realpath(tempfile.mkdtemp()) @@ -57,7 +44,7 @@ class BaseTest(unittest.TestCase): self.include = 'Include' else: self.bindir = 'bin' - self.lib = ('lib', 'python%s' % sys.version[:3]) + self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) self.include = 'include' if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in os.environ: executable = os.environ['__PYVENV_LAUNCHER__'] @@ -121,6 +108,17 @@ class BasicTest(BaseTest): print(' %r' % os.listdir(bd)) self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn) + def test_prompt(self): + env_name = os.path.split(self.env_dir)[1] + + builder = venv.EnvBuilder() + context = builder.ensure_directories(self.env_dir) + self.assertEqual(context.prompt, '(%s) ' % env_name) + + builder = venv.EnvBuilder(prompt='My prompt') + context = builder.ensure_directories(self.env_dir) + self.assertEqual(context.prompt, '(My prompt) ') + @skipInVenv def test_prefixes(self): """ @@ -318,16 +316,20 @@ class EnsurePipTest(BaseTest): self.run_with_capture(venv.create, self.env_dir, with_pip=False) self.assert_pip_not_installed() - @failsOnWindows - def test_devnull_exists_and_is_empty(self): + def test_devnull(self): # Fix for issue #20053 uses os.devnull to force a config file to # appear empty. However http://bugs.python.org/issue20541 means # that doesn't currently work properly on Windows. Once that is # fixed, the "win_location" part of test_with_pip should be restored - self.assertTrue(os.path.exists(os.devnull)) with open(os.devnull, "rb") as f: self.assertEqual(f.read(), b"") + # Issue #20541: os.path.exists('nul') is False on Windows + if os.devnull.lower() == 'nul': + self.assertFalse(os.path.exists(os.devnull)) + else: + self.assertTrue(os.path.exists(os.devnull)) + def do_test_with_pip(self, system_site_packages): rmtree(self.env_dir) with EnvironmentVarGuard() as envvars: @@ -418,8 +420,6 @@ class EnsurePipTest(BaseTest): if not system_site_packages: self.assert_pip_not_installed() - # Requesting pip fails without SSL (http://bugs.python.org/issue19744) - @unittest.skipIf(ssl is None, ensurepip._MISSING_SSL_MESSAGE) @unittest.skipUnless(threading, 'some dependencies of pip import threading' ' module unconditionally') # Issue #26610: pip/pep425tags.py requires ctypes diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 84a6fb5842..0cddf4a2f4 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -2,7 +2,9 @@ from contextlib import contextmanager import linecache import os from io import StringIO +import re import sys +import textwrap import unittest from test import support from test.support.script_helper import assert_python_ok, assert_python_failure @@ -720,6 +722,17 @@ class _WarningsTests(BaseTest, unittest.TestCase): result = stream.getvalue() self.assertIn(text, result) + def test_showwarnmsg_missing(self): + # Test that _showwarnmsg() missing is okay. + text = 'del _showwarnmsg test' + with original_warnings.catch_warnings(module=self.module): + self.module.filterwarnings("always", category=UserWarning) + del self.module._showwarnmsg + with support.captured_output('stderr') as stream: + self.module.warn(text) + result = stream.getvalue() + self.assertIn(text, result) + def test_showwarning_not_callable(self): with original_warnings.catch_warnings(module=self.module): self.module.filterwarnings("always", category=UserWarning) @@ -821,12 +834,44 @@ class WarningsDisplayTests(BaseTest): file_object, expected_file_line) self.assertEqual(expect, file_object.getvalue()) + class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = c_warnings class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = py_warnings + def test_tracemalloc(self): + self.addCleanup(support.unlink, support.TESTFN) + + with open(support.TESTFN, 'w') as fp: + fp.write(textwrap.dedent(""" + def func(): + f = open(__file__) + # Emit ResourceWarning + f = None + + func() + """)) + + res = assert_python_ok('-Wd', '-X', 'tracemalloc=2', support.TESTFN) + + stderr = res.err.decode('ascii', 'replace') + # normalize newlines + stderr = '\n'.join(stderr.splitlines()) + stderr = re.sub('<.*>', '<...>', stderr) + expected = textwrap.dedent(''' + {fname}:5: ResourceWarning: unclosed file <...> + f = None + Object allocated at (most recent call first): + File "{fname}", lineno 3 + f = open(__file__) + File "{fname}", lineno 7 + func() + ''') + expected = expected.format(fname=support.TESTFN).strip() + self.assertEqual(stderr, expected) + class CatchWarningTests(BaseTest): @@ -899,6 +944,51 @@ class CatchWarningTests(BaseTest): self.assertTrue(wmod.filters is not orig_filters) self.assertTrue(wmod.filters is orig_filters) + def test_record_override_showwarning_before(self): + # Issue #28835: If warnings.showwarning() was overriden, make sure + # that catch_warnings(record=True) overrides it again. + text = "This is a warning" + wmod = self.module + my_log = [] + + def my_logger(message, category, filename, lineno, file=None, line=None): + nonlocal my_log + my_log.append(message) + + # Override warnings.showwarning() before calling catch_warnings() + with support.swap_attr(wmod, 'showwarning', my_logger): + with wmod.catch_warnings(module=wmod, record=True) as log: + self.assertIsNot(wmod.showwarning, my_logger) + + wmod.simplefilter("always") + wmod.warn(text) + + self.assertIs(wmod.showwarning, my_logger) + + self.assertEqual(len(log), 1, log) + self.assertEqual(log[0].message.args[0], text) + self.assertEqual(my_log, []) + + def test_record_override_showwarning_inside(self): + # Issue #28835: It is possible to override warnings.showwarning() + # in the catch_warnings(record=True) context manager. + text = "This is a warning" + wmod = self.module + my_log = [] + + def my_logger(message, category, filename, lineno, file=None, line=None): + nonlocal my_log + my_log.append(message) + + with wmod.catch_warnings(module=wmod, record=True) as log: + wmod.simplefilter("always") + wmod.showwarning = my_logger + wmod.warn(text) + + self.assertEqual(len(my_log), 1, my_log) + self.assertEqual(my_log[0].args[0], text) + self.assertEqual(log, []) + def test_check_warnings(self): # Explicit tests for the test.support convenience wrapper wmod = self.module diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index 3eff773bca..8666f7269c 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -1,6 +1,6 @@ -from test.support import TESTFN import unittest from test import audiotests +from test import support from audioop import byteswap import sys import wave @@ -103,5 +103,11 @@ class WavePCM32Test(WaveTest, unittest.TestCase): frames = byteswap(frames, 4) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'WAVE_FORMAT_PCM'} + support.check__all__(self, wave, blacklist=blacklist) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 1aa354019d..43cf2c0fc2 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -943,7 +943,7 @@ class SubclassableWeakrefTestCase(TestBase): self.assertFalse(hasattr(r, "__dict__")) def test_subclass_refs_with_cycle(self): - # Bug #3110 + """Confirm https://bugs.python.org/issue3100 is fixed.""" # An instance of a weakref subclass can have attributes. # If such a weakref holds the only strong reference to the object, # deleting the weakref will delete the object. In this case, @@ -1357,13 +1357,16 @@ class MappingTestCase(TestBase): o = Object(123456) with testcontext(): n = len(dict) - dict.popitem() + # Since underlaying dict is ordered, first item is popped + dict.pop(next(dict.keys())) self.assertEqual(len(dict), n - 1) dict[o] = o self.assertEqual(len(dict), n) + # last item in objects is removed from dict in context shutdown with testcontext(): self.assertEqual(len(dict), n - 1) - dict.pop(next(dict.keys())) + # Then, (o, o) is popped + dict.popitem() self.assertEqual(len(dict), n - 2) with testcontext(): self.assertEqual(len(dict), n - 3) diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index 9ce672b815..691b95e77c 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -1,13 +1,6 @@ import unittest -from weakref import proxy, ref, WeakSet -import operator -import copy +from weakref import WeakSet import string -import os -from random import randrange, shuffle -import sys -import warnings -import collections from collections import UserString as ustr import gc import contextlib diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py new file mode 100644 index 0000000000..b1a2f7a302 --- /dev/null +++ b/Lib/test/test_winconsoleio.py @@ -0,0 +1,147 @@ +'''Tests for WindowsConsoleIO +''' + +import io +import unittest +import sys + +if sys.platform != 'win32': + raise unittest.SkipTest("test only relevant on win32") + +from _testconsole import write_input + +ConIO = io._WindowsConsoleIO + +class WindowsConsoleIOTests(unittest.TestCase): + def test_abc(self): + self.assertTrue(issubclass(ConIO, io.RawIOBase)) + self.assertFalse(issubclass(ConIO, io.BufferedIOBase)) + self.assertFalse(issubclass(ConIO, io.TextIOBase)) + + def test_open_fd(self): + try: + f = ConIO(0) + except ValueError: + # cannot open console because it's not a real console + pass + else: + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertEqual(0, f.fileno()) + f.close() # multiple close should not crash + f.close() + + try: + f = ConIO(1, 'w') + except ValueError: + # cannot open console because it's not a real console + pass + else: + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertEqual(1, f.fileno()) + f.close() + f.close() + + try: + f = ConIO(2, 'w') + except ValueError: + # cannot open console because it's not a real console + pass + else: + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertEqual(2, f.fileno()) + f.close() + f.close() + + def test_open_name(self): + f = ConIO("CON") + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertIsNotNone(f.fileno()) + f.close() # multiple close should not crash + f.close() + + f = ConIO('CONIN$') + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertIsNotNone(f.fileno()) + f.close() + f.close() + + f = ConIO('CONOUT$', 'w') + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertIsNotNone(f.fileno()) + f.close() + f.close() + + def assertStdinRoundTrip(self, text): + stdin = open('CONIN$', 'r') + old_stdin = sys.stdin + try: + sys.stdin = stdin + write_input( + stdin.buffer.raw, + (text + '\r\n').encode('utf-16-le', 'surrogatepass') + ) + actual = input() + finally: + sys.stdin = old_stdin + self.assertEqual(actual, text) + + def test_input(self): + # ASCII + self.assertStdinRoundTrip('abc123') + # Non-ASCII + self.assertStdinRoundTrip('ϼўТλФЙ') + # Combining characters + self.assertStdinRoundTrip('A͏B ﬖ̳AA̝') + # Non-BMP + self.assertStdinRoundTrip('\U00100000\U0010ffff\U0010fffd') + + def test_partial_reads(self): + # Test that reading less than 1 full character works when stdin + # contains multibyte UTF-8 sequences + source = 'ϼўТλФЙ\r\n'.encode('utf-16-le') + expected = 'ϼўТλФЙ\r\n'.encode('utf-8') + for read_count in range(1, 16): + with open('CONIN$', 'rb', buffering=0) as stdin: + write_input(stdin, source) + + actual = b'' + while not actual.endswith(b'\n'): + b = stdin.read(read_count) + actual += b + + self.assertEqual(actual, expected, 'stdin.read({})'.format(read_count)) + + def test_partial_surrogate_reads(self): + # Test that reading less than 1 full character works when stdin + # contains surrogate pairs that cannot be decoded to UTF-8 without + # reading an extra character. + source = '\U00101FFF\U00101001\r\n'.encode('utf-16-le') + expected = '\U00101FFF\U00101001\r\n'.encode('utf-8') + for read_count in range(1, 16): + with open('CONIN$', 'rb', buffering=0) as stdin: + write_input(stdin, source) + + actual = b'' + while not actual.endswith(b'\n'): + b = stdin.read(read_count) + actual += b + + self.assertEqual(actual, expected, 'stdin.read({})'.format(read_count)) + + def test_ctrl_z(self): + with open('CONIN$', 'rb', buffering=0) as stdin: + source = '\xC4\x1A\r\n'.encode('utf-16-le') + expected = '\xC4'.encode('utf-8') + write_input(stdin, source) + a, b = stdin.read(1), stdin.readall() + self.assertEqual(expected[0:1], a) + self.assertEqual(expected[1:], b) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index 60207fbb41..2be61ae15d 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -37,6 +37,7 @@ test_reflect_key_name = "SOFTWARE\\Classes\\" + test_key_base test_data = [ ("Int Value", 45, REG_DWORD), + ("Qword Value", 0x1122334455667788, REG_QWORD), ("String Val", "A string value", REG_SZ), ("StringExpand", "The path is %path%", REG_EXPAND_SZ), ("Multi-string", ["Lots", "of", "string", "values"], REG_MULTI_SZ), @@ -56,7 +57,7 @@ class BaseWinregTests(unittest.TestCase): def delete_tree(self, root, subkey): try: - hkey = OpenKey(root, subkey, KEY_ALL_ACCESS) + hkey = OpenKey(root, subkey, 0, KEY_ALL_ACCESS) except OSError: # subkey does not exist return @@ -367,6 +368,18 @@ class LocalWinregTests(BaseWinregTests): finally: DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_read_string_containing_null(self): + # Test for issue 25778: REG_SZ should not contain null characters + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + self.assertNotEqual(ck.handle, 0) + test_val = "A string\x00 with a null" + SetValueEx(ck, "test_name", 0, REG_SZ, test_val) + ret_val, ret_type = QueryValueEx(ck, "test_name") + self.assertEqual(ret_type, REG_SZ) + self.assertEqual(ret_val, "A string") + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) @unittest.skipUnless(REMOTE_NAME, "Skipping remote registry tests") diff --git a/Lib/test/test_winsound.py b/Lib/test/test_winsound.py index 4a8ab7de58..179e069a1c 100644 --- a/Lib/test/test_winsound.py +++ b/Lib/test/test_winsound.py @@ -51,6 +51,10 @@ class BeepTest(unittest.TestCase): for i in range(100, 2000, 100): safe_Beep(i, 75) + def test_keyword_args(self): + safe_Beep(duration=75, frequency=2000) + + class MessageBeepTest(unittest.TestCase): def tearDown(self): @@ -76,6 +80,9 @@ class MessageBeepTest(unittest.TestCase): def test_question(self): safe_MessageBeep(winsound.MB_ICONQUESTION) + def test_keyword_args(self): + safe_MessageBeep(type=winsound.MB_OK) + class PlaySoundTest(unittest.TestCase): @@ -87,6 +94,25 @@ class PlaySoundTest(unittest.TestCase): winsound.PlaySound, "none", winsound.SND_ASYNC | winsound.SND_MEMORY ) + self.assertRaises(TypeError, winsound.PlaySound, b"bad", 0) + self.assertRaises(TypeError, winsound.PlaySound, "bad", + winsound.SND_MEMORY) + self.assertRaises(TypeError, winsound.PlaySound, 1, 0) + + def test_keyword_args(self): + safe_PlaySound(flags=winsound.SND_ALIAS, sound="SystemExit") + + def test_snd_memory(self): + with open(support.findfile('pluck-pcm8.wav', + subdir='audiodata'), 'rb') as f: + audio_data = f.read() + safe_PlaySound(audio_data, winsound.SND_MEMORY) + audio_data = bytearray(audio_data) + safe_PlaySound(audio_data, winsound.SND_MEMORY) + + def test_snd_filename(self): + fn = support.findfile('pluck-pcm8.wav', subdir='audiodata') + safe_PlaySound(fn, winsound.SND_FILENAME | winsound.SND_NODEFAULT) def test_aliases(self): aliases = [ diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index e8d789bc2c..c70f6859b4 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -109,7 +109,7 @@ class FailureTestCase(unittest.TestCase): with foo: pass self.assertRaises(NameError, fooNotDeclared) - def testEnterAttributeError(self): + def testEnterAttributeError1(self): class LacksEnter(object): def __exit__(self, type, value, traceback): pass @@ -117,7 +117,16 @@ class FailureTestCase(unittest.TestCase): def fooLacksEnter(): foo = LacksEnter() with foo: pass - self.assertRaises(AttributeError, fooLacksEnter) + self.assertRaisesRegex(AttributeError, '__enter__', fooLacksEnter) + + def testEnterAttributeError2(self): + class LacksEnterAndExit(object): + pass + + def fooLacksEnterAndExit(): + foo = LacksEnterAndExit() + with foo: pass + self.assertRaisesRegex(AttributeError, '__enter__', fooLacksEnterAndExit) def testExitAttributeError(self): class LacksExit(object): @@ -127,7 +136,7 @@ class FailureTestCase(unittest.TestCase): def fooLacksExit(): foo = LacksExit() with foo: pass - self.assertRaises(AttributeError, fooLacksExit) + self.assertRaisesRegex(AttributeError, '__exit__', fooLacksExit) def assertRaisesSyntaxError(self, codestr): def shouldRaiseSyntaxError(s): @@ -140,11 +149,6 @@ class FailureTestCase(unittest.TestCase): 'with mock as (None):\n' ' pass') - def testAssignmentToEmptyTupleError(self): - self.assertRaisesSyntaxError( - 'with mock as ():\n' - ' pass') - def testAssignmentToTupleOnlyContainingNoneError(self): self.assertRaisesSyntaxError('with mock as None,:\n pass') self.assertRaisesSyntaxError( @@ -454,7 +458,7 @@ class ExceptionalTestCase(ContextmanagerAssertionMixin, unittest.TestCase): with cm(): raise StopIteration("from with") - with self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): self.assertRaises(StopIteration, shouldThrow) def testRaisedStopIteration2(self): @@ -482,7 +486,7 @@ class ExceptionalTestCase(ContextmanagerAssertionMixin, unittest.TestCase): with cm(): raise next(iter([])) - with self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): self.assertRaises(StopIteration, shouldThrow) def testRaisedGeneratorExit1(self): diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index 61a750c622..7708e20684 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -258,7 +258,7 @@ class IntegrationTests(TestCase): def app(environ, start_response): start_response("200 OK", []) - return [bytes(support.SOCK_MAX_SIZE)] + return [b'\0' * support.SOCK_MAX_SIZE] class WsgiHandler(NoLogRequestHandler, WSGIRequestHandler): pass diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 6c7616beae..c0144d1cb8 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -91,8 +91,6 @@ ENTITY_XML = """\ class ModuleTest(unittest.TestCase): - # TODO: this should be removed once we get rid of the global module vars - def test_sanity(self): # Import sanity. @@ -100,6 +98,10 @@ class ModuleTest(unittest.TestCase): from xml.etree import ElementInclude from xml.etree import ElementPath + def test_all(self): + names = ("xml.etree.ElementTree", "_elementtree") + support.check__all__(self, ET, names, blacklist=("HTML_EMPTY",)) + def serialize(elem, to_string=True, encoding='unicode', **options): if encoding != 'unicode': @@ -182,10 +184,12 @@ class ElementTreeTest(unittest.TestCase): def check_element(element): self.assertTrue(ET.iselement(element), msg="not an element") - self.assertTrue(hasattr(element, "tag"), msg="no tag member") - self.assertTrue(hasattr(element, "attrib"), msg="no attrib member") - self.assertTrue(hasattr(element, "text"), msg="no text member") - self.assertTrue(hasattr(element, "tail"), msg="no tail member") + direlem = dir(element) + for attr in 'tag', 'attrib', 'text', 'tail': + self.assertTrue(hasattr(element, attr), + msg='no %s member' % attr) + self.assertIn(attr, direlem, + msg='no %s visible by dir' % attr) check_string(element.tag) check_mapping(element.attrib) diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index bfced1225e..7c60699f91 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -1,5 +1,5 @@ # xml.etree test for cElementTree -import sys, struct +import struct from test import support from test.support import import_fresh_module import types @@ -118,7 +118,7 @@ class SizeofTest(unittest.TestCase): struct.calcsize('8P')) def test_main(): - from test import test_xml_etree, test_xml_etree_c + from test import test_xml_etree # Run the tests specific to the C implementation support.run_unittest( diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 02d9f5c650..df9c79e3df 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -1,5 +1,6 @@ import base64 import datetime +import decimal import sys import time import unittest @@ -9,7 +10,6 @@ import xmlrpc.server import http.client import http, http.server import socket -import os import re import io import contextlib @@ -238,6 +238,54 @@ class XMLRPCTestCase(unittest.TestCase): '</struct></value></param></params>') self.assertRaises(ResponseError, xmlrpclib.loads, data) + def check_loads(self, s, value, **kwargs): + dump = '<params><param><value>%s</value></param></params>' % s + result, m = xmlrpclib.loads(dump, **kwargs) + (newvalue,) = result + self.assertEqual(newvalue, value) + self.assertIs(type(newvalue), type(value)) + self.assertIsNone(m) + + def test_load_standard_types(self): + check = self.check_loads + check('string', 'string') + check('<string>string</string>', 'string') + check('<string>𝔘𝔫𝔦𝔠𝔬𝔡𝔢 string</string>', '𝔘𝔫𝔦𝔠𝔬𝔡𝔢 string') + check('<int>2056183947</int>', 2056183947) + check('<int>-2056183947</int>', -2056183947) + check('<i4>2056183947</i4>', 2056183947) + check('<double>46093.78125</double>', 46093.78125) + check('<boolean>0</boolean>', False) + check('<base64>AGJ5dGUgc3RyaW5n/w==</base64>', + xmlrpclib.Binary(b'\x00byte string\xff')) + check('<base64>AGJ5dGUgc3RyaW5n/w==</base64>', + b'\x00byte string\xff', use_builtin_types=True) + check('<dateTime.iso8601>20050210T11:41:23</dateTime.iso8601>', + xmlrpclib.DateTime('20050210T11:41:23')) + check('<dateTime.iso8601>20050210T11:41:23</dateTime.iso8601>', + datetime.datetime(2005, 2, 10, 11, 41, 23), + use_builtin_types=True) + check('<array><data>' + '<value><int>1</int></value><value><int>2</int></value>' + '</data></array>', [1, 2]) + check('<struct>' + '<member><name>b</name><value><int>2</int></value></member>' + '<member><name>a</name><value><int>1</int></value></member>' + '</struct>', {'a': 1, 'b': 2}) + + def test_load_extension_types(self): + check = self.check_loads + check('<nil/>', None) + check('<ex:nil/>', None) + check('<i1>205</i1>', 205) + check('<i2>20561</i2>', 20561) + check('<i8>9876543210</i8>', 9876543210) + check('<biginteger>98765432100123456789</biginteger>', + 98765432100123456789) + check('<float>93.78125</float>', 93.78125) + check('<bigdecimal>9876543210.0123456789</bigdecimal>', + decimal.Decimal('9876543210.0123456789')) + def test_get_host_info(self): # see bug #3613, this raised a TypeError transp = xmlrpc.client.Transport() @@ -1147,7 +1195,6 @@ def captured_stdout(encoding='utf-8'): """A variation on support.captured_stdout() which gives a text stream having a `buffer` attribute. """ - import io orig_stdout = sys.stdout sys.stdout = io.TextIOWrapper(io.BytesIO(), encoding=encoding) try: @@ -1220,7 +1267,7 @@ class CGIHandlerTestCase(unittest.TestCase): content = handle[handle.find("<?xml"):] self.assertEqual( - int(re.search('Content-Length: (\d+)', handle).group(1)), + int(re.search(r'Content-Length: (\d+)', handle).group(1)), len(content)) diff --git a/Lib/test/test_xmlrpc_net.py b/Lib/test/test_xmlrpc_net.py index b60b82f3e2..ae0a28e721 100644 --- a/Lib/test/test_xmlrpc_net.py +++ b/Lib/test/test_xmlrpc_net.py @@ -1,7 +1,4 @@ import collections.abc -import errno -import socket -import sys import unittest from test import support diff --git a/Lib/test/test_pep380.py b/Lib/test/test_yield_from.py index 23ffbed447..7e9711eaf5 100644 --- a/Lib/test/test_pep380.py +++ b/Lib/test/test_yield_from.py @@ -384,9 +384,10 @@ class TestPEP380Operation(unittest.TestCase): trace.append("Starting g1") yield "g1 ham" ret = yield from g2() - trace.append("g2 returned %s" % (ret,)) - ret = yield from g2(42) - trace.append("g2 returned %s" % (ret,)) + trace.append("g2 returned %r" % (ret,)) + for v in 1, (2,), StopIteration(3): + ret = yield from g2(v) + trace.append("g2 returned %r" % (ret,)) yield "g1 eggs" trace.append("Finishing g1") def g2(v = None): @@ -410,7 +411,17 @@ class TestPEP380Operation(unittest.TestCase): "Yielded g2 spam", "Yielded g2 more spam", "Finishing g2", - "g2 returned 42", + "g2 returned 1", + "Starting g2", + "Yielded g2 spam", + "Yielded g2 more spam", + "Finishing g2", + "g2 returned (2,)", + "Starting g2", + "Yielded g2 spam", + "Yielded g2 more spam", + "Finishing g2", + "g2 returned StopIteration(3,)", "Yielded g1 eggs", "Finishing g1", ]) @@ -670,14 +681,16 @@ class TestPEP380Operation(unittest.TestCase): next(gi) trace.append("f SHOULD NOT BE HERE") except StopIteration as e: - trace.append("f caught %s" % (repr(e),)) + trace.append("f caught %r" % (e,)) def g(r): trace.append("g starting") yield - trace.append("g returning %s" % (r,)) + trace.append("g returning %r" % (r,)) return r f(None) - f(42) + f(1) + f((2,)) + f(StopIteration(3)) self.assertEqual(trace,[ "g starting", "f resuming g", @@ -685,8 +698,16 @@ class TestPEP380Operation(unittest.TestCase): "f caught StopIteration()", "g starting", "f resuming g", - "g returning 42", - "f caught StopIteration(42,)", + "g returning 1", + "f caught StopIteration(1,)", + "g starting", + "f resuming g", + "g returning (2,)", + "f caught StopIteration((2,),)", + "g starting", + "f resuming g", + "g returning StopIteration(3,)", + "f caught StopIteration(StopIteration(3,),)", ]) def test_send_and_return_with_value(self): @@ -706,22 +727,34 @@ class TestPEP380Operation(unittest.TestCase): def g(r): trace.append("g starting") x = yield - trace.append("g received %s" % (x,)) - trace.append("g returning %s" % (r,)) + trace.append("g received %r" % (x,)) + trace.append("g returning %r" % (r,)) return r f(None) - f(42) - self.assertEqual(trace,[ + f(1) + f((2,)) + f(StopIteration(3)) + self.assertEqual(trace, [ "g starting", "f sending spam to g", - "g received spam", + "g received 'spam'", "g returning None", "f caught StopIteration()", "g starting", "f sending spam to g", - "g received spam", - "g returning 42", - "f caught StopIteration(42,)", + "g received 'spam'", + "g returning 1", + 'f caught StopIteration(1,)', + 'g starting', + 'f sending spam to g', + "g received 'spam'", + 'g returning (2,)', + 'f caught StopIteration((2,),)', + 'g starting', + 'f sending spam to g', + "g received 'spam'", + 'g returning StopIteration(3,)', + 'f caught StopIteration(StopIteration(3,),)' ]) def test_catching_exception_from_subgen_and_returning(self): @@ -729,27 +762,29 @@ class TestPEP380Operation(unittest.TestCase): Test catching an exception thrown into a subgenerator and returning a value """ - trace = [] def inner(): try: yield 1 except ValueError: trace.append("inner caught ValueError") - return 2 + return value def outer(): v = yield from inner() - trace.append("inner returned %r to outer" % v) + trace.append("inner returned %r to outer" % (v,)) yield v - g = outer() - trace.append(next(g)) - trace.append(g.throw(ValueError)) - self.assertEqual(trace,[ - 1, - "inner caught ValueError", - "inner returned 2 to outer", - 2, - ]) + + for value in 2, (2,), StopIteration(2): + trace = [] + g = outer() + trace.append(next(g)) + trace.append(repr(g.throw(ValueError))) + self.assertEqual(trace, [ + 1, + "inner caught ValueError", + "inner returned %r to outer" % (value,), + repr(value), + ]) def test_throwing_GeneratorExit_into_subgen_that_returns(self): """ diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 0dd4bcafdd..0a43b20e2b 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -1,8 +1,8 @@ import contextlib import io import os -import sys import importlib.util +import posixpath import time import struct import zipfile @@ -39,10 +39,6 @@ def get_files(test): yield f test.assertFalse(f.closed) -def openU(zipfp, fn): - with check_warnings(('', DeprecationWarning)): - return zipfp.open(fn, 'rU') - class AbstractTestsWithSourceFile: @classmethod def setUpClass(cls): @@ -62,6 +58,9 @@ class AbstractTestsWithSourceFile: zipfp.write(TESTFN, "another.name") zipfp.write(TESTFN, TESTFN) zipfp.writestr("strfile", self.data) + with zipfp.open('written-open-w', mode='w') as f: + for line in self.line_gen: + f.write(line) def zip_test(self, f, compression): self.make_test_archive(f, compression) @@ -77,7 +76,7 @@ class AbstractTestsWithSourceFile: zipfp.printdir(file=fp) directory = fp.getvalue() lines = directory.splitlines() - self.assertEqual(len(lines), 4) # Number of files + header + self.assertEqual(len(lines), 5) # Number of files + header self.assertIn('File Name', lines[0]) self.assertIn('Modified', lines[0]) @@ -91,23 +90,25 @@ class AbstractTestsWithSourceFile: # Check the namelist names = zipfp.namelist() - self.assertEqual(len(names), 3) + self.assertEqual(len(names), 4) self.assertIn(TESTFN, names) self.assertIn("another.name", names) self.assertIn("strfile", names) + self.assertIn("written-open-w", names) # Check infolist infos = zipfp.infolist() names = [i.filename for i in infos] - self.assertEqual(len(names), 3) + self.assertEqual(len(names), 4) self.assertIn(TESTFN, names) self.assertIn("another.name", names) self.assertIn("strfile", names) + self.assertIn("written-open-w", names) for i in infos: self.assertEqual(i.file_size, len(self.data)) # check getinfo - for nm in (TESTFN, "another.name", "strfile"): + for nm in (TESTFN, "another.name", "strfile", "written-open-w"): info = zipfp.getinfo(nm) self.assertEqual(info.filename, nm) self.assertEqual(info.file_size, len(self.data)) @@ -373,14 +374,18 @@ class StoredTestsWithSourceFile(AbstractTestsWithSourceFile, test_low_compression = None def zip_test_writestr_permissions(self, f, compression): - # Make sure that writestr creates files with mode 0600, - # when it is passed a name rather than a ZipInfo instance. + # Make sure that writestr and open(... mode='w') create files with + # mode 0600, when they are passed a name rather than a ZipInfo + # instance. self.make_test_archive(f, compression) with zipfile.ZipFile(f, "r") as zipfp: zinfo = zipfp.getinfo('strfile') self.assertEqual(zinfo.external_attr, 0o600 << 16) + zinfo2 = zipfp.getinfo('written-open-w') + self.assertEqual(zinfo2.external_attr, 0o600 << 16) + def test_writestr_permissions(self): for f in get_files(self): self.zip_test_writestr_permissions(f, zipfile.ZIP_STORED) @@ -488,12 +493,16 @@ class StoredTestsWithSourceFile(AbstractTestsWithSourceFile, def test_write_to_readonly(self): """Check that trying to call write() on a readonly ZipFile object - raises a RuntimeError.""" + raises a ValueError.""" with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: zipfp.writestr("somefile.txt", "bogus") with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: - self.assertRaises(RuntimeError, zipfp.write, TESTFN) + self.assertRaises(ValueError, zipfp.write, TESTFN) + + with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: + with self.assertRaises(ValueError): + zipfp.open(TESTFN, mode='w') def test_add_file_before_1980(self): # Set atime and mtime to 1970-01-01 @@ -1066,32 +1075,6 @@ class OtherTests(unittest.TestCase): data += zipfp.read(info) self.assertIn(data, {b"foobar", b"barfoo"}) - def test_universal_deprecation(self): - f = io.BytesIO() - with zipfile.ZipFile(f, "w") as zipfp: - zipfp.writestr('spam.txt', b'ababagalamaga') - - with zipfile.ZipFile(f, "r") as zipfp: - for mode in 'U', 'rU': - with self.assertWarns(DeprecationWarning): - zipopen = zipfp.open('spam.txt', mode) - zipopen.close() - - def test_universal_readaheads(self): - f = io.BytesIO() - - data = b'a\r\n' * 16 * 1024 - with zipfile.ZipFile(f, 'w', zipfile.ZIP_STORED) as zipfp: - zipfp.writestr(TESTFN, data) - - data2 = b'' - with zipfile.ZipFile(f, 'r') as zipfp, \ - openU(zipfp, TESTFN) as zipopen: - for line in zipopen: - data2 += line - - self.assertEqual(data, data2.replace(b'\n', b'\r\n')) - def test_writestr_extended_local_header_issue1202(self): with zipfile.ZipFile(TESTFN2, 'w') as orig_zip: for data in 'abcdefghijklmnop': @@ -1271,27 +1254,27 @@ class OtherTests(unittest.TestCase): fp.write("short file") self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, TESTFN) - def test_closed_zip_raises_RuntimeError(self): + def test_closed_zip_raises_ValueError(self): """Verify that testzip() doesn't swallow inappropriate exceptions.""" data = io.BytesIO() with zipfile.ZipFile(data, mode="w") as zipf: zipf.writestr("foo.txt", "O, for a Muse of Fire!") # This is correct; calling .read on a closed ZipFile should raise - # a RuntimeError, and so should calling .testzip. An earlier + # a ValueError, and so should calling .testzip. An earlier # version of .testzip would swallow this exception (and any other) # and report that the first file in the archive was corrupt. - self.assertRaises(RuntimeError, zipf.read, "foo.txt") - self.assertRaises(RuntimeError, zipf.open, "foo.txt") - self.assertRaises(RuntimeError, zipf.testzip) - self.assertRaises(RuntimeError, zipf.writestr, "bogus.txt", "bogus") + self.assertRaises(ValueError, zipf.read, "foo.txt") + self.assertRaises(ValueError, zipf.open, "foo.txt") + self.assertRaises(ValueError, zipf.testzip) + self.assertRaises(ValueError, zipf.writestr, "bogus.txt", "bogus") with open(TESTFN, 'w') as f: f.write('zipfile test data') - self.assertRaises(RuntimeError, zipf.write, TESTFN) + self.assertRaises(ValueError, zipf.write, TESTFN) def test_bad_constructor_mode(self): """Check that bad modes passed to ZipFile constructor are caught.""" - self.assertRaises(RuntimeError, zipfile.ZipFile, TESTFN, "q") + self.assertRaises(ValueError, zipfile.ZipFile, TESTFN, "q") def test_bad_open_mode(self): """Check that bad modes passed to ZipFile.open are caught.""" @@ -1299,9 +1282,12 @@ class OtherTests(unittest.TestCase): zipf.writestr("foo.txt", "O, for a Muse of Fire!") with zipfile.ZipFile(TESTFN, mode="r") as zipf: - # read the data to make sure the file is there + # read the data to make sure the file is there zipf.read("foo.txt") - self.assertRaises(RuntimeError, zipf.open, "foo.txt", "q") + self.assertRaises(ValueError, zipf.open, "foo.txt", "q") + # universal newlines support is removed + self.assertRaises(ValueError, zipf.open, "foo.txt", "U") + self.assertRaises(ValueError, zipf.open, "foo.txt", "rU") def test_read0(self): """Check that calling read(0) on a ZipExtFile object returns an empty @@ -1324,7 +1310,7 @@ class OtherTests(unittest.TestCase): def test_bad_compression_mode(self): """Check that bad compression methods passed to ZipFile.open are caught.""" - self.assertRaises(RuntimeError, zipfile.ZipFile, TESTFN, "w", -1) + self.assertRaises(NotImplementedError, zipfile.ZipFile, TESTFN, "w", -1) def test_unsupported_compression(self): # data is declared as shrunk, but actually deflated @@ -1472,6 +1458,35 @@ class OtherTests(unittest.TestCase): # testzip returns the name of the first corrupt file, or None self.assertIsNone(zipf.testzip()) + def test_open_conflicting_handles(self): + # It's only possible to open one writable file handle at a time + msg1 = b"It's fun to charter an accountant!" + msg2 = b"And sail the wide accountant sea" + msg3 = b"To find, explore the funds offshore" + with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_STORED) as zipf: + with zipf.open('foo', mode='w') as w2: + w2.write(msg1) + with zipf.open('bar', mode='w') as w1: + with self.assertRaises(ValueError): + zipf.open('handle', mode='w') + with self.assertRaises(ValueError): + zipf.open('foo', mode='r') + with self.assertRaises(ValueError): + zipf.writestr('str', 'abcde') + with self.assertRaises(ValueError): + zipf.write(__file__, 'file') + with self.assertRaises(ValueError): + zipf.close() + w1.write(msg2) + with zipf.open('baz', mode='w') as w2: + w2.write(msg3) + + with zipfile.ZipFile(TESTFN2, 'r') as zipf: + self.assertEqual(zipf.read('foo'), msg1) + self.assertEqual(zipf.read('bar'), msg2) + self.assertEqual(zipf.read('baz'), msg3) + self.assertEqual(zipf.namelist(), ['foo', 'bar', 'baz']) + def tearDown(self): unlink(TESTFN) unlink(TESTFN2) @@ -1805,6 +1820,22 @@ class UnseekableTests(unittest.TestCase): with zipf.open('twos') as zopen: self.assertEqual(zopen.read(), b'222') + def test_open_write(self): + for wrapper in (lambda f: f), Tellable, Unseekable: + with self.subTest(wrapper=wrapper): + f = io.BytesIO() + f.write(b'abc') + bf = io.BufferedWriter(f) + with zipfile.ZipFile(wrapper(bf), 'w', zipfile.ZIP_STORED) as zipf: + with zipf.open('ones', 'w') as zopen: + zopen.write(b'111') + with zipf.open('twos', 'w') as zopen: + zopen.write(b'222') + self.assertEqual(f.getvalue()[:5], b'abcPK') + with zipfile.ZipFile(f) as zipf: + self.assertEqual(zipf.read('ones'), b'111') + self.assertEqual(zipf.read('twos'), b'222') + @requires_zlib class TestsWithMultipleOpens(unittest.TestCase): @@ -1915,6 +1946,19 @@ class TestsWithMultipleOpens(unittest.TestCase): with open(os.devnull) as f: self.assertLess(f.fileno(), 100) + def test_write_while_reading(self): + with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_DEFLATED) as zipf: + zipf.writestr('ones', self.data1) + with zipfile.ZipFile(TESTFN2, 'a', zipfile.ZIP_DEFLATED) as zipf: + with zipf.open('ones', 'r') as r1: + data1 = r1.read(500) + with zipf.open('twos', 'w') as w1: + w1.write(self.data2) + data1 += r1.read() + self.assertEqual(data1, self.data1) + with zipfile.ZipFile(TESTFN2) as zipf: + self.assertEqual(zipf.read('twos'), self.data2) + def tearDown(self): unlink(TESTFN2) @@ -1984,137 +2028,19 @@ class TestWithDirectory(unittest.TestCase): unlink(TESTFN) -class AbstractUniversalNewlineTests: - @classmethod - def setUpClass(cls): - cls.line_gen = [bytes("Test of zipfile line %d." % i, "ascii") - for i in range(FIXEDTEST_SIZE)] - cls.seps = (b'\r', b'\r\n', b'\n') - cls.arcdata = {} - for n, s in enumerate(cls.seps): - cls.arcdata[s] = s.join(cls.line_gen) + s - - def setUp(self): - self.arcfiles = {} - for n, s in enumerate(self.seps): - self.arcfiles[s] = '%s-%d' % (TESTFN, n) - with open(self.arcfiles[s], "wb") as f: - f.write(self.arcdata[s]) - - def make_test_archive(self, f, compression): - # Create the ZIP archive - with zipfile.ZipFile(f, "w", compression) as zipfp: - for fn in self.arcfiles.values(): - zipfp.write(fn, fn) - - def read_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r") as zipfp: - for sep, fn in self.arcfiles.items(): - with openU(zipfp, fn) as fp: - zipdata = fp.read() - self.assertEqual(self.arcdata[sep], zipdata) - - def test_read(self): - for f in get_files(self): - self.read_test(f, self.compression) - - def readline_read_test(self, f, compression): - self.make_test_archive(f, compression) +class ZipInfoTests(unittest.TestCase): + def test_from_file(self): + zi = zipfile.ZipInfo.from_file(__file__) + self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') + self.assertFalse(zi.is_dir()) - # Read the ZIP archive - with zipfile.ZipFile(f, "r") as zipfp: - for sep, fn in self.arcfiles.items(): - with openU(zipfp, fn) as zipopen: - data = b'' - while True: - read = zipopen.readline() - if not read: - break - data += read - - read = zipopen.read(5) - if not read: - break - data += read - - self.assertEqual(data, self.arcdata[b'\n']) - - def test_readline_read(self): - for f in get_files(self): - self.readline_read_test(f, self.compression) - - def readline_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r") as zipfp: - for sep, fn in self.arcfiles.items(): - with openU(zipfp, fn) as zipopen: - for line in self.line_gen: - linedata = zipopen.readline() - self.assertEqual(linedata, line + b'\n') - - def test_readline(self): - for f in get_files(self): - self.readline_test(f, self.compression) - - def readlines_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r") as zipfp: - for sep, fn in self.arcfiles.items(): - with openU(zipfp, fn) as fp: - ziplines = fp.readlines() - for line, zipline in zip(self.line_gen, ziplines): - self.assertEqual(zipline, line + b'\n') - - def test_readlines(self): - for f in get_files(self): - self.readlines_test(f, self.compression) - - def iterlines_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r") as zipfp: - for sep, fn in self.arcfiles.items(): - with openU(zipfp, fn) as fp: - for line, zipline in zip(self.line_gen, fp): - self.assertEqual(zipline, line + b'\n') - - def test_iterlines(self): - for f in get_files(self): - self.iterlines_test(f, self.compression) - - def tearDown(self): - for sep, fn in self.arcfiles.items(): - unlink(fn) - unlink(TESTFN) - unlink(TESTFN2) - - -class StoredUniversalNewlineTests(AbstractUniversalNewlineTests, - unittest.TestCase): - compression = zipfile.ZIP_STORED - -@requires_zlib -class DeflateUniversalNewlineTests(AbstractUniversalNewlineTests, - unittest.TestCase): - compression = zipfile.ZIP_DEFLATED - -@requires_bz2 -class Bzip2UniversalNewlineTests(AbstractUniversalNewlineTests, - unittest.TestCase): - compression = zipfile.ZIP_BZIP2 - -@requires_lzma -class LzmaUniversalNewlineTests(AbstractUniversalNewlineTests, - unittest.TestCase): - compression = zipfile.ZIP_LZMA + def test_from_dir(self): + dirpath = os.path.dirname(os.path.abspath(__file__)) + zi = zipfile.ZipInfo.from_file(dirpath, 'stdlib_tests') + self.assertEqual(zi.filename, 'stdlib_tests/') + self.assertTrue(zi.is_dir()) + self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) + self.assertEqual(zi.file_size, 0) class CommandLineTest(unittest.TestCase): @@ -2175,7 +2101,7 @@ class CommandLineTest(unittest.TestCase): for zi in zf.infolist(): path = os.path.join(extdir, zi.filename.replace('/', os.sep)) - if zi.filename.endswith('/'): + if zi.is_dir(): self.assertTrue(os.path.isdir(path)) else: self.assertTrue(os.path.isfile(path)) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index d5b3b22ae1..7ddbc509f9 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -398,7 +398,8 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): packdir2 = packdir + TESTPACK2 + os.sep files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), + "spam" + pyc_ext: (NOW, test_pyc)} z = ZipFile(TEMP_ZIP, "w") try: @@ -412,6 +413,14 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): zi = zipimport.zipimporter(TEMP_ZIP) self.assertEqual(zi.archive, TEMP_ZIP) self.assertEqual(zi.is_package(TESTPACK), True) + + find_mod = zi.find_module('spam') + self.assertIsNotNone(find_mod) + self.assertIsInstance(find_mod, zipimport.zipimporter) + self.assertFalse(find_mod.is_package('spam')) + load_mod = find_mod.load_module('spam') + self.assertEqual(find_mod.get_filename('spam'), load_mod.__file__) + mod = zi.load_module(TESTPACK) self.assertEqual(zi.get_filename(TESTPACK), mod.__file__) @@ -471,6 +480,16 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): self.assertEqual( zi.is_package(TESTPACK2 + os.sep + TESTMOD), False) + pkg_path = TEMP_ZIP + os.sep + packdir + TESTPACK2 + zi2 = zipimport.zipimporter(pkg_path) + find_mod_dotted = zi2.find_module(TESTMOD) + self.assertIsNotNone(find_mod_dotted) + self.assertIsInstance(find_mod_dotted, zipimport.zipimporter) + self.assertFalse(zi2.is_package(TESTMOD)) + load_mod = find_mod_dotted.load_module(TESTMOD) + self.assertEqual( + find_mod_dotted.get_filename(TESTMOD), load_mod.__file__) + mod_path = TESTPACK2 + os.sep + TESTMOD mod_name = module_path_to_dotted_name(mod_path) mod = importlib.import_module(mod_name) @@ -623,8 +642,10 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): zipimport.zipimporter(filename) zipimport.zipimporter(os.fsencode(filename)) - zipimport.zipimporter(bytearray(os.fsencode(filename))) - zipimport.zipimporter(memoryview(os.fsencode(filename))) + with self.assertWarns(DeprecationWarning): + zipimport.zipimporter(bytearray(os.fsencode(filename))) + with self.assertWarns(DeprecationWarning): + zipimport.zipimporter(memoryview(os.fsencode(filename))) @support.requires_zlib diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index 5913622f56..84d526c3db 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -12,7 +12,6 @@ import zipimport import doctest import inspect import linecache -import pdb import unittest from test.support.script_helper import (spawn_python, kill_python, assert_python_ok, make_script, make_zip_script) diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 6fea893993..20174d8343 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -164,6 +164,16 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): x = zlib.compress(HAMLET_SCENE) self.assertEqual(zlib.decompress(x), HAMLET_SCENE) + def test_keywords(self): + x = zlib.compress(HAMLET_SCENE, level=3) + self.assertEqual(zlib.decompress(x), HAMLET_SCENE) + with self.assertRaises(TypeError): + zlib.compress(data=HAMLET_SCENE, level=3) + self.assertEqual(zlib.decompress(x, + wbits=zlib.MAX_WBITS, + bufsize=zlib.DEF_BUF_SIZE), + HAMLET_SCENE) + def test_speech128(self): # compress more data data = HAMLET_SCENE * 128 @@ -234,6 +244,27 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): self.assertIsInstance(dco.unconsumed_tail, bytes) self.assertIsInstance(dco.unused_data, bytes) + def test_keywords(self): + level = 2 + method = zlib.DEFLATED + wbits = -12 + memLevel = 9 + strategy = zlib.Z_FILTERED + co = zlib.compressobj(level=level, + method=method, + wbits=wbits, + memLevel=memLevel, + strategy=strategy, + zdict=b"") + do = zlib.decompressobj(wbits=wbits, zdict=b"") + with self.assertRaises(TypeError): + co.compress(data=HAMLET_SCENE) + with self.assertRaises(TypeError): + do.decompress(data=zlib.compress(HAMLET_SCENE)) + x = co.compress(HAMLET_SCENE) + co.flush() + y = do.decompress(x, max_length=len(HAMLET_SCENE)) + do.flush() + self.assertEqual(HAMLET_SCENE, y) + def test_compressoptions(self): # specify lots of options to compressobj() level = 2 @@ -249,10 +280,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): y2 = dco.flush() self.assertEqual(HAMLET_SCENE, y1 + y2) - # keyword arguments should also be supported - zlib.compressobj(level=level, method=method, wbits=wbits, - memLevel=memLevel, strategy=strategy, zdict=b"") - def test_compressincremental(self): # compress object in steps, decompress object as one-shot data = HAMLET_SCENE * 128 |