From 0066ebf6461f1d7a7cae7190aa169c6fdd6fdceb Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 30 May 2015 13:24:54 -0600 Subject: Issue #16991: Ensure that the proper OrderedDict is used in tests. --- Lib/test/test_collections.py | 1 + 1 file changed, 1 insertion(+) (limited to 'Lib') diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 5836cb37ea..e65824f42c 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1979,6 +1979,7 @@ class OrderedDictTests: self.assertGreater(sys.getsizeof(od), sys.getsizeof(d)) def test_views(self): + OrderedDict = self.module.OrderedDict # See http://bugs.python.org/issue24286 s = 'the quick brown fox jumped over a lazy dog yesterday before dawn'.split() od = OrderedDict.fromkeys(s) -- cgit v1.2.1 From a163095bd658cb9bc862286886cf81f358c28150 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 3 Jun 2015 19:04:42 -0700 Subject: Update typing.py: remove isinstance() support (Mark struck it from the PEP). --- Lib/test/test_typing.py | 245 ++++++++++-------------------------------------- Lib/typing.py | 156 +++++------------------------- 2 files changed, 71 insertions(+), 330 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index c37e1130a1..97404258a3 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -41,11 +41,9 @@ class ManagingFounder(Manager, Founder): class AnyTests(TestCase): - def test_any_instance(self): - self.assertIsInstance(Employee(), Any) - self.assertIsInstance(42, Any) - self.assertIsInstance(None, Any) - self.assertIsInstance(object(), Any) + def test_any_instance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, Any) def test_any_subclass(self): self.assertTrue(issubclass(Employee, Any)) @@ -109,9 +107,6 @@ class TypeVarTests(TestCase): def test_basic_plain(self): T = TypeVar('T') - # Nothing is an instance if T. - with self.assertRaises(TypeError): - isinstance('', T) # Every class is a subclass of T. assert issubclass(int, T) assert issubclass(str, T) @@ -119,12 +114,16 @@ class TypeVarTests(TestCase): assert T == T # T is a subclass of itself. assert issubclass(T, T) + # T is an instance of TypeVar + assert isinstance(T, TypeVar) + + def test_typevar_instance_type_error(self): + T = TypeVar('T') + with self.assertRaises(TypeError): + isinstance(42, T) def test_basic_constrained(self): A = TypeVar('A', str, bytes) - # Nothing is an instance of A. - with self.assertRaises(TypeError): - isinstance('', A) # Only str and bytes are subclasses of A. assert issubclass(str, A) assert issubclass(bytes, A) @@ -213,8 +212,6 @@ class UnionTests(TestCase): def test_basics(self): u = Union[int, float] self.assertNotEqual(u, Union) - self.assertIsInstance(42, u) - self.assertIsInstance(3.14, u) self.assertTrue(issubclass(int, u)) self.assertTrue(issubclass(float, u)) @@ -247,7 +244,6 @@ class UnionTests(TestCase): def test_subclass(self): u = Union[int, Employee] - self.assertIsInstance(Manager(), u) self.assertTrue(issubclass(Manager, u)) def test_self_subclass(self): @@ -256,7 +252,6 @@ class UnionTests(TestCase): def test_multiple_inheritance(self): u = Union[int, Employee] - self.assertIsInstance(ManagingFounder(), u) self.assertTrue(issubclass(ManagingFounder, u)) def test_single_class_disappears(self): @@ -309,9 +304,6 @@ class UnionTests(TestCase): o = Optional[int] u = Union[int, None] self.assertEqual(o, u) - self.assertIsInstance(42, o) - self.assertIsInstance(None, o) - self.assertNotIsInstance(3.14, o) def test_empty(self): with self.assertRaises(TypeError): @@ -321,11 +313,9 @@ class UnionTests(TestCase): assert issubclass(Union[int, str], Union) assert not issubclass(int, Union) - def test_isinstance_union(self): - # Nothing is an instance of bare Union. - assert not isinstance(42, Union) - assert not isinstance(int, Union) - assert not isinstance(Union[int, str], Union) + def test_union_instance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, Union[int, str]) class TypeVarUnionTests(TestCase): @@ -352,22 +342,11 @@ class TypeVarUnionTests(TestCase): TU = TypeVar('TU', Union[int, float], None) assert issubclass(int, TU) assert issubclass(float, TU) - with self.assertRaises(TypeError): - isinstance(42, TU) - with self.assertRaises(TypeError): - isinstance('', TU) class TupleTests(TestCase): def test_basics(self): - self.assertIsInstance((42, 3.14, ''), Tuple) - self.assertIsInstance((42, 3.14, ''), Tuple[int, float, str]) - self.assertIsInstance((42,), Tuple[int]) - self.assertNotIsInstance((3.14,), Tuple[int]) - self.assertNotIsInstance((42, 3.14), Tuple[int, float, str]) - self.assertNotIsInstance((42, 3.14, 100), Tuple[int, float, str]) - self.assertNotIsInstance((42, 3.14, 100), Tuple[int, float]) self.assertTrue(issubclass(Tuple[int, str], Tuple)) self.assertTrue(issubclass(Tuple[int, str], Tuple[int, str])) self.assertFalse(issubclass(int, Tuple)) @@ -382,14 +361,11 @@ class TupleTests(TestCase): pass self.assertTrue(issubclass(MyTuple, Tuple)) - def test_tuple_ellipsis(self): - t = Tuple[int, ...] - assert isinstance((), t) - assert isinstance((1,), t) - assert isinstance((1, 2), t) - assert isinstance((1, 2, 3), t) - assert not isinstance((3.14,), t) - assert not isinstance((1, 2, 3.14,), t) + def test_tuple_instance_type_error(self): + with self.assertRaises(TypeError): + isinstance((0, 0), Tuple[int, int]) + with self.assertRaises(TypeError): + isinstance((0, 0), Tuple) def test_tuple_ellipsis_subclass(self): @@ -419,18 +395,6 @@ class TupleTests(TestCase): class CallableTests(TestCase): - def test_basics(self): - c = Callable[[int, float], str] - - def flub(a: int, b: float) -> str: - return str(a * b) - - def flob(a: int, b: int) -> str: - return str(a * b) - - self.assertIsInstance(flub, c) - self.assertNotIsInstance(flob, c) - def test_self_subclass(self): self.assertTrue(issubclass(Callable[[int], int], Callable)) self.assertFalse(issubclass(Callable, Callable[[int], int])) @@ -453,91 +417,6 @@ class CallableTests(TestCase): self.assertNotEqual(Callable[[int], int], Callable[[], int]) self.assertNotEqual(Callable[[int], int], Callable) - def test_with_none(self): - c = Callable[[None], None] - - def flub(self: None) -> None: - pass - - def flab(self: Any) -> None: - pass - - def flob(self: None) -> Any: - pass - - self.assertIsInstance(flub, c) - self.assertIsInstance(flab, c) - self.assertNotIsInstance(flob, c) # Test contravariance. - - def test_with_subclasses(self): - c = Callable[[Employee, Manager], Employee] - - def flub(a: Employee, b: Employee) -> Manager: - return Manager() - - def flob(a: Manager, b: Manager) -> Employee: - return Employee() - - self.assertIsInstance(flub, c) - self.assertNotIsInstance(flob, c) - - def test_with_default_args(self): - c = Callable[[int], int] - - def flub(a: int, b: float = 3.14) -> int: - return a - - def flab(a: int, *, b: float = 3.14) -> int: - return a - - def flob(a: int = 42) -> int: - return a - - self.assertIsInstance(flub, c) - self.assertIsInstance(flab, c) - self.assertIsInstance(flob, c) - - def test_with_varargs(self): - c = Callable[[int], int] - - def flub(*args) -> int: - return 42 - - def flab(*args: int) -> int: - return 42 - - def flob(*args: float) -> int: - return 42 - - self.assertIsInstance(flub, c) - self.assertIsInstance(flab, c) - self.assertNotIsInstance(flob, c) - - def test_with_method(self): - - class C: - - def imethod(self, arg: int) -> int: - self.last_arg = arg - return arg + 1 - - @classmethod - def cmethod(cls, arg: int) -> int: - cls.last_cls_arg = arg - return arg + 1 - - @staticmethod - def smethod(arg: int) -> int: - return arg + 1 - - ct = Callable[[int], int] - self.assertIsInstance(C().imethod, ct) - self.assertIsInstance(C().cmethod, ct) - self.assertIsInstance(C.cmethod, ct) - self.assertIsInstance(C().smethod, ct) - self.assertIsInstance(C.smethod, ct) - self.assertIsInstance(C.imethod, Callable[[Any, int], int]) - def test_cannot_subclass(self): with self.assertRaises(TypeError): @@ -556,21 +435,21 @@ class CallableTests(TestCase): with self.assertRaises(TypeError): c() - def test_varargs(self): - ct = Callable[..., int] + def test_callable_instance_works(self): + f = lambda: None + assert isinstance(f, Callable) + assert not isinstance(None, Callable) - def foo(a, b) -> int: - return 42 - - def bar(a=42) -> int: - return a - - def baz(*, x, y, z) -> int: - return 100 - - self.assertIsInstance(foo, ct) - self.assertIsInstance(bar, ct) - self.assertIsInstance(baz, ct) + def test_callable_instance_type_error(self): + f = lambda: None + with self.assertRaises(TypeError): + assert isinstance(f, Callable[[], None]) + with self.assertRaises(TypeError): + assert isinstance(f, Callable[[], Any]) + with self.assertRaises(TypeError): + assert not isinstance(None, Callable[[], None]) + with self.assertRaises(TypeError): + assert not isinstance(None, Callable[[], Any]) def test_repr(self): ct0 = Callable[[], bool] @@ -659,6 +538,10 @@ class ProtocolTests(TestCase): assert issubclass(list, typing.Reversible) assert not issubclass(int, typing.Reversible) + def test_protocol_instance_type_error(self): + with self.assertRaises(TypeError): + isinstance([], typing.Reversible) + class GenericTests(TestCase): @@ -889,6 +772,11 @@ class ForwardRefTests(TestCase): right_hints = get_type_hints(t.add_right, globals(), locals()) assert right_hints['node'] == Optional[Node[T]] + def test_forwardref_instance_type_error(self): + fr = typing._ForwardRef('int') + with self.assertRaises(TypeError): + isinstance(42, fr) + def test_union_forward(self): def foo(a: Union['T']): @@ -1069,50 +957,17 @@ class CollectionsAbcTests(TestCase): def test_list(self): assert issubclass(list, typing.List) - assert isinstance([], typing.List) - assert not isinstance((), typing.List) - t = typing.List[int] - assert isinstance([], t) - assert isinstance([42], t) - assert not isinstance([''], t) def test_set(self): assert issubclass(set, typing.Set) assert not issubclass(frozenset, typing.Set) - assert isinstance(set(), typing.Set) - assert not isinstance({}, typing.Set) - t = typing.Set[int] - assert isinstance(set(), t) - assert isinstance({42}, t) - assert not isinstance({''}, t) def test_frozenset(self): assert issubclass(frozenset, typing.FrozenSet) assert not issubclass(set, typing.FrozenSet) - assert isinstance(frozenset(), typing.FrozenSet) - assert not isinstance({}, typing.FrozenSet) - t = typing.FrozenSet[int] - assert isinstance(frozenset(), t) - assert isinstance(frozenset({42}), t) - assert not isinstance(frozenset({''}), t) - assert not isinstance({42}, t) - - def test_mapping_views(self): - # TODO: These tests are kind of lame. - assert isinstance({}.keys(), typing.KeysView) - assert isinstance({}.items(), typing.ItemsView) - assert isinstance({}.values(), typing.ValuesView) def test_dict(self): assert issubclass(dict, typing.Dict) - assert isinstance({}, typing.Dict) - assert not isinstance([], typing.Dict) - t = typing.Dict[int, str] - assert isinstance({}, t) - assert isinstance({42: ''}, t) - assert not isinstance({42: 42}, t) - assert not isinstance({'': 42}, t) - assert not isinstance({'': ''}, t) def test_no_list_instantiation(self): with self.assertRaises(TypeError): @@ -1191,8 +1046,6 @@ class CollectionsAbcTests(TestCase): yield 42 g = foo() assert issubclass(type(g), typing.Generator) - assert isinstance(g, typing.Generator) - assert not isinstance(foo, typing.Generator) assert issubclass(typing.Generator[Manager, Employee, Manager], typing.Generator[Employee, Manager, Employee]) assert not issubclass(typing.Generator[Manager, Manager, Manager], @@ -1228,12 +1081,6 @@ class CollectionsAbcTests(TestCase): assert len(MMB[str, str]()) == 0 assert len(MMB[KT, VT]()) == 0 - def test_recursive_dict(self): - D = typing.Dict[int, 'D'] # Uses a _ForwardRef - assert isinstance({}, D) # Easy - assert isinstance({0: {}}, D) # Touches _ForwardRef - assert isinstance({0: {0: {}}}, D) # Etc... - class NamedTupleTests(TestCase): @@ -1294,8 +1141,6 @@ class RETests(TestCase): def test_basics(self): pat = re.compile('[a-z]+', re.I) assert issubclass(pat.__class__, Pattern) - assert isinstance(pat, Pattern[str]) - assert not isinstance(pat, Pattern[bytes]) assert issubclass(type(pat), Pattern) assert issubclass(type(pat), Pattern[str]) @@ -1307,12 +1152,10 @@ class RETests(TestCase): assert issubclass(type(mat), Match[str]) p = Pattern[Union[str, bytes]] - assert isinstance(pat, p) assert issubclass(Pattern[str], Pattern) assert issubclass(Pattern[str], p) m = Match[Union[bytes, str]] - assert isinstance(mat, m) assert issubclass(Match[bytes], Match) assert issubclass(Match[bytes], m) @@ -1327,6 +1170,12 @@ class RETests(TestCase): with self.assertRaises(TypeError): # Too complicated? m[str] + with self.assertRaises(TypeError): + # We don't support isinstance(). + isinstance(42, Pattern) + with self.assertRaises(TypeError): + # We don't support isinstance(). + isinstance(42, Pattern[str]) def test_repr(self): assert repr(Pattern) == 'Pattern[~AnyStr]' diff --git a/Lib/typing.py b/Lib/typing.py index 38e07ad50b..66bee917d8 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -176,6 +176,9 @@ class _ForwardRef(TypingMeta): self.__forward_evaluated__ = True return self.__forward_value__ + def __instancecheck__(self, obj): + raise TypeError("Forward references cannot be used with isinstance().") + def __subclasscheck__(self, cls): if not self.__forward_evaluated__: globalns = self.__forward_frame__.f_globals @@ -186,16 +189,6 @@ class _ForwardRef(TypingMeta): return False # Too early. return issubclass(cls, self.__forward_value__) - def __instancecheck__(self, obj): - if not self.__forward_evaluated__: - globalns = self.__forward_frame__.f_globals - localns = self.__forward_frame__.f_locals - try: - self._eval_type(globalns, localns) - except NameError: - return False # Too early. - return isinstance(obj, self.__forward_value__) - def __repr__(self): return '_ForwardRef(%r)' % (self.__forward_arg__,) @@ -259,8 +252,7 @@ class _TypeAlias: self.impl_type, self.type_checker) def __instancecheck__(self, obj): - return (isinstance(obj, self.impl_type) and - isinstance(self.type_checker(obj), self.type_var)) + raise TypeError("Type aliases cannot be used with isinstance().") def __subclasscheck__(self, cls): if cls is Any: @@ -332,8 +324,8 @@ class AnyMeta(TypingMeta): self = super().__new__(cls, name, bases, namespace, _root=_root) return self - def __instancecheck__(self, instance): - return True + def __instancecheck__(self, obj): + raise TypeError("Any cannot be used with isinstance().") def __subclasscheck__(self, cls): if not isinstance(cls, type): @@ -548,9 +540,8 @@ class UnionMeta(TypingMeta): def __hash__(self): return hash(self.__union_set_params__) - def __instancecheck__(self, instance): - return (self.__union_set_params__ is not None and - any(isinstance(instance, t) for t in self.__union_params__)) + def __instancecheck__(self, obj): + raise TypeError("Unions cannot be used with isinstance().") def __subclasscheck__(self, cls): if cls is Any: @@ -709,18 +700,8 @@ class TupleMeta(TypingMeta): def __hash__(self): return hash(self.__tuple_params__) - def __instancecheck__(self, t): - if not isinstance(t, tuple): - return False - if self.__tuple_params__ is None: - return True - if self.__tuple_use_ellipsis__: - p = self.__tuple_params__[0] - return all(isinstance(x, p) for x in t) - else: - return (len(t) == len(self.__tuple_params__) and - all(isinstance(x, p) - for x, p in zip(t, self.__tuple_params__))) + def __instancecheck__(self, obj): + raise TypeError("Tuples cannot be used with isinstance().") def __subclasscheck__(self, cls): if cls is Any: @@ -826,57 +807,14 @@ class CallableMeta(TypingMeta): def __hash__(self): return hash(self.__args__) ^ hash(self.__result__) - def __instancecheck__(self, instance): - if not callable(instance): - return False + def __instancecheck__(self, obj): + # For unparametrized Callable we allow this, because + # typing.Callable should be equivalent to + # collections.abc.Callable. if self.__args__ is None and self.__result__ is None: - return True - assert self.__args__ is not None - assert self.__result__ is not None - my_args, my_result = self.__args__, self.__result__ - import inspect # TODO: Avoid this import. - # Would it be better to use Signature objects? - try: - (args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, - annotations) = inspect.getfullargspec(instance) - except TypeError: - return False # We can't find the signature. Give up. - msg = ("When testing isinstance(, Callable[...], " - "'s annotations must be types.") - if my_args is not Ellipsis: - if kwonlyargs and (not kwonlydefaults or - len(kwonlydefaults) < len(kwonlyargs)): - return False - if isinstance(instance, types.MethodType): - # For methods, getfullargspec() includes self/cls, - # but it's not part of the call signature, so drop it. - del args[0] - min_call_args = len(args) - if defaults: - min_call_args -= len(defaults) - if varargs: - max_call_args = 999999999 - if len(args) < len(my_args): - args += [varargs] * (len(my_args) - len(args)) - else: - max_call_args = len(args) - if not min_call_args <= len(my_args) <= max_call_args: - return False - for my_arg_type, name in zip(my_args, args): - if name in annotations: - annot_type = _type_check(annotations[name], msg) - else: - annot_type = Any - if not issubclass(my_arg_type, annot_type): - return False - # TODO: If mutable type, check invariance? - if 'return' in annotations: - annot_return_type = _type_check(annotations['return'], msg) - # Note contravariance here! - if not issubclass(annot_return_type, my_result): - return False - # Can't find anything wrong... - return True + return isinstance(obj, collections_abc.Callable) + else: + raise TypeError("Callable[] cannot be used with isinstance().") def __subclasscheck__(self, cls): if cls is Any: @@ -1073,13 +1011,6 @@ class GenericMeta(TypingMeta, abc.ABCMeta): return False return issubclass(cls, self.__extra__) - def __instancecheck__(self, obj): - if super().__instancecheck__(obj): - return True - if self.__extra__ is None: - return False - return isinstance(obj, self.__extra__) - class Generic(metaclass=GenericMeta): """Abstract base class for generic types. @@ -1234,6 +1165,9 @@ class _ProtocolMeta(GenericMeta): from Generic. """ + def __instancecheck__(self, obj): + raise TypeError("Protocols cannot be used with isinstance().") + def __subclasscheck__(self, cls): if not self._is_protocol: # No structural checks since this isn't a protocol. @@ -1399,19 +1333,7 @@ class ByteString(Sequence[int], extra=collections_abc.ByteString): ByteString.register(type(memoryview(b''))) -class _ListMeta(GenericMeta): - - def __instancecheck__(self, obj): - if not super().__instancecheck__(obj): - return False - itemtype = self.__parameters__[0] - for x in obj: - if not isinstance(x, itemtype): - return False - return True - - -class List(list, MutableSequence[T], metaclass=_ListMeta): +class List(list, MutableSequence[T]): def __new__(cls, *args, **kwds): if _geqv(cls, List): @@ -1420,19 +1342,7 @@ class List(list, MutableSequence[T], metaclass=_ListMeta): return list.__new__(cls, *args, **kwds) -class _SetMeta(GenericMeta): - - def __instancecheck__(self, obj): - if not super().__instancecheck__(obj): - return False - itemtype = self.__parameters__[0] - for x in obj: - if not isinstance(x, itemtype): - return False - return True - - -class Set(set, MutableSet[T], metaclass=_SetMeta): +class Set(set, MutableSet[T]): def __new__(cls, *args, **kwds): if _geqv(cls, Set): @@ -1441,7 +1351,7 @@ class Set(set, MutableSet[T], metaclass=_SetMeta): return set.__new__(cls, *args, **kwds) -class _FrozenSetMeta(_SetMeta): +class _FrozenSetMeta(GenericMeta): """This metaclass ensures set is not a subclass of FrozenSet. Without this metaclass, set would be considered a subclass of @@ -1454,11 +1364,6 @@ class _FrozenSetMeta(_SetMeta): return False return super().__subclasscheck__(cls) - def __instancecheck__(self, obj): - if issubclass(obj.__class__, Set): - return False - return super().__instancecheck__(obj) - class FrozenSet(frozenset, AbstractSet[T_co], metaclass=_FrozenSetMeta): @@ -1488,20 +1393,7 @@ class ValuesView(MappingView[VT_co], extra=collections_abc.ValuesView): pass -class _DictMeta(GenericMeta): - - def __instancecheck__(self, obj): - if not super().__instancecheck__(obj): - return False - keytype, valuetype = self.__parameters__ - for key, value in obj.items(): - if not (isinstance(key, keytype) and - isinstance(value, valuetype)): - return False - return True - - -class Dict(dict, MutableMapping[KT, VT], metaclass=_DictMeta): +class Dict(dict, MutableMapping[KT, VT]): def __new__(cls, *args, **kwds): if _geqv(cls, Dict): -- cgit v1.2.1 From 421bdf0a873272d3b3b4630aeace6030d19652a5 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 7 Jun 2015 13:36:19 -0700 Subject: Mapping key type is invariant. --- Lib/test/test_typing.py | 6 +++--- Lib/typing.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 97404258a3..2bb21edc18 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -707,11 +707,11 @@ class VarianceTests(TestCase): typing.Sequence[Manager]) def test_covariance_mapping(self): - # Ditto for Mapping (a generic class with two parameters). + # Ditto for Mapping (covariant in the value, invariant in the key). assert issubclass(typing.Mapping[Employee, Manager], typing.Mapping[Employee, Employee]) - assert issubclass(typing.Mapping[Manager, Employee], - typing.Mapping[Employee, Employee]) + assert not issubclass(typing.Mapping[Manager, Employee], + typing.Mapping[Employee, Employee]) assert not issubclass(typing.Mapping[Employee, Manager], typing.Mapping[Manager, Manager]) assert not issubclass(typing.Mapping[Manager, Employee], diff --git a/Lib/typing.py b/Lib/typing.py index 66bee917d8..bc6fcdd03d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -439,7 +439,6 @@ KT = TypeVar('KT') # Key type. VT = TypeVar('VT') # Value type. T_co = TypeVar('T_co', covariant=True) # Any type covariant containers. V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. -KT_co = TypeVar('KT_co', covariant=True) # Key type covariant containers. VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. @@ -1308,7 +1307,8 @@ class MutableSet(AbstractSet[T], extra=collections_abc.MutableSet): pass -class Mapping(Sized, Iterable[KT_co], Container[KT_co], Generic[KT_co, VT_co], +# NOTE: Only the value type is covariant. +class Mapping(Sized, Iterable[KT], Container[KT], Generic[KT, VT_co], extra=collections_abc.Mapping): pass @@ -1378,13 +1378,13 @@ class MappingView(Sized, Iterable[T_co], extra=collections_abc.MappingView): pass -class KeysView(MappingView[KT_co], AbstractSet[KT_co], +class KeysView(MappingView[KT], AbstractSet[KT], extra=collections_abc.KeysView): pass -# TODO: Enable Set[Tuple[KT_co, VT_co]] instead of Generic[KT_co, VT_co]. -class ItemsView(MappingView, Generic[KT_co, VT_co], +# TODO: Enable Set[Tuple[KT, VT_co]] instead of Generic[KT, VT_co]. +class ItemsView(MappingView, Generic[KT, VT_co], extra=collections_abc.ItemsView): pass -- cgit v1.2.1 From 98b72d5cfe462bccc66b9002ea8a8bb5799cc55d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 12 Jun 2015 17:26:23 +0200 Subject: Issue #15745: Rewrite os.utime() tests in test_os * Don't use the timestamp of an existing file anymore, only use fixed timestamp * Enhance the code checking the resolution of the filesystem timestamps. * Check timestamps with a resolution of 1 microsecond instead of 1 millisecond * When os.utime() uses the current system clock, tolerate a delta of 20 ms. Before some os.utime() tolerated a different of 10 seconds. * Merge duplicated tests and simplify the code --- Lib/test/test_os.py | 431 ++++++++++++++++++++++++++-------------------------- 1 file changed, 219 insertions(+), 212 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index d2dfcaffbe..987b4af8de 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2,33 +2,34 @@ # does add tests for a few functions which have been determined to be more # portable than they had been thought to be. -import os +import asynchat +import asyncore +import codecs +import contextlib +import decimal import errno +import fractions import getpass -import unittest -import warnings -import sys -import signal -import subprocess -import time -import shutil -from test import support -import contextlib +import itertools +import locale +import math import mmap +import os +import pickle import platform import re -import uuid -import asyncore -import asynchat +import shutil +import signal import socket -import itertools import stat -import locale -import codecs -import decimal -import fractions -import pickle +import subprocess +import sys import sysconfig +import time +import unittest +import uuid +import warnings +from test import support try: import threading except ImportError: @@ -70,16 +71,6 @@ root_in_posix = False if hasattr(os, 'geteuid'): root_in_posix = (os.geteuid() == 0) -with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - os.stat_float_times(True) -st = os.stat(__file__) -stat_supports_subsecond = ( - # check if float and int timestamps are different - (st.st_atime != st[7]) - or (st.st_mtime != st[8]) - or (st.st_ctime != st[9])) - # Detect whether we're on a Linux system that uses the (now outdated # and unmaintained) linuxthreads threading library. There's an issue # when combining linuxthreads with a failed execv call: see @@ -223,15 +214,10 @@ class FileTests(unittest.TestCase): # Test attributes on return values from os.*stat* family. class StatAttributeTests(unittest.TestCase): def setUp(self): - os.mkdir(support.TESTFN) - self.fname = os.path.join(support.TESTFN, "f1") - f = open(self.fname, 'wb') - f.write(b"ABC") - f.close() - - def tearDown(self): - os.unlink(self.fname) - os.rmdir(support.TESTFN) + self.fname = support.TESTFN + self.addCleanup(support.unlink, self.fname) + with open(self.fname, 'wb') as fp: + fp.write(b"ABC") @unittest.skipUnless(hasattr(os, 'stat'), 'test needs os.stat()') def check_stat_attributes(self, fname): @@ -383,179 +369,6 @@ class StatAttributeTests(unittest.TestCase): unpickled = pickle.loads(p) self.assertEqual(result, unpickled) - def test_utime_dir(self): - delta = 1000000 - st = os.stat(support.TESTFN) - # round to int, because some systems may support sub-second - # time stamps in stat, but not in utime. - os.utime(support.TESTFN, (st.st_atime, int(st.st_mtime-delta))) - st2 = os.stat(support.TESTFN) - self.assertEqual(st2.st_mtime, int(st.st_mtime-delta)) - - def _test_utime(self, filename, attr, utime, delta): - # Issue #13327 removed the requirement to pass None as the - # second argument. Check that the previous methods of passing - # a time tuple or None work in addition to no argument. - st0 = os.stat(filename) - # Doesn't set anything new, but sets the time tuple way - utime(filename, (attr(st0, "st_atime"), attr(st0, "st_mtime"))) - # Setting the time to the time you just read, then reading again, - # should always return exactly the same times. - st1 = os.stat(filename) - self.assertEqual(attr(st0, "st_mtime"), attr(st1, "st_mtime")) - self.assertEqual(attr(st0, "st_atime"), attr(st1, "st_atime")) - # Set to the current time in the old explicit way. - os.utime(filename, None) - st2 = os.stat(support.TESTFN) - # Set to the current time in the new way - os.utime(filename) - st3 = os.stat(filename) - self.assertAlmostEqual(attr(st2, "st_mtime"), attr(st3, "st_mtime"), delta=delta) - - def test_utime(self): - def utime(file, times): - return os.utime(file, times) - self._test_utime(self.fname, getattr, utime, 10) - self._test_utime(support.TESTFN, getattr, utime, 10) - - - def _test_utime_ns(self, set_times_ns, test_dir=True): - def getattr_ns(o, attr): - return getattr(o, attr + "_ns") - ten_s = 10 * 1000 * 1000 * 1000 - self._test_utime(self.fname, getattr_ns, set_times_ns, ten_s) - if test_dir: - self._test_utime(support.TESTFN, getattr_ns, set_times_ns, ten_s) - - def test_utime_ns(self): - def utime_ns(file, times): - return os.utime(file, ns=times) - self._test_utime_ns(utime_ns) - - requires_utime_dir_fd = unittest.skipUnless( - os.utime in os.supports_dir_fd, - "dir_fd support for utime required for this test.") - requires_utime_fd = unittest.skipUnless( - os.utime in os.supports_fd, - "fd support for utime required for this test.") - requires_utime_nofollow_symlinks = unittest.skipUnless( - os.utime in os.supports_follow_symlinks, - "follow_symlinks support for utime required for this test.") - - @requires_utime_nofollow_symlinks - def test_lutimes_ns(self): - def lutimes_ns(file, times): - return os.utime(file, ns=times, follow_symlinks=False) - self._test_utime_ns(lutimes_ns) - - @requires_utime_fd - def test_futimes_ns(self): - def futimes_ns(file, times): - with open(file, "wb") as f: - os.utime(f.fileno(), ns=times) - self._test_utime_ns(futimes_ns, test_dir=False) - - def _utime_invalid_arguments(self, name, arg): - with self.assertRaises(ValueError): - getattr(os, name)(arg, (5, 5), ns=(5, 5)) - - def test_utime_invalid_arguments(self): - self._utime_invalid_arguments('utime', self.fname) - - - @unittest.skipUnless(stat_supports_subsecond, - "os.stat() doesn't has a subsecond resolution") - def _test_utime_subsecond(self, set_time_func): - asec, amsec = 1, 901 - atime = asec + amsec * 1e-3 - msec, mmsec = 2, 901 - mtime = msec + mmsec * 1e-3 - filename = self.fname - os.utime(filename, (0, 0)) - set_time_func(filename, atime, mtime) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - os.stat_float_times(True) - st = os.stat(filename) - self.assertAlmostEqual(st.st_atime, atime, places=3) - self.assertAlmostEqual(st.st_mtime, mtime, places=3) - - def test_utime_subsecond(self): - def set_time(filename, atime, mtime): - os.utime(filename, (atime, mtime)) - self._test_utime_subsecond(set_time) - - @requires_utime_fd - def test_futimes_subsecond(self): - def set_time(filename, atime, mtime): - with open(filename, "wb") as f: - os.utime(f.fileno(), times=(atime, mtime)) - self._test_utime_subsecond(set_time) - - @requires_utime_fd - def test_futimens_subsecond(self): - def set_time(filename, atime, mtime): - with open(filename, "wb") as f: - os.utime(f.fileno(), times=(atime, mtime)) - self._test_utime_subsecond(set_time) - - @requires_utime_dir_fd - def test_futimesat_subsecond(self): - def set_time(filename, atime, mtime): - dirname = os.path.dirname(filename) - dirfd = os.open(dirname, os.O_RDONLY) - try: - os.utime(os.path.basename(filename), dir_fd=dirfd, - times=(atime, mtime)) - finally: - os.close(dirfd) - self._test_utime_subsecond(set_time) - - @requires_utime_nofollow_symlinks - def test_lutimes_subsecond(self): - def set_time(filename, atime, mtime): - os.utime(filename, (atime, mtime), follow_symlinks=False) - self._test_utime_subsecond(set_time) - - @requires_utime_dir_fd - def test_utimensat_subsecond(self): - def set_time(filename, atime, mtime): - dirname = os.path.dirname(filename) - dirfd = os.open(dirname, os.O_RDONLY) - try: - os.utime(os.path.basename(filename), dir_fd=dirfd, - times=(atime, mtime)) - finally: - os.close(dirfd) - self._test_utime_subsecond(set_time) - - # Restrict tests to Win32, since there is no guarantee other - # systems support centiseconds - def get_file_system(path): - if sys.platform == 'win32': - root = os.path.splitdrive(os.path.abspath(path))[0] + '\\' - import ctypes - kernel32 = ctypes.windll.kernel32 - buf = ctypes.create_unicode_buffer("", 100) - if kernel32.GetVolumeInformationW(root, None, 0, None, None, None, buf, len(buf)): - return buf.value - - @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") - @unittest.skipUnless(get_file_system(support.TESTFN) == "NTFS", - "requires NTFS") - def test_1565150(self): - t1 = 1159195039.25 - os.utime(self.fname, (t1, t1)) - self.assertEqual(os.stat(self.fname).st_mtime, t1) - - @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") - @unittest.skipUnless(get_file_system(support.TESTFN) == "NTFS", - "requires NTFS") - def test_large_time(self): - t1 = 5000000000 # some day in 2128 - os.utime(self.fname, (t1, t1)) - self.assertEqual(os.stat(self.fname).st_mtime, t1) - @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") def test_1686475(self): # Verify that an open file can be stat'ed @@ -596,12 +409,206 @@ class StatAttributeTests(unittest.TestCase): 0) # test directory st_file_attributes (FILE_ATTRIBUTE_DIRECTORY set) - result = os.stat(support.TESTFN) + dirname = support.TESTFN + "dir" + os.mkdir(dirname) + self.addCleanup(os.rmdir, dirname) + + result = os.stat(dirname) self.check_file_attributes(result) self.assertEqual( result.st_file_attributes & stat.FILE_ATTRIBUTE_DIRECTORY, stat.FILE_ATTRIBUTE_DIRECTORY) + +class UtimeTests(unittest.TestCase): + def setUp(self): + self.dirname = support.TESTFN + self.fname = os.path.join(self.dirname, "f1") + + self.addCleanup(support.rmtree, self.dirname) + os.mkdir(self.dirname) + with open(self.fname, 'wb') as fp: + fp.write(b"ABC") + + # ensure that st_atime and st_mtime are float + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + + old_state = os.stat_float_times(-1) + self.addCleanup(os.stat_float_times, old_state) + + os.stat_float_times(True) + + def support_subsecond(self, filename): + # Heuristic to check if the filesystem supports timestamp with + # subsecond resolution: check if float and int timestamps are different + st = os.stat(filename) + return ((st.st_atime != st[7]) + or (st.st_mtime != st[8]) + or (st.st_ctime != st[9])) + + def _test_utime(self, set_time, filename=None): + if not filename: + filename = self.fname + + support_subsecond = self.support_subsecond(filename) + if support_subsecond: + # Timestamp with a resolution of 1 microsecond (10^-6). + # + # The resolution of the C internal function used by os.utime() + # depends on the platform: 1 sec, 1 us, 1 ns. Writing a portable + # test with a resolution of 1 ns requires more work: + # see the issue #15745. + atime_ns = 1002003000 # 1.002003 seconds + mtime_ns = 4005006000 # 4.005006 seconds + else: + # use a resolution of 1 second + atime_ns = 5 * 10**9 + mtime_ns = 8 * 10**9 + + set_time(filename, (atime_ns, mtime_ns)) + st = os.stat(filename) + + if support_subsecond: + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) + self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) + else: + self.assertEqual(st.st_atime, atime_ns * 1e-9) + self.assertEqual(st.st_mtime, mtime_ns * 1e-9) + self.assertEqual(st.st_atime_ns, atime_ns) + self.assertEqual(st.st_mtime_ns, mtime_ns) + + def test_utime(self): + def set_time(filename, ns): + # test the ns keyword parameter + os.utime(filename, ns=ns) + self._test_utime(set_time) + + @staticmethod + def ns_to_sec(ns): + # Convert a number of nanosecond (int) to a number of seconds (float). + # Round towards infinity by adding 0.5 nanosecond to avoid rounding + # issue, os.utime() rounds towards minus infinity. + return (ns * 1e-9) + 0.5e-9 + + def test_utime_by_indexed(self): + # pass times as floating point seconds as the second indexed parameter + def set_time(filename, ns): + atime_ns, mtime_ns = ns + atime = self.ns_to_sec(atime_ns) + mtime = self.ns_to_sec(mtime_ns) + # test utimensat(timespec), utimes(timeval), utime(utimbuf) + # or utime(time_t) + os.utime(filename, (atime, mtime)) + self._test_utime(set_time) + + def test_utime_by_times(self): + def set_time(filename, ns): + atime_ns, mtime_ns = ns + atime = self.ns_to_sec(atime_ns) + mtime = self.ns_to_sec(mtime_ns) + # test the times keyword parameter + os.utime(filename, times=(atime, mtime)) + self._test_utime(set_time) + + @unittest.skipUnless(os.utime in os.supports_follow_symlinks, + "follow_symlinks support for utime required " + "for this test.") + def test_utime_nofollow_symlinks(self): + def set_time(filename, ns): + # use follow_symlinks=False to test utimensat(timespec) + # or lutimes(timeval) + os.utime(filename, ns=ns, follow_symlinks=False) + self._test_utime(set_time) + + @unittest.skipUnless(os.utime in os.supports_fd, + "fd support for utime required for this test.") + def test_utime_fd(self): + def set_time(filename, ns): + with open(filename, 'wb') as fp: + # use a file descriptor to test futimens(timespec) + # or futimes(timeval) + os.utime(fp.fileno(), ns=ns) + self._test_utime(set_time) + + @unittest.skipUnless(os.utime in os.supports_dir_fd, + "dir_fd support for utime required for this test.") + def test_utime_dir_fd(self): + def set_time(filename, ns): + dirname, name = os.path.split(filename) + dirfd = os.open(dirname, os.O_RDONLY) + try: + # pass dir_fd to test utimensat(timespec) or futimesat(timeval) + os.utime(name, dir_fd=dirfd, ns=ns) + finally: + os.close(dirfd) + self._test_utime(set_time) + + def test_utime_directory(self): + def set_time(filename, ns): + # test calling os.utime() on a directory + os.utime(filename, ns=ns) + self._test_utime(set_time, filename=self.dirname) + + def _test_utime_current(self, set_time): + # Get the system clock + current = time.time() + + # Call os.utime() to set the timestamp to the current system clock + set_time(self.fname) + + if not self.support_subsecond(self.fname): + delta = 1.0 + else: + # On Windows, the usual resolution of time.time() is 15.6 ms + delta = 0.020 + st = os.stat(self.fname) + msg = ("st_time=%r, current=%r, dt=%r" + % (st.st_mtime, current, st.st_mtime - current)) + self.assertAlmostEqual(st.st_mtime, current, + delta=delta, msg=msg) + + def test_utime_current(self): + def set_time(filename): + # Set to the current time in the new way + os.utime(self.fname) + self._test_utime_current(set_time) + + def test_utime_current_old(self): + def set_time(filename): + # Set to the current time in the old explicit way. + os.utime(self.fname, None) + self._test_utime_current(set_time) + + def get_file_system(self, path): + if sys.platform == 'win32': + root = os.path.splitdrive(os.path.abspath(path))[0] + '\\' + import ctypes + kernel32 = ctypes.windll.kernel32 + buf = ctypes.create_unicode_buffer("", 100) + ok = kernel32.GetVolumeInformationW(root, None, 0, + None, None, None, + buf, len(buf)) + if ok: + return buf.value + # return None if the filesystem is unknown + + def test_large_time(self): + # Many filesystems are limited to the year 2038. At least, the test + # pass with NTFS filesystem. + if self.get_file_system(self.dirname) != "NTFS": + self.skipTest("requires NTFS") + + large = 5000000000 # some day in 2128 + os.utime(self.fname, (large, large)) + self.assertEqual(os.stat(self.fname).st_mtime, large) + + def test_utime_invalid_arguments(self): + # seconds and nanoseconds parameters are mutually exclusive + with self.assertRaises(ValueError): + os.utime(self.fname, (5, 5), ns=(5, 5)) + + from test import mapping_tests class EnvironTests(mapping_tests.BasicTestMappingProtocol): -- cgit v1.2.1 From 692f1972c67a782e3ed0bec76cd573feb3f535b9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 12 Jun 2015 21:57:50 +0200 Subject: Remove unused import on test_os --- Lib/test/test_os.py | 1 - 1 file changed, 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 987b4af8de..fd8269d10f 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -12,7 +12,6 @@ import fractions import getpass import itertools import locale -import math import mmap import os import pickle -- cgit v1.2.1 From 22341f49d37941bb47451f990566f9f7a74eaeec Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 21 Jun 2015 14:06:55 +0300 Subject: Issue #24426: Fast searching optimization in regular expressions now works for patterns that starts with capturing groups. Fast searching optimization now can't be disabled at compile time. --- Lib/sre_compile.py | 117 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 68 insertions(+), 49 deletions(-) (limited to 'Lib') diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py index 502b0616c6..4edb03fa30 100644 --- a/Lib/sre_compile.py +++ b/Lib/sre_compile.py @@ -409,57 +409,39 @@ def _generate_overlap_table(prefix): table[i] = idx + 1 return table -def _compile_info(code, pattern, flags): - # internal: compile an info block. in the current version, - # this contains min/max pattern width, and an optional literal - # prefix or a character map - lo, hi = pattern.getwidth() - if hi > MAXCODE: - hi = MAXCODE - if lo == 0: - code.extend([INFO, 4, 0, lo, hi]) - return - # look for a literal prefix +def _get_literal_prefix(pattern): + # look for literal prefix prefix = [] prefixappend = prefix.append - prefix_skip = 0 + prefix_skip = None + got_all = True + for op, av in pattern.data: + if op is LITERAL: + prefixappend(av) + elif op is SUBPATTERN: + prefix1, prefix_skip1, got_all = _get_literal_prefix(av[1]) + if prefix_skip is None: + if av[0] is not None: + prefix_skip = len(prefix) + elif prefix_skip1 is not None: + prefix_skip = len(prefix) + prefix_skip1 + prefix.extend(prefix1) + if not got_all: + break + else: + got_all = False + break + return prefix, prefix_skip, got_all + +def _get_charset_prefix(pattern): charset = [] # not used charsetappend = charset.append - if not (flags & SRE_FLAG_IGNORECASE): - # look for literal prefix - for op, av in pattern.data: + if pattern.data: + op, av = pattern.data[0] + if op is SUBPATTERN and av[1]: + op, av = av[1][0] if op is LITERAL: - if len(prefix) == prefix_skip: - prefix_skip = prefix_skip + 1 - prefixappend(av) - elif op is SUBPATTERN and len(av[1]) == 1: - op, av = av[1][0] - if op is LITERAL: - prefixappend(av) - else: - break - else: - break - # if no prefix, look for charset prefix - if not prefix and pattern.data: - op, av = pattern.data[0] - if op is SUBPATTERN and av[1]: - op, av = av[1][0] - if op is LITERAL: - charsetappend((op, av)) - elif op is BRANCH: - c = [] - cappend = c.append - for p in av[1]: - if not p: - break - op, av = p[0] - if op is LITERAL: - cappend((op, av)) - else: - break - else: - charset = c + charsetappend((op, av)) elif op is BRANCH: c = [] cappend = c.append @@ -473,8 +455,43 @@ def _compile_info(code, pattern, flags): break else: charset = c - elif op is IN: - charset = av + elif op is BRANCH: + c = [] + cappend = c.append + for p in av[1]: + if not p: + break + op, av = p[0] + if op is LITERAL: + cappend((op, av)) + else: + break + else: + charset = c + elif op is IN: + charset = av + return charset + +def _compile_info(code, pattern, flags): + # internal: compile an info block. in the current version, + # this contains min/max pattern width, and an optional literal + # prefix or a character map + lo, hi = pattern.getwidth() + if hi > MAXCODE: + hi = MAXCODE + if lo == 0: + code.extend([INFO, 4, 0, lo, hi]) + return + # look for a literal prefix + prefix = [] + prefix_skip = 0 + charset = [] # not used + if not (flags & SRE_FLAG_IGNORECASE): + # look for literal prefix + prefix, prefix_skip, got_all = _get_literal_prefix(pattern) + # if no prefix, look for charset prefix + if not prefix: + charset = _get_charset_prefix(pattern) ## if prefix: ## print("*** PREFIX", prefix, prefix_skip) ## if charset: @@ -487,7 +504,7 @@ def _compile_info(code, pattern, flags): mask = 0 if prefix: mask = SRE_INFO_PREFIX - if len(prefix) == prefix_skip == len(pattern.data): + if prefix_skip is None and got_all: mask = mask | SRE_INFO_LITERAL elif charset: mask = mask | SRE_INFO_CHARSET @@ -502,6 +519,8 @@ def _compile_info(code, pattern, flags): # add literal prefix if prefix: emit(len(prefix)) # length + if prefix_skip is None: + prefix_skip = len(prefix) emit(prefix_skip) # skip code.extend(prefix) # generate overlap table -- cgit v1.2.1 From 96c397fe9255d9f36f12e73c61279fd687dd9593 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 14 Jul 2015 13:51:40 +1200 Subject: Issue #23661: unittest.mock side_effects can now be exceptions again. This was a regression vs Python 3.4. Patch from Ignacio Rossi --- Lib/unittest/mock.py | 3 ++- Lib/unittest/test/testmock/testmock.py | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index c3ab4e82e8..191a175a41 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -506,7 +506,8 @@ class NonCallableMock(Base): if delegated is None: return self._mock_side_effect sf = delegated.side_effect - if sf is not None and not callable(sf) and not isinstance(sf, _MockIter): + if (sf is not None and not callable(sf) + and not isinstance(sf, _MockIter) and not _is_exception(sf)): sf = _MockIter(sf) delegated.side_effect = sf return sf diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py index 3a104cb2b9..f4a723d163 100644 --- a/Lib/unittest/test/testmock/testmock.py +++ b/Lib/unittest/test/testmock/testmock.py @@ -173,6 +173,15 @@ class MockTest(unittest.TestCase): self.assertEqual([mock(), mock(), mock()], [3, 2, 1], "callable side effect not used correctly") + def test_autospec_side_effect_exception(self): + # Test for issue 23661 + def f(): + pass + + mock = create_autospec(f) + mock.side_effect = ValueError('Bazinga!') + self.assertRaisesRegex(ValueError, 'Bazinga!', mock) + @unittest.skipUnless('java' in sys.platform, 'This test only applies to Jython') def test_java_exception_side_effect(self): -- cgit v1.2.1 From d93b47d44962ab87d1124e41b79c8b34f7242567 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Tue, 21 Jul 2015 00:54:19 -0700 Subject: Close issue6549: minor ttk.Style fixes --- Lib/tkinter/ttk.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index b9c57ad704..bad9596d12 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -381,7 +381,9 @@ class Style(object): a sequence identifying the value for that option.""" if query_opt is not None: kw[query_opt] = None - return _val_or_dict(self.tk, kw, self._name, "configure", style) + result = _val_or_dict(self.tk, kw, self._name, "configure", style) + if result or query_opt: + return result def map(self, style, query_opt=None, **kw): @@ -466,12 +468,14 @@ class Style(object): def element_names(self): """Returns the list of elements defined in the current theme.""" - return self.tk.splitlist(self.tk.call(self._name, "element", "names")) + return tuple(n.lstrip('-') for n in self.tk.splitlist( + self.tk.call(self._name, "element", "names"))) def element_options(self, elementname): """Return the list of elementname's options.""" - return self.tk.splitlist(self.tk.call(self._name, "element", "options", elementname)) + return tuple(o.lstrip('-') for o in self.tk.splitlist( + self.tk.call(self._name, "element", "options", elementname))) def theme_create(self, themename, parent=None, settings=None): -- cgit v1.2.1 From 2041f17ffeeb3d30ed620167148e83d074c6cfe5 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 23 Jul 2015 02:57:56 +1200 Subject: Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. --- Lib/test/test_zipimport.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index a97a7784bd..4f1953515d 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -214,7 +214,8 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): packdir2 = packdir + TESTPACK2 + os.sep files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), + "spam" + pyc_ext: (NOW, test_pyc)} z = ZipFile(TEMP_ZIP, "w") try: @@ -228,6 +229,14 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): zi = zipimport.zipimporter(TEMP_ZIP) self.assertEqual(zi.archive, TEMP_ZIP) self.assertEqual(zi.is_package(TESTPACK), True) + + find_mod = zi.find_module('spam') + self.assertIsNotNone(find_mod) + self.assertIsInstance(find_mod, zipimport.zipimporter) + self.assertFalse(find_mod.is_package('spam')) + load_mod = find_mod.load_module('spam') + self.assertEqual(find_mod.get_filename('spam'), load_mod.__file__) + mod = zi.load_module(TESTPACK) self.assertEqual(zi.get_filename(TESTPACK), mod.__file__) @@ -287,6 +296,16 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): self.assertEqual( zi.is_package(TESTPACK2 + os.sep + TESTMOD), False) + pkg_path = TEMP_ZIP + os.sep + packdir + TESTPACK2 + zi2 = zipimport.zipimporter(pkg_path) + find_mod_dotted = zi2.find_module(TESTMOD) + self.assertIsNotNone(find_mod_dotted) + self.assertIsInstance(find_mod_dotted, zipimport.zipimporter) + self.assertFalse(zi2.is_package(TESTMOD)) + load_mod = find_mod_dotted.load_module(TESTMOD) + self.assertEqual( + find_mod_dotted.get_filename(TESTMOD), load_mod.__file__) + mod_path = TESTPACK2 + os.sep + TESTMOD mod_name = module_path_to_dotted_name(mod_path) __import__(mod_name) -- cgit v1.2.1 From 520d790eda9fa446dc24d0f81903e6ff785b7999 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 23 Jul 2015 06:19:18 +1200 Subject: Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. --- Lib/lib2to3/fixes/fix_types.py | 2 +- Lib/lib2to3/tests/test_fixers.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/lib2to3/fixes/fix_types.py b/Lib/lib2to3/fixes/fix_types.py index db34104785..00327a78e8 100644 --- a/Lib/lib2to3/fixes/fix_types.py +++ b/Lib/lib2to3/fixes/fix_types.py @@ -42,7 +42,7 @@ _TYPE_MAPPING = { 'NotImplementedType' : 'type(NotImplemented)', 'SliceType' : 'slice', 'StringType': 'bytes', # XXX ? - 'StringTypes' : 'str', # XXX ? + 'StringTypes' : '(str,)', # XXX ? 'TupleType': 'tuple', 'TypeType' : 'type', 'UnicodeType': 'str', diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index 06b0033b8f..def9b0e47a 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -3322,6 +3322,10 @@ class Test_types(FixerTestCase): a = """type(None)""" self.check(b, a) + b = "types.StringTypes" + a = "(str,)" + self.check(b, a) + class Test_idioms(FixerTestCase): fixer = "idioms" -- cgit v1.2.1 From 19c25730809ee0430ee5e3b10b1e57e7e42f9896 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 23 Jul 2015 17:36:02 +0300 Subject: Issue #13248: Remove inspect.getargspec from 3.6 (deprecated from 3.0) --- Lib/inspect.py | 25 ------------------------- Lib/test/test_inspect.py | 40 +++++----------------------------------- 2 files changed, 5 insertions(+), 60 deletions(-) (limited to 'Lib') diff --git a/Lib/inspect.py b/Lib/inspect.py index 24c8df7225..4e78d80cf5 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1002,31 +1002,6 @@ def _getfullargs(co): varkw = co.co_varnames[nargs] return args, varargs, kwonlyargs, varkw - -ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') - -def getargspec(func): - """Get the names and default values of a function's arguments. - - A tuple of four things is returned: (args, varargs, keywords, defaults). - 'args' is a list of the argument names, including keyword-only argument names. - 'varargs' and 'keywords' are the names of the * and ** arguments or None. - 'defaults' is an n-tuple of the default values of the last n arguments. - - Use the getfullargspec() API for Python 3 code, as annotations - and keyword arguments are supported. getargspec() will raise ValueError - if the func has either annotations or keyword arguments. - """ - warnings.warn("inspect.getargspec() is deprecated, " - "use inspect.signature() instead", DeprecationWarning, - stacklevel=2) - args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ - getfullargspec(func) - if kwonlyargs or ann: - raise ValueError("Function has keyword-only arguments or annotations" - ", use getfullargspec() API which can support them") - return ArgSpec(args, varargs, varkw, defaults) - FullArgSpec = namedtuple('FullArgSpec', 'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations') diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 042617b60d..bd0605e6c3 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -629,18 +629,6 @@ class TestClassesAndFunctions(unittest.TestCase): got = inspect.getmro(D) self.assertEqual(expected, got) - def assertArgSpecEquals(self, routine, args_e, varargs_e=None, - varkw_e=None, defaults_e=None, formatted=None): - with self.assertWarns(DeprecationWarning): - args, varargs, varkw, defaults = inspect.getargspec(routine) - self.assertEqual(args, args_e) - self.assertEqual(varargs, varargs_e) - self.assertEqual(varkw, varkw_e) - self.assertEqual(defaults, defaults_e) - if formatted is not None: - self.assertEqual(inspect.formatargspec(args, varargs, varkw, defaults), - formatted) - def assertFullArgSpecEquals(self, routine, args_e, varargs_e=None, varkw_e=None, defaults_e=None, kwonlyargs_e=[], kwonlydefaults_e=None, @@ -659,23 +647,6 @@ class TestClassesAndFunctions(unittest.TestCase): kwonlyargs, kwonlydefaults, ann), formatted) - def test_getargspec(self): - self.assertArgSpecEquals(mod.eggs, ['x', 'y'], formatted='(x, y)') - - self.assertArgSpecEquals(mod.spam, - ['a', 'b', 'c', 'd', 'e', 'f'], - 'g', 'h', (3, 4, 5), - '(a, b, c, d=3, e=4, f=5, *g, **h)') - - self.assertRaises(ValueError, self.assertArgSpecEquals, - mod2.keyworded, []) - - self.assertRaises(ValueError, self.assertArgSpecEquals, - mod2.annotated, []) - self.assertRaises(ValueError, self.assertArgSpecEquals, - mod2.keyword_only_arg, []) - - def test_getfullargspec(self): self.assertFullArgSpecEquals(mod2.keyworded, [], varargs_e='arg1', kwonlyargs_e=['arg2'], @@ -689,20 +660,19 @@ class TestClassesAndFunctions(unittest.TestCase): kwonlyargs_e=['arg'], formatted='(*, arg)') - def test_argspec_api_ignores_wrapped(self): + def test_fullargspec_api_ignores_wrapped(self): # Issue 20684: low level introspection API must ignore __wrapped__ @functools.wraps(mod.spam) def ham(x, y): pass # Basic check - self.assertArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)') self.assertFullArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)') self.assertFullArgSpecEquals(functools.partial(ham), ['x', 'y'], formatted='(x, y)') # Other variants def check_method(f): - self.assertArgSpecEquals(f, ['self', 'x', 'y'], - formatted='(self, x, y)') + self.assertFullArgSpecEquals(f, ['self', 'x', 'y'], + formatted='(self, x, y)') class C: @functools.wraps(mod.spam) def ham(self, x, y): @@ -780,11 +750,11 @@ class TestClassesAndFunctions(unittest.TestCase): with self.assertRaises(TypeError): inspect.getfullargspec(builtin) - def test_getargspec_method(self): + def test_getfullargspec_method(self): class A(object): def m(self): pass - self.assertArgSpecEquals(A.m, ['self']) + self.assertFullArgSpecEquals(A.m, ['self']) def test_classify_newstyle(self): class A(object): -- cgit v1.2.1 From ae3283e00f7e55127ea01afa0bbb104999657085 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 23 Jul 2015 17:49:00 +0300 Subject: Issue #13248: Remove inspect.getmoduleinfo() from 3.6 (deprecated in 3.3) --- Lib/inspect.py | 17 ----------------- 1 file changed, 17 deletions(-) (limited to 'Lib') diff --git a/Lib/inspect.py b/Lib/inspect.py index 4e78d80cf5..305aafe6ed 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -623,23 +623,6 @@ def getfile(object): raise TypeError('{!r} is not a module, class, method, ' 'function, traceback, frame, or code object'.format(object)) -ModuleInfo = namedtuple('ModuleInfo', 'name suffix mode module_type') - -def getmoduleinfo(path): - """Get the module name, suffix, mode, and module type for a given file.""" - warnings.warn('inspect.getmoduleinfo() is deprecated', DeprecationWarning, - 2) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', PendingDeprecationWarning) - import imp - filename = os.path.basename(path) - suffixes = [(-len(suffix), suffix, mode, mtype) - for suffix, mode, mtype in imp.get_suffixes()] - suffixes.sort() # try longest suffixes first, in case they overlap - for neglen, suffix, mode, mtype in suffixes: - if filename[neglen:] == suffix: - return ModuleInfo(filename[:neglen], suffix, mode, mtype) - def getmodulename(path): """Return the module name for a given file, or None.""" fname = os.path.basename(path) -- cgit v1.2.1 From e24755b4428db490e064d9c4ea570284e7812b5f Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Sun, 26 Jul 2015 06:43:13 +1200 Subject: - Issue #2091: error correctly on open() with mode 'U' and '+' open() accepted a 'U' mode string containing '+', but 'U' can only be used with 'r'. Patch from Jeff Balogh and John O'Connor. --- Lib/_pyio.py | 4 ++-- Lib/test/test_file.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 50ad9ff996..33d8a3f8e0 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -181,8 +181,8 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, text = "t" in modes binary = "b" in modes if "U" in modes: - if creating or writing or appending: - raise ValueError("can't use U and writing mode at once") + if creating or writing or appending or updating: + raise ValueError("mode U cannot be combined with 'x', 'w', 'a', or '+'") import warnings warnings.warn("'U' mode is deprecated", DeprecationWarning, 2) diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py index d54e976143..94f189a4dc 100644 --- a/Lib/test/test_file.py +++ b/Lib/test/test_file.py @@ -139,7 +139,7 @@ class OtherFileTests: def testModeStrings(self): # check invalid mode strings - for mode in ("", "aU", "wU+"): + for mode in ("", "aU", "wU+", "U+", "+U", "rU+"): try: f = self.open(TESTFN, mode) except ValueError: -- cgit v1.2.1 From a52703d41cc0a0353254f90d88246977c4298ce5 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Sun, 26 Jul 2015 06:50:51 +1200 Subject: Issue #24710: Use cls in TracebackException.from_exception. Minor cleanup patch from Berker Peksag. --- Lib/traceback.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/traceback.py b/Lib/traceback.py index 02edeb6217..3d2e5e0685 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -477,10 +477,9 @@ class TracebackException: self._load_lines() @classmethod - def from_exception(self, exc, *args, **kwargs): + def from_exception(cls, exc, *args, **kwargs): """Create a TracebackException from an exception.""" - return TracebackException( - type(exc), exc, exc.__traceback__, *args, **kwargs) + return cls(type(exc), exc, exc.__traceback__, *args, **kwargs) def _load_lines(self): """Private API. force all lines in the stack to be loaded.""" -- cgit v1.2.1 From 29c5460b7153a6b0bcbd78da089c093fde520d89 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 26 Jul 2015 13:11:49 +0200 Subject: Closes #20544: use specific asserts in operator tests. Patch by Serhiy. --- Lib/test/test_operator.py | 50 +++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index da9c8ef34f..54fd1f4e52 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -120,63 +120,63 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.add) self.assertRaises(TypeError, operator.add, None, None) - self.assertTrue(operator.add(3, 4) == 7) + self.assertEqual(operator.add(3, 4), 7) def test_bitwise_and(self): operator = self.module self.assertRaises(TypeError, operator.and_) self.assertRaises(TypeError, operator.and_, None, None) - self.assertTrue(operator.and_(0xf, 0xa) == 0xa) + self.assertEqual(operator.and_(0xf, 0xa), 0xa) def test_concat(self): operator = self.module self.assertRaises(TypeError, operator.concat) self.assertRaises(TypeError, operator.concat, None, None) - self.assertTrue(operator.concat('py', 'thon') == 'python') - self.assertTrue(operator.concat([1, 2], [3, 4]) == [1, 2, 3, 4]) - self.assertTrue(operator.concat(Seq1([5, 6]), Seq1([7])) == [5, 6, 7]) - self.assertTrue(operator.concat(Seq2([5, 6]), Seq2([7])) == [5, 6, 7]) + self.assertEqual(operator.concat('py', 'thon'), 'python') + self.assertEqual(operator.concat([1, 2], [3, 4]), [1, 2, 3, 4]) + self.assertEqual(operator.concat(Seq1([5, 6]), Seq1([7])), [5, 6, 7]) + self.assertEqual(operator.concat(Seq2([5, 6]), Seq2([7])), [5, 6, 7]) self.assertRaises(TypeError, operator.concat, 13, 29) def test_countOf(self): operator = self.module self.assertRaises(TypeError, operator.countOf) self.assertRaises(TypeError, operator.countOf, None, None) - self.assertTrue(operator.countOf([1, 2, 1, 3, 1, 4], 3) == 1) - self.assertTrue(operator.countOf([1, 2, 1, 3, 1, 4], 5) == 0) + self.assertEqual(operator.countOf([1, 2, 1, 3, 1, 4], 3), 1) + self.assertEqual(operator.countOf([1, 2, 1, 3, 1, 4], 5), 0) def test_delitem(self): operator = self.module a = [4, 3, 2, 1] self.assertRaises(TypeError, operator.delitem, a) self.assertRaises(TypeError, operator.delitem, a, None) - self.assertTrue(operator.delitem(a, 1) is None) - self.assertTrue(a == [4, 2, 1]) + self.assertIsNone(operator.delitem(a, 1)) + self.assertEqual(a, [4, 2, 1]) def test_floordiv(self): operator = self.module self.assertRaises(TypeError, operator.floordiv, 5) self.assertRaises(TypeError, operator.floordiv, None, None) - self.assertTrue(operator.floordiv(5, 2) == 2) + self.assertEqual(operator.floordiv(5, 2), 2) def test_truediv(self): operator = self.module self.assertRaises(TypeError, operator.truediv, 5) self.assertRaises(TypeError, operator.truediv, None, None) - self.assertTrue(operator.truediv(5, 2) == 2.5) + self.assertEqual(operator.truediv(5, 2), 2.5) def test_getitem(self): operator = self.module a = range(10) self.assertRaises(TypeError, operator.getitem) self.assertRaises(TypeError, operator.getitem, a, None) - self.assertTrue(operator.getitem(a, 2) == 2) + self.assertEqual(operator.getitem(a, 2), 2) def test_indexOf(self): operator = self.module self.assertRaises(TypeError, operator.indexOf) self.assertRaises(TypeError, operator.indexOf, None, None) - self.assertTrue(operator.indexOf([4, 3, 2, 1], 3) == 1) + self.assertEqual(operator.indexOf([4, 3, 2, 1], 3), 1) self.assertRaises(ValueError, operator.indexOf, [4, 3, 2, 1], 0) def test_invert(self): @@ -189,21 +189,21 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.lshift) self.assertRaises(TypeError, operator.lshift, None, 42) - self.assertTrue(operator.lshift(5, 1) == 10) - self.assertTrue(operator.lshift(5, 0) == 5) + self.assertEqual(operator.lshift(5, 1), 10) + self.assertEqual(operator.lshift(5, 0), 5) self.assertRaises(ValueError, operator.lshift, 2, -1) def test_mod(self): operator = self.module self.assertRaises(TypeError, operator.mod) self.assertRaises(TypeError, operator.mod, None, 42) - self.assertTrue(operator.mod(5, 2) == 1) + self.assertEqual(operator.mod(5, 2), 1) def test_mul(self): operator = self.module self.assertRaises(TypeError, operator.mul) self.assertRaises(TypeError, operator.mul, None, None) - self.assertTrue(operator.mul(5, 2) == 10) + self.assertEqual(operator.mul(5, 2), 10) def test_matmul(self): operator = self.module @@ -227,7 +227,7 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.or_) self.assertRaises(TypeError, operator.or_, None, None) - self.assertTrue(operator.or_(0xa, 0x5) == 0xf) + self.assertEqual(operator.or_(0xa, 0x5), 0xf) def test_pos(self): operator = self.module @@ -250,8 +250,8 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.rshift) self.assertRaises(TypeError, operator.rshift, None, 42) - self.assertTrue(operator.rshift(5, 1) == 2) - self.assertTrue(operator.rshift(5, 0) == 5) + self.assertEqual(operator.rshift(5, 1), 2) + self.assertEqual(operator.rshift(5, 0), 5) self.assertRaises(ValueError, operator.rshift, 2, -1) def test_contains(self): @@ -266,15 +266,15 @@ class OperatorTestCase: a = list(range(3)) self.assertRaises(TypeError, operator.setitem, a) self.assertRaises(TypeError, operator.setitem, a, None, None) - self.assertTrue(operator.setitem(a, 0, 2) is None) - self.assertTrue(a == [2, 1, 2]) + self.assertIsNone(operator.setitem(a, 0, 2)) + self.assertEqual(a, [2, 1, 2]) self.assertRaises(IndexError, operator.setitem, a, 4, 2) def test_sub(self): operator = self.module self.assertRaises(TypeError, operator.sub) self.assertRaises(TypeError, operator.sub, None, None) - self.assertTrue(operator.sub(5, 2) == 3) + self.assertEqual(operator.sub(5, 2), 3) def test_truth(self): operator = self.module @@ -292,7 +292,7 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.xor) self.assertRaises(TypeError, operator.xor, None, None) - self.assertTrue(operator.xor(0xb, 0xc) == 0x7) + self.assertEqual(operator.xor(0xb, 0xc), 0x7) def test_is(self): operator = self.module -- cgit v1.2.1 From a6d79b5164cac2ee3207a00f7ae440e6b2719e0f Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 28 Jul 2015 15:55:07 +1200 Subject: Issue #23426: run_setup was broken in distutils. Patch from Alexander Belopolsky. --- Lib/distutils/core.py | 5 ++--- Lib/distutils/tests/test_core.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/distutils/core.py b/Lib/distutils/core.py index f05b34b295..d603d4a45a 100644 --- a/Lib/distutils/core.py +++ b/Lib/distutils/core.py @@ -204,16 +204,15 @@ def run_setup (script_name, script_args=None, stop_after="run"): global _setup_stop_after, _setup_distribution _setup_stop_after = stop_after - save_argv = sys.argv + save_argv = sys.argv.copy() g = {'__file__': script_name} - l = {} try: try: sys.argv[0] = script_name if script_args is not None: sys.argv[1:] = script_args with open(script_name, 'rb') as f: - exec(f.read(), g, l) + exec(f.read(), g) finally: sys.argv = save_argv _setup_stop_after = None diff --git a/Lib/distutils/tests/test_core.py b/Lib/distutils/tests/test_core.py index 41321f7db4..57856f19b6 100644 --- a/Lib/distutils/tests/test_core.py +++ b/Lib/distutils/tests/test_core.py @@ -28,6 +28,21 @@ from distutils.core import setup setup() """ +setup_does_nothing = """\ +from distutils.core import setup +setup() +""" + + +setup_defines_subclass = """\ +from distutils.core import setup +from distutils.command.install import install as _install + +class install(_install): + sub_commands = _install.sub_commands + ['cmd'] + +setup(cmdclass={'install': install}) +""" class CoreTestCase(support.EnvironGuard, unittest.TestCase): @@ -65,6 +80,21 @@ class CoreTestCase(support.EnvironGuard, unittest.TestCase): distutils.core.run_setup( self.write_setup(setup_using___file__)) + def test_run_setup_preserves_sys_argv(self): + # Make sure run_setup does not clobber sys.argv + argv_copy = sys.argv.copy() + distutils.core.run_setup( + self.write_setup(setup_does_nothing)) + self.assertEqual(sys.argv, argv_copy) + + def test_run_setup_defines_subclass(self): + # Make sure the script can use __file__; if that's missing, the test + # setup.py script will raise NameError. + dist = distutils.core.run_setup( + self.write_setup(setup_defines_subclass)) + install = dist.get_command_obj('install') + self.assertIn('cmd', install.sub_commands) + def test_run_setup_uses_current_dir(self): # This tests that the setup script is run with the current directory # as its own current directory; this was temporarily broken by a -- cgit v1.2.1 From 84fff856414eee58532ac19848c1ddff21207153 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 29 Jul 2015 23:51:47 +0300 Subject: Issue #24360: Improve __repr__ of argparse.Namespace() for invalid identifiers. Patch by Matthias Bussonnier. --- Lib/argparse.py | 8 +++++++- Lib/test/test_argparse.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/argparse.py b/Lib/argparse.py index 9a067196da..cc538415d2 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -118,10 +118,16 @@ class _AttributeHolder(object): def __repr__(self): type_name = type(self).__name__ arg_strings = [] + star_args = {} for arg in self._get_args(): arg_strings.append(repr(arg)) for name, value in self._get_kwargs(): - arg_strings.append('%s=%r' % (name, value)) + if name.isidentifier(): + arg_strings.append('%s=%r' % (name, value)) + else: + star_args[name] = value + if star_args: + arg_strings.append('**%s' % repr(star_args)) return '%s(%s)' % (type_name, ', '.join(arg_strings)) def _get_kwargs(self): diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 27bfad5fb6..893ec394f6 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -4512,6 +4512,21 @@ class TestStrings(TestCase): string = "Namespace(bar='spam', foo=42)" self.assertStringEqual(ns, string) + def test_namespace_starkwargs_notidentifier(self): + ns = argparse.Namespace(**{'"': 'quote'}) + string = """Namespace(**{'"': 'quote'})""" + self.assertStringEqual(ns, string) + + def test_namespace_kwargs_and_starkwargs_notidentifier(self): + ns = argparse.Namespace(a=1, **{'"': 'quote'}) + string = """Namespace(a=1, **{'"': 'quote'})""" + self.assertStringEqual(ns, string) + + def test_namespace_starkwargs_identifier(self): + ns = argparse.Namespace(**{'valid': True}) + string = "Namespace(valid=True)" + self.assertStringEqual(ns, string) + def test_parser(self): parser = argparse.ArgumentParser(prog='PROG') string = ( -- cgit v1.2.1 From ee73a7753ee9d543261b6c28a573a31ffc9bddda Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 31 Jul 2015 04:11:29 +0300 Subject: Issue #13248: Delete remaining references of inspect.getargspec(). Noticed by Yaroslav Halchenko. --- Lib/inspect.py | 7 ++----- Lib/test/test_inspect.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) (limited to 'Lib') diff --git a/Lib/inspect.py b/Lib/inspect.py index 1e94a9c539..a089be696e 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -16,7 +16,7 @@ Here are some of the useful functions provided by this module: getmodule() - determine the module that an object came from getclasstree() - arrange classes so as to represent their hierarchy - getargspec(), getargvalues(), getcallargs() - get info about function arguments + getargvalues(), getcallargs() - get info about function arguments getfullargspec() - same, with support for Python 3 features formatargspec(), formatargvalues() - format an argument spec getouterframes(), getinnerframes() - get info about frames @@ -1018,8 +1018,6 @@ def getfullargspec(func): 'kwonlydefaults' is a dictionary mapping names from kwonlyargs to defaults. 'annotations' is a dictionary mapping argument names to annotations. - The first four items in the tuple correspond to getargspec(). - This function is deprecated, use inspect.signature() instead. """ @@ -1130,8 +1128,7 @@ def formatargspec(args, varargs=None, varkw=None, defaults=None, formatvalue=lambda value: '=' + repr(value), formatreturns=lambda text: ' -> ' + text, formatannotation=formatannotation): - """Format an argument spec from the values returned by getargspec - or getfullargspec. + """Format an argument spec from the values returned by getfullargspec. The first seven arguments are (args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations). The other five arguments diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 309ee2a797..db15b39d39 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -38,7 +38,7 @@ from test.test_import import _ready_to_import # ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode, # isbuiltin, isroutine, isgenerator, isgeneratorfunction, getmembers, # getdoc, getfile, getmodule, getsourcefile, getcomments, getsource, -# getclasstree, getargspec, getargvalues, formatargspec, formatargvalues, +# getclasstree, getargvalues, formatargspec, formatargvalues, # currentframe, stack, trace, isdatadescriptor # NOTE: There are some additional tests relating to interaction with -- cgit v1.2.1 From 81921fe3575bba724e020e4eeebd2dff19c4da3b Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 9 Aug 2015 00:35:00 -0700 Subject: Add more tests of hash effectiveness. --- Lib/test/test_set.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index 54de508a83..ade39fb758 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -10,6 +10,8 @@ import sys import warnings import collections import collections.abc +import itertools +import string class PassThru(Exception): pass @@ -711,6 +713,28 @@ class TestFrozenSet(TestJointOps, unittest.TestCase): addhashvalue(hash(frozenset([e for e, m in elemmasks if m&i]))) self.assertEqual(len(hashvalues), 2**n) + def letter_range(n): + return string.ascii_letters[:n] + + def zf_range(n): + # https://en.wikipedia.org/wiki/Set-theoretic_definition_of_natural_numbers + nums = [frozenset()] + for i in range(n-1): + num = frozenset(nums) + nums.append(num) + return nums[:n] + + def powerset(s): + for i in range(len(s)+1): + yield from map(frozenset, itertools.combinations(s, i)) + + for n in range(18): + t = 2 ** n + mask = t - 1 + for nums in (range, letter_range, zf_range): + u = len({h & mask for h in map(hash, powerset(nums(n)))}) + self.assertGreater(4*u, t) + class FrozenSetSubclass(frozenset): pass -- cgit v1.2.1 From a8cb41e9ae2c7caf5fbb1025e6c1e6534690fe7f Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Mon, 10 Aug 2015 09:53:30 +1200 Subject: Issue #20059: urllib.parse raises ValueError on all invalid ports. Patch by Martin Panter. --- Lib/test/test_urlparse.py | 36 +++++++++++++++++------------------- Lib/urllib/parse.py | 3 +-- 2 files changed, 18 insertions(+), 21 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 0552f90594..fcf508259e 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -554,29 +554,27 @@ class UrlParseTestCase(unittest.TestCase): self.assertEqual(p.port, 80) self.assertEqual(p.geturl(), url) - # Verify an illegal port is returned as None + # Verify an illegal port raises ValueError url = b"HTTP://WWW.PYTHON.ORG:65536/doc/#frag" p = urllib.parse.urlsplit(url) - self.assertEqual(p.port, None) + with self.assertRaisesRegex(ValueError, "out of range"): + p.port def test_attributes_bad_port(self): - """Check handling of non-integer ports.""" - p = urllib.parse.urlsplit("http://www.example.net:foo") - self.assertEqual(p.netloc, "www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) - - p = urllib.parse.urlparse("http://www.example.net:foo") - self.assertEqual(p.netloc, "www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) - - # Once again, repeat ourselves to test bytes - p = urllib.parse.urlsplit(b"http://www.example.net:foo") - self.assertEqual(p.netloc, b"www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) - - p = urllib.parse.urlparse(b"http://www.example.net:foo") - self.assertEqual(p.netloc, b"www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) + """Check handling of invalid ports.""" + for bytes in (False, True): + for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): + for port in ("foo", "1.5", "-1", "0x10"): + with self.subTest(bytes=bytes, parse=parse, port=port): + netloc = "www.example.net:" + port + url = "http://" + netloc + if bytes: + netloc = netloc.encode("ascii") + url = url.encode("ascii") + p = parse(url) + self.assertEqual(p.netloc, netloc) + with self.assertRaises(ValueError): + p.port def test_attributes_without_netloc(self): # This example is straight from RFC 3261. It looks like it diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 01c9e587fb..5e2155ccaf 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -156,9 +156,8 @@ class _NetlocResultMixinBase(object): port = self._hostinfo[1] if port is not None: port = int(port, 10) - # Return None on an illegal port if not ( 0 <= port <= 65535): - return None + raise ValueError("Port out of range 0-65535") return port -- cgit v1.2.1 From 59040a0452fc468e0cc3c654187806958acc055a Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Wed, 12 Aug 2015 08:00:06 +1200 Subject: Issue #9232: Support trailing commas in function declarations. For example, "def f(*, a = 3,): pass" is now legal. Patch from Mark Dickinson. --- Lib/test/test_grammar.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index ec3d7833f7..8f8d71ce85 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -295,6 +295,10 @@ class GrammarTests(unittest.TestCase): pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200) pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100) + self.assertRaises(SyntaxError, eval, "def f(*): pass") + self.assertRaises(SyntaxError, eval, "def f(*,): pass") + self.assertRaises(SyntaxError, eval, "def f(*, **kwds): pass") + # keyword arguments after *arglist def f(*args, **kwargs): return args, kwargs @@ -352,6 +356,23 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "f(*g(1=2))") check_syntax_error(self, "f(**g(1=2))") + # Check trailing commas are permitted in funcdef argument list + def f(a,): pass + def f(*args,): pass + def f(**kwds,): pass + def f(a, *args,): pass + def f(a, **kwds,): pass + def f(*args, b,): pass + def f(*, b,): pass + def f(*args, **kwds,): pass + def f(a, *args, b,): pass + def f(a, *, b,): pass + def f(a, *args, **kwds,): pass + def f(*args, b, **kwds,): pass + def f(*, b, **kwds,): pass + def f(a, *args, b, **kwds,): pass + def f(a, *, b, **kwds,): pass + def test_lambdef(self): ### lambdef: 'lambda' [varargslist] ':' test l1 = lambda : 0 @@ -370,6 +391,23 @@ class GrammarTests(unittest.TestCase): self.assertEqual(l6(1,2), 1+2+20) self.assertEqual(l6(1,2,k=10), 1+2+10) + # check that trailing commas are permitted + l10 = lambda a,: 0 + l11 = lambda *args,: 0 + l12 = lambda **kwds,: 0 + l13 = lambda a, *args,: 0 + l14 = lambda a, **kwds,: 0 + l15 = lambda *args, b,: 0 + l16 = lambda *, b,: 0 + l17 = lambda *args, **kwds,: 0 + l18 = lambda a, *args, b,: 0 + l19 = lambda a, *, b,: 0 + l20 = lambda a, *args, **kwds,: 0 + l21 = lambda *args, b, **kwds,: 0 + l22 = lambda *, b, **kwds,: 0 + l23 = lambda a, *args, b, **kwds,: 0 + l24 = lambda a, *, b, **kwds,: 0 + ### stmt: simple_stmt | compound_stmt # Tested below -- cgit v1.2.1 From 91ea76bd466bf8853d3b357649b11f7dd046b68b Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 15 Aug 2015 13:51:59 -0700 Subject: Fix crash in itertools.cycle.__setstate__() caused by lack of type checking. Will backport after the 3.6 release is done. --- Lib/test/test_itertools.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index fcd886911d..53d65649ff 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -613,6 +613,39 @@ class TestBasicOps(unittest.TestCase): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, cycle('abc')) + def test_cycle_setstate(self): + # Verify both modes for restoring state + + # Mode 0 is efficient. It uses an incompletely consumed input + # iterator to build a cycle object and then passes in state with + # a list of previously consumed values. There is no data + # overlap bewteen the two. + c = cycle('defg') + c.__setstate__((list('abc'), 0)) + self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) + + # Mode 1 is inefficient. It starts with a cycle object built + # from an iterator over the remaining elements in a partial + # cycle and then passes in state with all of the previously + # seen values (this overlaps values included in the iterator). + c = cycle('defg') + c.__setstate__((list('abcdefg'), 1)) + self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) + + # The first argument to setstate needs to be a tuple + with self.assertRaises(SystemError): + cycle('defg').__setstate__([list('abcdefg'), 0]) + + # The first argument in the setstate tuple must be a list + with self.assertRaises(TypeError): + c = cycle('defg') + c.__setstate__((dict.fromkeys('defg'), 0)) + take(20, c) + + # The first argument in the setstate tuple must be a list + with self.assertRaises(TypeError): + cycle('defg').__setstate__((list('abcdefg'), 'x')) + def test_groupby(self): # Check whether it accepts arguments correctly self.assertEqual([], list(groupby([]))) -- cgit v1.2.1 From ec7b354e27b2241ae42fcf7df43276f1d6d912d5 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 15 Aug 2015 14:45:49 -0700 Subject: Add more tests for pickling itertools.cycle --- Lib/test/test_itertools.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 53d65649ff..c012efd752 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -613,6 +613,23 @@ class TestBasicOps(unittest.TestCase): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, cycle('abc')) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + # test with partial consumed input iterable + it = iter('abcde') + c = cycle(it) + _ = [next(c) for i in range(2)] # consume to 2 of 5 inputs + p = pickle.dumps(c, proto) + d = pickle.loads(p) # rebuild the cycle object + self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + + # test with completely consumed input iterable + it = iter('abcde') + c = cycle(it) + _ = [next(c) for i in range(7)] # consume to 7 of 5 inputs + p = pickle.dumps(c, proto) + d = pickle.loads(p) # rebuild the cycle object + self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + def test_cycle_setstate(self): # Verify both modes for restoring state -- cgit v1.2.1 From c00d6c87853b1a978f7b6e6498c5874873810e49 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 15 Aug 2015 14:47:27 -0700 Subject: Improve comment --- Lib/test/test_itertools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index c012efd752..81e1711d21 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -617,7 +617,7 @@ class TestBasicOps(unittest.TestCase): # test with partial consumed input iterable it = iter('abcde') c = cycle(it) - _ = [next(c) for i in range(2)] # consume to 2 of 5 inputs + _ = [next(c) for i in range(2)] # consume 2 of 5 inputs p = pickle.dumps(c, proto) d = pickle.loads(p) # rebuild the cycle object self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) @@ -625,7 +625,7 @@ class TestBasicOps(unittest.TestCase): # test with completely consumed input iterable it = iter('abcde') c = cycle(it) - _ = [next(c) for i in range(7)] # consume to 7 of 5 inputs + _ = [next(c) for i in range(7)] # consume 7 of 5 inputs p = pickle.dumps(c, proto) d = pickle.loads(p) # rebuild the cycle object self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) -- cgit v1.2.1 From 233eb5a36cca6ccbb2aa507065bf2231b630e836 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Aug 2015 17:08:48 -0700 Subject: Add error information to a failing test to help diagnose a buildbot failure. --- Lib/test/test_symbol.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_symbol.py b/Lib/test/test_symbol.py index 4475bbcac4..c126be3d01 100644 --- a/Lib/test/test_symbol.py +++ b/Lib/test/test_symbol.py @@ -38,7 +38,10 @@ class TestSymbolGeneration(unittest.TestCase): self.assertFalse(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE)) self.assertEqual((0, b''), self._generate_symbols(GRAMMAR_FILE, TEST_PY_FILE)) - self.assertTrue(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE)) + self.assertTrue(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE), + 'symbol stat: %r\ntest_py stat: %r\n' % + (os.stat(SYMBOL_FILE), + os.stat(TEST_PY_FILE))) if __name__ == "__main__": -- cgit v1.2.1 From 43ccf69ddc0dac43b8475dcf7e7c57e482192962 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Aug 2015 19:43:34 -0700 Subject: Issue #24379: Add operator.subscript() as a convenience for building slices. --- Lib/operator.py | 28 +++++++++++++++++++++++++++- Lib/test/test_operator.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/operator.py b/Lib/operator.py index 0e2e53efc6..bc2a9478b8 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -17,7 +17,7 @@ __all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', - 'setitem', 'sub', 'truediv', 'truth', 'xor'] + 'setitem', 'sub', 'subscript', 'truediv', 'truth', 'xor'] from builtins import abs as _abs @@ -408,6 +408,32 @@ def ixor(a, b): return a +@object.__new__ # create a singleton instance +class subscript: + """ + A helper to turn subscript notation into indexing objects. This can be + used to create item access patterns ahead of time to pass them into + various subscriptable objects. + + For example: + subscript[5] == 5 + subscript[3:7:2] == slice(3, 7, 2) + subscript[5, 8] == (5, 8) + """ + __slots__ = () + + def __new__(cls): + raise TypeError("cannot create '{}' instances".format(cls.__name__)) + + @staticmethod + def __getitem__(key): + return key + + @staticmethod + def __reduce__(): + return 'subscript' + + try: from _operator import * except ImportError: diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 54fd1f4e52..27501c2466 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -596,5 +596,38 @@ class CCOperatorPickleTestCase(OperatorPickleTestCase, unittest.TestCase): module2 = c_operator +class SubscriptTestCase: + def test_subscript(self): + subscript = self.module.subscript + self.assertIsNone(subscript[None]) + self.assertEqual(subscript[0], 0) + self.assertEqual(subscript[0:1:2], slice(0, 1, 2)) + self.assertEqual( + subscript[0, ..., :2, ...], + (0, Ellipsis, slice(2), Ellipsis), + ) + + def test_pickle(self): + from operator import subscript + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + self.assertIs( + pickle.loads(pickle.dumps(subscript, proto)), + subscript, + ) + + def test_singleton(self): + with self.assertRaises(TypeError): + type(self.module.subscript)() + + def test_immutable(self): + with self.assertRaises(AttributeError): + self.module.subscript.attr = None + + +class PySubscriptTestCase(SubscriptTestCase, PyOperatorTestCase): + pass + + if __name__ == "__main__": unittest.main() -- cgit v1.2.1 From 987091c333ad1a6be824a843b87816faca8937c4 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 17 Aug 2015 22:04:45 -0700 Subject: Issue #24878: Add docstrings to selected namedtuples --- Lib/aifc.py | 8 ++++++++ Lib/dis.py | 9 +++++++++ Lib/sched.py | 11 +++++++++++ Lib/selectors.py | 12 ++++++++++-- Lib/shutil.py | 3 +++ Lib/sndhdr.py | 12 ++++++++++++ 6 files changed, 53 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/aifc.py b/Lib/aifc.py index 7ebdbeb68c..1556b53019 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -257,6 +257,14 @@ from collections import namedtuple _aifc_params = namedtuple('_aifc_params', 'nchannels sampwidth framerate nframes comptype compname') +_aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)' +_aifc_params.sampwidth.__doc__ = 'Ample width in bytes' +_aifc_params.framerate.__doc__ = 'Sampling frequency' +_aifc_params.nframes.__doc__ = 'Number of audio frames' +_aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)' +_aifc_params.compname.__doc__ = ("""A human-readable version ofcompression type +('not compressed' for AIFF files)'""") + class Aifc_read: # Variables used in this class: diff --git a/Lib/dis.py b/Lib/dis.py index af37cdf0c6..3540b4714d 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -162,6 +162,15 @@ def show_code(co, *, file=None): _Instruction = collections.namedtuple("_Instruction", "opname opcode arg argval argrepr offset starts_line is_jump_target") +_Instruction.opname.__doc__ = "Human readable name for operation" +_Instruction.opcode.__doc__ = "Numeric code for operation" +_Instruction.arg.__doc__ = "Numeric argument to operation (if any), otherwise None" +_Instruction.argval.__doc__ = "Resolved arg value (if known), otherwise same as arg" +_Instruction.argrepr.__doc__ = "Human readable description of operation argument" +_Instruction.offset.__doc__ = "Start index of operation within bytecode sequence" +_Instruction.starts_line.__doc__ = "Line started by this opcode (if any), otherwise None" +_Instruction.is_jump_target.__doc__ = "True if other code jumps to here, otherwise False" + class Instruction(_Instruction): """Details for a bytecode operation diff --git a/Lib/sched.py b/Lib/sched.py index b47648d973..bd7c0f1b6f 100644 --- a/Lib/sched.py +++ b/Lib/sched.py @@ -46,6 +46,17 @@ class Event(namedtuple('Event', 'time, priority, action, argument, kwargs')): def __gt__(s, o): return (s.time, s.priority) > (o.time, o.priority) def __ge__(s, o): return (s.time, s.priority) >= (o.time, o.priority) +Event.time.__doc__ = ('''Numeric type compatible with the return value of the +timefunc function passed to the constructor.''') +Event.priority.__doc__ = ('''Events scheduled for the same time will be executed +in the order of their priority.''') +Event.action.__doc__ = ('''Executing the event means executing +action(*argument, **kwargs)''') +Event.argument.__doc__ = ('''argument is a sequence holding the positional +arguments for the action.''') +Event.kwargs.__doc__ = ('''kwargs is a dictionary holding the keyword +arguments for the action.''') + _sentinel = object() class scheduler: diff --git a/Lib/selectors.py b/Lib/selectors.py index 6d569c30ad..ecd8632d70 100644 --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -43,9 +43,17 @@ def _fileobj_to_fd(fileobj): SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) -"""Object used to associate a file object to its backing file descriptor, -selected event mask and attached data.""" +SelectorKey.__doc__ = """SelectorKey(fileobj, fd, events, data) + + Object used to associate a file object to its backing + file descriptor, selected event mask, and attached data. +""" +SelectorKey.fileobj.__doc__ = 'File object registered.' +SelectorKey.fd.__doc__ = 'Underlying file descriptor.' +SelectorKey.events.__doc__ = 'Events that must be waited for on this file object.' +SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object. +For example, this could be used to store a per-client session ID.''') class _SelectorMapping(Mapping): """Mapping of file objects to selector keys.""" diff --git a/Lib/shutil.py b/Lib/shutil.py index a5da58790e..8edabdb332 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -965,6 +965,9 @@ if hasattr(os, 'statvfs'): __all__.append('disk_usage') _ntuple_diskusage = collections.namedtuple('usage', 'total used free') + _ntuple_diskusage.total.__doc__ = 'Total space in bytes' + _ntuple_diskusage.used.__doc__ = 'Used space in bytes' + _ntuple_diskusage.free.__doc__ = 'Free space in bytes' def disk_usage(path): """Return disk usage statistics about the given path. diff --git a/Lib/sndhdr.py b/Lib/sndhdr.py index e5901ec583..7ecafb40e8 100644 --- a/Lib/sndhdr.py +++ b/Lib/sndhdr.py @@ -37,6 +37,18 @@ from collections import namedtuple SndHeaders = namedtuple('SndHeaders', 'filetype framerate nchannels nframes sampwidth') +SndHeaders.filetype.__doc__ = ("""The value for type indicates the data type +and will be one of the strings 'aifc', 'aiff', 'au','hcom', +'sndr', 'sndt', 'voc', 'wav', '8svx', 'sb', 'ub', or 'ul'.""") +SndHeaders.framerate.__doc__ = ("""The sampling_rate will be either the actual +value or 0 if unknown or difficult to decode.""") +SndHeaders.nchannels.__doc__ = ("""The number of channels or 0 if it cannot be +determined or if the value is difficult to decode.""") +SndHeaders.nframes.__doc__ = ("""The value for frames will be either the number +of frames or -1.""") +SndHeaders.sampwidth.__doc__ = ("""Either the sample size in bits or +'A' for A-LAW or 'U' for u-LAW.""") + def what(filename): """Guess the type of a sound file.""" res = whathdr(filename) -- cgit v1.2.1 From e8a3563e9c434e35fb4e1880cc11dd4af6031ea4 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 18 Aug 2015 00:35:52 -0700 Subject: Add missing docstring --- Lib/urllib/request.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) (limited to 'Lib') diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index a7fd017e10..e6abf34fa0 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -138,6 +138,71 @@ __version__ = sys.version[:3] _opener = None def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, *, cafile=None, capath=None, cadefault=False, context=None): + '''Open the URL url, which can be either a string or a Request object. + + *data* must be a bytes object specifying additional data to be sent to the + server, or None if no such data is needed. data may also be an iterable + object and in that case Content-Length value must be specified in the + headers. Currently HTTP requests are the only ones that use data; the HTTP + request will be a POST instead of a GET when the data parameter is + provided. + + *data* should be a buffer in the standard application/x-www-form-urlencoded + format. The urllib.parse.urlencode() function takes a mapping or sequence + of 2-tuples and returns a string in this format. It should be encoded to + bytes before being used as the data parameter. The charset parameter in + Content-Type header may be used to specify the encoding. If charset + parameter is not sent with the Content-Type header, the server following + the HTTP 1.1 recommendation may assume that the data is encoded in + ISO-8859-1 encoding. It is advisable to use charset parameter with encoding + used in Content-Type header with the Request. + + urllib.request module uses HTTP/1.1 and includes a "Connection:close" + header in its HTTP requests. + + The optional *timeout* parameter specifies a timeout in seconds for + blocking operations like the connection attempt (if not specified, the + global default timeout setting will be used). This only works for HTTP, + HTTPS and FTP connections. + + If *context* is specified, it must be a ssl.SSLContext instance describing + the various SSL options. See HTTPSConnection for more details. + + The optional *cafile* and *capath* parameters specify a set of trusted CA + certificates for HTTPS requests. cafile should point to a single file + containing a bundle of CA certificates, whereas capath should point to a + directory of hashed certificate files. More information can be found in + ssl.SSLContext.load_verify_locations(). + + The *cadefault* parameter is ignored. + + For http and https urls, this function returns a http.client.HTTPResponse + object which has the following HTTPResponse Objects methods. + + For ftp, file, and data urls and requests explicitly handled by legacy + URLopener and FancyURLopener classes, this function returns a + urllib.response.addinfourl object which can work as context manager and has + methods such as: + + * geturl() — return the URL of the resource retrieved, commonly used to + determine if a redirect was followed + + * info() — return the meta-information of the page, such as headers, in the + form of an email.message_from_string() instance (see Quick Reference to + HTTP Headers) + + * getcode() – return the HTTP status code of the response. Raises URLError + on errors. + + Note that *None& may be returned if no handler handles the request (though + the default installed global OpenerDirector uses UnknownHandler to ensure + this never happens). + + In addition, if proxy settings are detected (for example, when a *_proxy + environment variable like http_proxy is set), ProxyHandler is default + installed and makes sure the requests are handled through the proxy. + + ''' global _opener if cafile or capath or cadefault: if context is not None: -- cgit v1.2.1 From 2f5ec38319de63e4dbfbffbc0844ea9c4056fc7f Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Wed, 19 Aug 2015 12:20:37 +1200 Subject: Issue #24054: decouple linecache tests from inspect tests Patch from David D. Riddle --- Lib/test/test_linecache.py | 74 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 11 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 21ef738932..e74aef3580 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -3,6 +3,7 @@ import linecache import unittest import os.path +import tempfile from test import support @@ -10,8 +11,6 @@ FILENAME = linecache.__file__ NONEXISTENT_FILENAME = FILENAME + '.missing' INVALID_NAME = '!@$)(!@#_1' EMPTY = '' -TESTS = 'inspect_fodder inspect_fodder2 mapping_tests' -TESTS = TESTS.split() TEST_PATH = os.path.dirname(__file__) MODULES = "linecache abc".split() MODULE_PATH = os.path.dirname(FILENAME) @@ -37,6 +36,65 @@ def f(): return 3''' # No ending newline +class TempFile: + + def setUp(self): + super().setUp() + with tempfile.NamedTemporaryFile(delete=False) as fp: + self.file_name = fp.name + fp.write(self.file_byte_string) + self.addCleanup(support.unlink, self.file_name) + + +class GetLineTestsGoodData(TempFile): + # file_list = ['list\n', 'of\n', 'good\n', 'strings\n'] + + def setUp(self): + self.file_byte_string = ''.join(self.file_list).encode('utf-8') + super().setUp() + + def test_getline(self): + with open(self.file_name) as fp: + for index, line in enumerate(fp): + if not line.endswith('\n'): + line += '\n' + + cached_line = linecache.getline(self.file_name, index + 1) + self.assertEqual(line, cached_line) + + def test_getlines(self): + lines = linecache.getlines(self.file_name) + self.assertEqual(lines, self.file_list) + + +class GetLineTestsBadData(TempFile): + # file_byte_string = b'Bad data goes here' + + def test_getline(self): + self.assertRaises((SyntaxError, UnicodeDecodeError), + linecache.getline, self.file_name, 1) + + def test_getlines(self): + self.assertRaises((SyntaxError, UnicodeDecodeError), + linecache.getlines, self.file_name) + + +class EmptyFile(GetLineTestsGoodData, unittest.TestCase): + file_list = [] + + +class SingleEmptyLine(GetLineTestsGoodData, unittest.TestCase): + file_list = ['\n'] + + +class GoodUnicode(GetLineTestsGoodData, unittest.TestCase): + file_list = ['á\n', 'b\n', 'abcdef\n', 'ááááá\n'] + + +class BadUnicode(GetLineTestsBadData, unittest.TestCase): + file_byte_string = b'\x80abc' + + class LineCacheTests(unittest.TestCase): def test_getline(self): @@ -53,13 +111,6 @@ class LineCacheTests(unittest.TestCase): self.assertEqual(getline(EMPTY, 1), EMPTY) self.assertEqual(getline(INVALID_NAME, 1), EMPTY) - # Check whether lines correspond to those from file iteration - for entry in TESTS: - filename = os.path.join(TEST_PATH, entry) + '.py' - with open(filename) as file: - for index, line in enumerate(file): - self.assertEqual(line, getline(filename, index + 1)) - # Check module loading for entry in MODULES: filename = os.path.join(MODULE_PATH, entry) + '.py' @@ -80,12 +131,13 @@ class LineCacheTests(unittest.TestCase): def test_clearcache(self): cached = [] - for entry in TESTS: - filename = os.path.join(TEST_PATH, entry) + '.py' + for entry in MODULES: + filename = os.path.join(MODULE_PATH, entry) + '.py' cached.append(filename) linecache.getline(filename, 1) # Are all files cached? + self.assertNotEqual(cached, []) cached_empty = [fn for fn in cached if fn not in linecache.cache] self.assertEqual(cached_empty, []) -- cgit v1.2.1 From acedfd119e133823b987f4f09ba08da2c22de612 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 18 Aug 2015 22:03:08 -0700 Subject: Add in missing docstrings. --- Lib/http/client.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) (limited to 'Lib') diff --git a/Lib/http/client.py b/Lib/http/client.py index 80c80cf576..155c2e3ec4 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -405,6 +405,7 @@ class HTTPResponse(io.BufferedIOBase): self.fp.flush() def readable(self): + """Always returns True""" return True # End of "raw stream" methods @@ -452,6 +453,10 @@ class HTTPResponse(io.BufferedIOBase): return s def readinto(self, b): + """Read up to len(b) bytes into bytearray b and return the number + of bytes read. + """ + if self.fp is None: return 0 @@ -683,6 +688,17 @@ class HTTPResponse(io.BufferedIOBase): return self.fp.fileno() def getheader(self, name, default=None): + '''Returns the value of the header matching *name*. + + If there are multiple matching headers, the values are + combined into a single string separated by commas and spaces. + + If no matching header is found, returns *default* or None if + the *default* is not specified. + + If the headers are unknown, raises http.client.ResponseNotReady. + + ''' if self.headers is None: raise ResponseNotReady() headers = self.headers.get_all(name) or default @@ -705,12 +721,45 @@ class HTTPResponse(io.BufferedIOBase): # For compatibility with old-style urllib responses. def info(self): + '''Returns an instance of the class mimetools.Message containing + meta-information associated with the URL. + + When the method is HTTP, these headers are those returned by + the server at the head of the retrieved HTML page (including + Content-Length and Content-Type). + + When the method is FTP, a Content-Length header will be + present if (as is now usual) the server passed back a file + length in response to the FTP retrieval request. A + Content-Type header will be present if the MIME type can be + guessed. + + When the method is local-file, returned headers will include + a Date representing the file’s last-modified time, a + Content-Length giving file size, and a Content-Type + containing a guess at the file’s type. See also the + description of the mimetools module. + + ''' return self.headers def geturl(self): + '''Return the real URL of the page. + + In some cases, the HTTP server redirects a client to another + URL. The urlopen() function handles this transparently, but in + some cases the caller needs to know which URL the client was + redirected to. The geturl() method can be used to get at this + redirected URL. + + ''' return self.url def getcode(self): + '''Return the HTTP status code that was sent with the response, + or None if the URL is not an HTTP URL. + + ''' return self.status class HTTPConnection: -- cgit v1.2.1 From 161585817b3f8aa994b4ccfe02961c5238d1d5b8 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 18 Aug 2015 22:25:16 -0700 Subject: Issue #24879: Teach pydoc to display named tuple fields in the order they were defined. --- Lib/pydoc.py | 19 +++++++++++++++---- Lib/test/test_pydoc.py | 16 ++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) (limited to 'Lib') diff --git a/Lib/pydoc.py b/Lib/pydoc.py index ee558bfea3..f4f253010f 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -209,6 +209,18 @@ def classify_class_attrs(object): results.append((name, kind, cls, value)) return results +def sort_attributes(attrs, object): + 'Sort the attrs list in-place by _fields and then alphabetically by name' + # This allows data descriptors to be ordered according + # to a _fields attribute if present. + fields = getattr(object, '_fields', []) + try: + field_order = {name : i-len(fields) for (i, name) in enumerate(fields)} + except TypeError: + field_order = {} + keyfunc = lambda attr: (field_order.get(attr[0], 0), attr[0]) + attrs.sort(key=keyfunc) + # ----------------------------------------------------- module manipulation def ispackage(path): @@ -867,8 +879,7 @@ class HTMLDoc(Doc): object.__module__) tag += ':
\n' - # Sort attrs by name. - attrs.sort(key=lambda t: t[0]) + sort_attributes(attrs, object) # Pump out the attrs, segregated by kind. attrs = spill('Methods %s' % tag, attrs, @@ -1286,8 +1297,8 @@ location listed above. else: tag = "inherited from %s" % classname(thisclass, object.__module__) - # Sort attrs by name. - attrs.sort() + + sort_attributes(attrs, object) # Pump out the attrs, segregated by kind. attrs = spill("Methods %s:\n" % tag, attrs, diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index ec5c31ba72..0533a03fa7 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -811,6 +811,22 @@ class TestDescriptions(unittest.TestCase): self.assertEqual(self._get_summary_line(t.wrap), "wrap(text) method of textwrap.TextWrapper instance") + def test_field_order_for_named_tuples(self): + Person = namedtuple('Person', ['nickname', 'firstname', 'agegroup']) + s = pydoc.render_doc(Person) + self.assertLess(s.index('nickname'), s.index('firstname')) + self.assertLess(s.index('firstname'), s.index('agegroup')) + + class NonIterableFields: + _fields = None + + class NonHashableFields: + _fields = [[]] + + # Make sure these doesn't fail + pydoc.render_doc(NonIterableFields) + pydoc.render_doc(NonHashableFields) + @requires_docstrings def test_bound_builtin_method(self): s = StringIO() -- cgit v1.2.1 From 9d734bd331c8a29cae47bbfd1497853481b7d0b7 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 20 Aug 2015 10:48:46 +1200 Subject: Fix test_linecache change. In non-UTF8 locales open() will have the wrong encoding for the test data. We need to open the file via tokenize, to get a detected encoding instead. --- Lib/test/test_linecache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index e74aef3580..240db7f874 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -4,6 +4,7 @@ import linecache import unittest import os.path import tempfile +import tokenize from test import support @@ -54,7 +55,7 @@ class GetLineTestsGoodData(TempFile): super().setUp() def test_getline(self): - with open(self.file_name) as fp: + with tokenize.open(self.file_name) as fp: for index, line in enumerate(fp): if not line.endswith('\n'): line += '\n' -- cgit v1.2.1 From 4023aac217e9f7a011534f54884291548817605f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 23 Aug 2015 11:28:01 -0700 Subject: Issue #24878: Fix typos and line wrap --- Lib/aifc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/aifc.py b/Lib/aifc.py index 1556b53019..692d0bfd27 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -258,12 +258,13 @@ _aifc_params = namedtuple('_aifc_params', 'nchannels sampwidth framerate nframes comptype compname') _aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)' -_aifc_params.sampwidth.__doc__ = 'Ample width in bytes' +_aifc_params.sampwidth.__doc__ = 'Sample width in bytes' _aifc_params.framerate.__doc__ = 'Sampling frequency' _aifc_params.nframes.__doc__ = 'Number of audio frames' _aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)' -_aifc_params.compname.__doc__ = ("""A human-readable version ofcompression type -('not compressed' for AIFF files)'""") +_aifc_params.compname.__doc__ = ("""\ +A human-readable version of the compression type +('not compressed' for AIFF files)""") class Aifc_read: -- cgit v1.2.1 From bc8f3e6cc50d3427b29739ff1c0a212325ef8111 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Mon, 24 Aug 2015 11:46:56 +1200 Subject: Issue #24633: site-packages/README -> README.txt. --- Lib/site-packages/README | 2 -- Lib/site-packages/README.txt | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 Lib/site-packages/README create mode 100644 Lib/site-packages/README.txt (limited to 'Lib') diff --git a/Lib/site-packages/README b/Lib/site-packages/README deleted file mode 100644 index 273f6251a7..0000000000 --- a/Lib/site-packages/README +++ /dev/null @@ -1,2 +0,0 @@ -This directory exists so that 3rd party packages can be installed -here. Read the source for site.py for more details. diff --git a/Lib/site-packages/README.txt b/Lib/site-packages/README.txt new file mode 100644 index 0000000000..273f6251a7 --- /dev/null +++ b/Lib/site-packages/README.txt @@ -0,0 +1,2 @@ +This directory exists so that 3rd party packages can be installed +here. Read the source for site.py for more details. -- cgit v1.2.1 From 935b9373bdbf267f5a602591bdcd8314a538c3a4 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Wed, 26 Aug 2015 12:40:28 +1200 Subject: Issue #23552: Timeit now warns when there is substantial (4x) variance between best and worst times. Patch from Serhiy Storchaka. --- Lib/timeit.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) (limited to 'Lib') diff --git a/Lib/timeit.py b/Lib/timeit.py index 2de88f7271..98cb3eb89a 100755 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -317,20 +317,26 @@ def main(args=None, *, _wrap_timer=None): print("%d loops," % number, end=' ') usec = best * 1e6 / number if time_unit is not None: - print("best of %d: %.*g %s per loop" % (repeat, precision, - usec/units[time_unit], time_unit)) + scale = units[time_unit] else: - if usec < 1000: - print("best of %d: %.*g usec per loop" % (repeat, precision, usec)) - else: - msec = usec / 1000 - if msec < 1000: - print("best of %d: %.*g msec per loop" % (repeat, - precision, msec)) - else: - sec = msec / 1000 - print("best of %d: %.*g sec per loop" % (repeat, - precision, sec)) + scales = [(scale, unit) for unit, scale in units.items()] + scales.sort(reverse=True) + for scale, time_unit in scales: + if usec >= scale: + break + print("best of %d: %.*g %s per loop" % (repeat, precision, + usec/scale, time_unit)) + best = min(r) + usec = best * 1e6 / number + worst = max(r) + if worst >= best * 4: + usec = worst * 1e6 / number + import warnings + warnings.warn_explicit( + "The test results are likely unreliable. The worst\n" + "time (%.*g %s) was more than four times slower than the best time." % + (precision, usec/scale, time_unit), + UserWarning, '', 0) return None if __name__ == "__main__": -- cgit v1.2.1 From d0ee151e3b640eae2a63b1c13860837d3ece2cda Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 01:43:56 +0200 Subject: Issue #23517: Add "half up" rounding mode to the _PyTime API --- Lib/test/test_time.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 6334e022e0..ed20470912 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -30,8 +30,11 @@ class _PyTime(enum.IntEnum): ROUND_FLOOR = 0 # Round towards infinity (+inf) ROUND_CEILING = 1 + # Round to nearest with ties going away from zero + ROUND_HALF_UP = 2 -ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING) +ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING, + _PyTime.ROUND_HALF_UP) class TimeTestCase(unittest.TestCase): @@ -753,11 +756,11 @@ class TestPyTime_t(unittest.TestCase): (123.0, 123 * SEC_TO_NS), (-7.0, -7 * SEC_TO_NS), - # nanosecond are kept for value <= 2^23 seconds + # nanosecond are kept for value <= 2^23 seconds, + # except 2**23-1e-9 with HALF_UP (2**22 - 1e-9, 4194303999999999), (2**22, 4194304000000000), (2**22 + 1e-9, 4194304000000001), - (2**23 - 1e-9, 8388607999999999), (2**23, 8388608000000000), # start loosing precision for value > 2^23 seconds @@ -790,24 +793,36 @@ class TestPyTime_t(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for obj, ts, rnd in ( # close to zero ( 1e-10, 0, FLOOR), ( 1e-10, 1, CEILING), + ( 1e-10, 0, HALF_UP), (-1e-10, -1, FLOOR), (-1e-10, 0, CEILING), + (-1e-10, 0, HALF_UP), # test rounding of the last nanosecond ( 1.1234567899, 1123456789, FLOOR), ( 1.1234567899, 1123456790, CEILING), + ( 1.1234567899, 1123456790, HALF_UP), (-1.1234567899, -1123456790, FLOOR), (-1.1234567899, -1123456789, CEILING), + (-1.1234567899, -1123456790, HALF_UP), # close to 1 second ( 0.9999999999, 999999999, FLOOR), ( 0.9999999999, 1000000000, CEILING), + ( 0.9999999999, 1000000000, HALF_UP), (-0.9999999999, -1000000000, FLOOR), (-0.9999999999, -999999999, CEILING), + (-0.9999999999, -1000000000, HALF_UP), + + # close to 2^23 seconds + (2**23 - 1e-9, 8388607999999999, FLOOR), + (2**23 - 1e-9, 8388607999999999, CEILING), + (2**23 - 1e-9, 8388608000000000, HALF_UP), ): with self.subTest(obj=obj, round=rnd, timestamp=ts): self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) @@ -875,18 +890,33 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for ns, tv, rnd in ( # nanoseconds (1, (0, 0), FLOOR), (1, (0, 1), CEILING), + (1, (0, 0), HALF_UP), (-1, (-1, 999999), FLOOR), (-1, (0, 0), CEILING), + (-1, (0, 0), HALF_UP), # seconds + nanoseconds (1234567001, (1, 234567), FLOOR), (1234567001, (1, 234568), CEILING), + (1234567001, (1, 234567), HALF_UP), (-1234567001, (-2, 765432), FLOOR), (-1234567001, (-2, 765433), CEILING), + (-1234567001, (-2, 765433), HALF_UP), + + # half up + (499, (0, 0), HALF_UP), + (500, (0, 1), HALF_UP), + (501, (0, 1), HALF_UP), + (999, (0, 1), HALF_UP), + (-499, (0, 0), HALF_UP), + (-500, (0, 0), HALF_UP), + (-501, (-1, 999999), HALF_UP), + (-999, (-1, 999999), HALF_UP), ): with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) @@ -929,18 +959,33 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for ns, ms, rnd in ( # nanoseconds (1, 0, FLOOR), (1, 1, CEILING), + (1, 0, HALF_UP), (-1, 0, FLOOR), (-1, -1, CEILING), + (-1, 0, HALF_UP), # seconds + nanoseconds (1234 * MS_TO_NS + 1, 1234, FLOOR), (1234 * MS_TO_NS + 1, 1235, CEILING), + (1234 * MS_TO_NS + 1, 1234, HALF_UP), (-1234 * MS_TO_NS - 1, -1234, FLOOR), (-1234 * MS_TO_NS - 1, -1235, CEILING), + (-1234 * MS_TO_NS - 1, -1234, HALF_UP), + + # half up + (499999, 0, HALF_UP), + (499999, 0, HALF_UP), + (500000, 1, HALF_UP), + (999999, 1, HALF_UP), + (-499999, 0, HALF_UP), + (-500000, -1, HALF_UP), + (-500001, -1, HALF_UP), + (-999999, -1, HALF_UP), ): with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): self.assertEqual(PyTime_AsMilliseconds(ns, rnd), ms) @@ -966,18 +1011,31 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for ns, ms, rnd in ( # nanoseconds (1, 0, FLOOR), (1, 1, CEILING), + (1, 0, HALF_UP), (-1, 0, FLOOR), (-1, -1, CEILING), + (-1, 0, HALF_UP), # seconds + nanoseconds (1234 * US_TO_NS + 1, 1234, FLOOR), (1234 * US_TO_NS + 1, 1235, CEILING), + (1234 * US_TO_NS + 1, 1234, HALF_UP), (-1234 * US_TO_NS - 1, -1234, FLOOR), (-1234 * US_TO_NS - 1, -1235, CEILING), + (-1234 * US_TO_NS - 1, -1234, HALF_UP), + + # half up + (1499, 1, HALF_UP), + (1500, 2, HALF_UP), + (1501, 2, HALF_UP), + (-1499, -1, HALF_UP), + (-1500, -2, HALF_UP), + (-1501, -2, HALF_UP), ): with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): self.assertEqual(PyTime_AsMicroseconds(ns, rnd), ms) -- cgit v1.2.1 From e3bcab0b9cf1d0a8fcb9fd46d1c1d95950297ebd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 10:39:40 +0200 Subject: Issue #23517: Reintroduce unit tests for the old PyTime API since it's still used. --- Lib/test/test_time.py | 154 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index ed20470912..926e700d93 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -727,6 +727,9 @@ class TestPytime(unittest.TestCase): @unittest.skipUnless(_testcapi is not None, 'need the _testcapi module') class TestPyTime_t(unittest.TestCase): + """ + Test the _PyTime_t API. + """ def test_FromSeconds(self): from _testcapi import PyTime_FromSeconds for seconds in (0, 3, -456, _testcapi.INT_MAX, _testcapi.INT_MIN): @@ -1041,5 +1044,156 @@ class TestPyTime_t(unittest.TestCase): self.assertEqual(PyTime_AsMicroseconds(ns, rnd), ms) +@unittest.skipUnless(_testcapi is not None, + 'need the _testcapi module') +class TestOldPyTime(unittest.TestCase): + """ + Test the old _PyTime_t API: _PyTime_ObjectToXXX() functions. + """ + def setUp(self): + self.invalid_values = ( + -(2 ** 100), 2 ** 100, + -(2.0 ** 100.0), 2.0 ** 100.0, + ) + + @support.cpython_only + def test_time_t(self): + from _testcapi import pytime_object_to_time_t + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, time_t in ( + # int + (0, 0), + (-1, -1), + + # float + (1.0, 1), + (-1.0, -1), + ): + self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) + + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP + for obj, time_t, rnd in ( + (-1.9, -2, FLOOR), + (-1.9, -2, HALF_UP), + (-1.9, -1, CEILING), + (1.9, 1, FLOOR), + (1.9, 2, HALF_UP), + (1.9, 2, CEILING), + ): + self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) + + # Test OverflowError + rnd = _PyTime.ROUND_FLOOR + for invalid in self.invalid_values: + self.assertRaises(OverflowError, + pytime_object_to_time_t, invalid, rnd) + + def test_timeval(self): + from _testcapi import pytime_object_to_timeval + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, timeval in ( + # int + (0, (0, 0)), + (-1, (-1, 0)), + + # float + (-1.0, (-1, 0)), + (-1.2, (-2, 800000)), + (-1e-6, (-1, 999999)), + (1e-6, (0, 1)), + ): + with self.subTest(obj=obj, round=rnd, timeval=timeval): + self.assertEqual(pytime_object_to_timeval(obj, rnd), + timeval) + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP + for obj, timeval, rnd in ( + (-1e-7, (-1, 999999), FLOOR), + (-1e-7, (0, 0), CEILING), + (-1e-7, (0, 0), HALF_UP), + + (1e-7, (0, 0), FLOOR), + (1e-7, (0, 1), CEILING), + (1e-7, (0, 0), HALF_UP), + + (0.4e-6, (0, 0), HALF_UP), + (0.5e-6, (0, 1), HALF_UP), + (0.6e-6, (0, 1), HALF_UP), + + (0.9999999, (0, 999999), FLOOR), + (0.9999999, (1, 0), CEILING), + (0.9999999, (1, 0), HALF_UP), + ): + with self.subTest(obj=obj, round=rnd, timeval=timeval): + self.assertEqual(pytime_object_to_timeval(obj, rnd), timeval) + + rnd = _PyTime.ROUND_FLOOR + for invalid in self.invalid_values: + self.assertRaises(OverflowError, + pytime_object_to_timeval, invalid, rnd) + + @support.cpython_only + def test_timespec(self): + from _testcapi import pytime_object_to_timespec + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, timespec in ( + # int + (0, (0, 0)), + (-1, (-1, 0)), + + # float + (-1.0, (-1, 0)), + (-1e-9, (-1, 999999999)), + (1e-9, (0, 1)), + (-1.2, (-2, 800000000)), + ): + with self.subTest(obj=obj, round=rnd, timespec=timespec): + self.assertEqual(pytime_object_to_timespec(obj, rnd), + timespec) + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP + for obj, timespec, rnd in ( + (-1e-10, (-1, 999999999), FLOOR), + (-1e-10, (0, 0), CEILING), + (-1e-10, (0, 0), HALF_UP), + + (1e-10, (0, 0), FLOOR), + (1e-10, (0, 1), CEILING), + (1e-10, (0, 0), HALF_UP), + + (0.4e-9, (0, 0), HALF_UP), + (0.5e-9, (0, 1), HALF_UP), + (0.6e-9, (0, 1), HALF_UP), + + (0.9999999999, (0, 999999999), FLOOR), + (0.9999999999, (1, 0), CEILING), + (0.9999999999, (1, 0), HALF_UP), + ): + with self.subTest(obj=obj, round=rnd, timespec=timespec): + self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) + + # Test OverflowError + rnd = FLOOR + for invalid in self.invalid_values: + self.assertRaises(OverflowError, + pytime_object_to_timespec, invalid, rnd) + + if __name__ == "__main__": unittest.main() -- cgit v1.2.1 From 833503880d2ac7d4a89945c2c47bcb3fa698f10e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 11:05:32 +0200 Subject: test_time: add more tests on HALF_UP rounding mode --- Lib/test/test_time.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 926e700d93..db962779b3 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1082,9 +1082,18 @@ class TestOldPyTime(unittest.TestCase): (-1.9, -2, FLOOR), (-1.9, -2, HALF_UP), (-1.9, -1, CEILING), + (1.9, 1, FLOOR), (1.9, 2, HALF_UP), (1.9, 2, CEILING), + + (-0.6, -1, HALF_UP), + (-0.5, -1, HALF_UP), + (-0.4, 0, HALF_UP), + + (0.4, 0, HALF_UP), + (0.5, 1, HALF_UP), + (0.6, 1, HALF_UP), ): self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) @@ -1127,13 +1136,19 @@ class TestOldPyTime(unittest.TestCase): (1e-7, (0, 1), CEILING), (1e-7, (0, 0), HALF_UP), - (0.4e-6, (0, 0), HALF_UP), - (0.5e-6, (0, 1), HALF_UP), - (0.6e-6, (0, 1), HALF_UP), - (0.9999999, (0, 999999), FLOOR), (0.9999999, (1, 0), CEILING), (0.9999999, (1, 0), HALF_UP), + + (-0.6e-6, (-1, 999999), HALF_UP), + # skipped, -0.5e-6 is inexact in base 2 + #(-0.5e-6, (-1, 999999), HALF_UP), + (-0.4e-6, (0, 0), HALF_UP), + + (0.4e-6, (0, 0), HALF_UP), + # skipped, 0.5e-6 is inexact in base 2 + #(0.5e-6, (0, 1), HALF_UP), + (0.6e-6, (0, 1), HALF_UP), ): with self.subTest(obj=obj, round=rnd, timeval=timeval): self.assertEqual(pytime_object_to_timeval(obj, rnd), timeval) @@ -1177,13 +1192,18 @@ class TestOldPyTime(unittest.TestCase): (1e-10, (0, 1), CEILING), (1e-10, (0, 0), HALF_UP), - (0.4e-9, (0, 0), HALF_UP), - (0.5e-9, (0, 1), HALF_UP), - (0.6e-9, (0, 1), HALF_UP), - (0.9999999999, (0, 999999999), FLOOR), (0.9999999999, (1, 0), CEILING), (0.9999999999, (1, 0), HALF_UP), + + (-0.6e-9, (-1, 999999999), HALF_UP), + # skipped, 0.5e-6 is inexact in base 2 + #(-0.5e-9, (-1, 999999999), HALF_UP), + (-0.4e-9, (0, 0), HALF_UP), + + (0.4e-9, (0, 0), HALF_UP), + (0.5e-9, (0, 1), HALF_UP), + (0.6e-9, (0, 1), HALF_UP), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) -- cgit v1.2.1 From 492745cafe23bafe5405495e30316c4c3eea728a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 13:54:28 +0200 Subject: Issue #23517: test_time, skip a test checking a corner case on floating point rounding --- Lib/test/test_time.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index db962779b3..5947f4041f 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -825,7 +825,9 @@ class TestPyTime_t(unittest.TestCase): # close to 2^23 seconds (2**23 - 1e-9, 8388607999999999, FLOOR), (2**23 - 1e-9, 8388607999999999, CEILING), - (2**23 - 1e-9, 8388608000000000, HALF_UP), + # Issue #23517: skip HALF_UP test because the result is different + # depending on the FPU and how the compiler optimize the code :-/ + #(2**23 - 1e-9, 8388608000000000, HALF_UP), ): with self.subTest(obj=obj, round=rnd, timestamp=ts): self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) -- cgit v1.2.1 From b998bd414e210becac93d965fd42fa8f71ab12f1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 14:23:40 +0200 Subject: Issue 24297: Fix test_symbol on Windows Don't rely on end of line. Open files in text mode, not in binary mode. --- Lib/test/test_symbol.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_symbol.py b/Lib/test/test_symbol.py index c126be3d01..2dcb9de8b0 100644 --- a/Lib/test/test_symbol.py +++ b/Lib/test/test_symbol.py @@ -15,12 +15,11 @@ TEST_PY_FILE = 'symbol_test.py' class TestSymbolGeneration(unittest.TestCase): def _copy_file_without_generated_symbols(self, source_file, dest_file): - with open(source_file, 'rb') as fp: + with open(source_file) as fp: lines = fp.readlines() - nl = lines[0][len(lines[0].rstrip()):] - with open(dest_file, 'wb') as fp: - fp.writelines(lines[:lines.index(b"#--start constants--" + nl) + 1]) - fp.writelines(lines[lines.index(b"#--end constants--" + nl):]) + with open(dest_file, 'w') as fp: + fp.writelines(lines[:lines.index("#--start constants--\n") + 1]) + fp.writelines(lines[lines.index("#--end constants--\n"):]) def _generate_symbols(self, grammar_file, target_symbol_py_file): proc = subprocess.Popen([sys.executable, @@ -30,18 +29,26 @@ class TestSymbolGeneration(unittest.TestCase): stderr = proc.communicate()[1] return proc.returncode, stderr + def compare_files(self, file1, file2): + with open(file1) as fp: + lines1 = fp.readlines() + with open(file2) as fp: + lines2 = fp.readlines() + self.assertEqual(lines1, lines2) + @unittest.skipIf(not os.path.exists(GRAMMAR_FILE), 'test only works from source build directory') def test_real_grammar_and_symbol_file(self): - self._copy_file_without_generated_symbols(SYMBOL_FILE, TEST_PY_FILE) - self.addCleanup(support.unlink, TEST_PY_FILE) - self.assertFalse(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE)) - self.assertEqual((0, b''), self._generate_symbols(GRAMMAR_FILE, - TEST_PY_FILE)) - self.assertTrue(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE), - 'symbol stat: %r\ntest_py stat: %r\n' % - (os.stat(SYMBOL_FILE), - os.stat(TEST_PY_FILE))) + output = support.TESTFN + self.addCleanup(support.unlink, output) + + self._copy_file_without_generated_symbols(SYMBOL_FILE, output) + + exitcode, stderr = self._generate_symbols(GRAMMAR_FILE, output) + self.assertEqual(b'', stderr) + self.assertEqual(0, exitcode) + + self.compare_files(SYMBOL_FILE, output) if __name__ == "__main__": -- cgit v1.2.1 From 85ba2bd267057b7fdc8c7949f87a7014352a2e4b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 15:44:22 +0200 Subject: test_gdb: add debug info to investigate failure on "s390x SLES 3.x" buildbot --- Lib/test/test_gdb.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 0322677793..193c97afa3 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -28,9 +28,13 @@ except OSError: # This is what "no gdb" looks like. There may, however, be other # errors that manifest this way too. raise unittest.SkipTest("Couldn't find gdb on the path") -gdb_version_number = re.search(b"^GNU gdb [^\d]*(\d+)\.(\d)", gdb_version) -gdb_major_version = int(gdb_version_number.group(1)) -gdb_minor_version = int(gdb_version_number.group(2)) +try: + gdb_version_number = re.search(b"^GNU gdb [^\d]*(\d+)\.(\d)", gdb_version) + gdb_major_version = int(gdb_version_number.group(1)) + gdb_minor_version = int(gdb_version_number.group(2)) +except Exception: + raise ValueError("unable to parse GDB version: %r" % gdb_version) + if gdb_major_version < 7: raise unittest.SkipTest("gdb versions before 7.0 didn't support python embedding" " Saw:\n" + gdb_version.decode('ascii', 'replace')) -- cgit v1.2.1 From 52b498c5e777e288c304702b08b557e11252ab71 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 15:46:00 +0200 Subject: test_gdb: fix ResourceWarning if the test is interrupted --- Lib/test/test_gdb.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 193c97afa3..a51b552503 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -63,9 +63,11 @@ def run_gdb(*args, **env_vars): base_cmd = ('gdb', '--batch', '-nx') if (gdb_major_version, gdb_minor_version) >= (7, 4): base_cmd += ('-iex', 'add-auto-load-safe-path ' + checkout_hook_path) - out, err = subprocess.Popen(base_cmd + args, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, - ).communicate() + proc = subprocess.Popen(base_cmd + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env) + with proc: + out, err = proc.communicate() return out.decode('utf-8', 'replace'), err.decode('utf-8', 'replace') # Verify that "gdb" was built with the embedded python support enabled: -- cgit v1.2.1 From 4a1a917ce43392207cddd98c61b7d008d61b191f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 17:19:04 +0200 Subject: test_eintr: try to debug hang on FreeBSD --- Lib/test/eintrdata/eintr_tester.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'Lib') diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index f755880514..73711a47b0 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -8,6 +8,7 @@ Signals are generated in-process using setitimer(ITIMER_REAL), which allows sub-second periodicity (contrarily to signal()). """ +import faulthandler import io import os import select @@ -36,10 +37,17 @@ class EINTRBaseTest(unittest.TestCase): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) + if hasattr(faulthandler, 'dump_traceback_later'): + # Most tests take less than 30 seconds, so 15 minutes should be + # enough. dump_traceback_later() is implemented with a thread, but + # pthread_sigmask() is used to mask all signaled on this thread. + faulthandler.dump_traceback_later(15 * 60, exit=True) @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) + if hasattr(faulthandler, 'cancel_dump_traceback_later'): + faulthandler.cancel_dump_traceback_later() @classmethod def tearDownClass(cls): -- cgit v1.2.1 From ea58c6adb1a11497266bc69f0629feb2fb541fa9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 19:16:07 +0200 Subject: Issue #23517: datetime.timedelta constructor now rounds microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of rounding to nearest with ties going to nearest even integer (ROUND_HALF_EVEN). --- Lib/datetime.py | 12 ++++++++++-- Lib/test/datetimetester.py | 14 +++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) (limited to 'Lib') diff --git a/Lib/datetime.py b/Lib/datetime.py index db13b12ef3..d661460fa8 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -316,6 +316,14 @@ def _divide_and_round(a, b): return q +def _round_half_up(x): + """Round to nearest with ties going away from zero.""" + if x >= 0.0: + return _math.floor(x + 0.5) + else: + return _math.ceil(x - 0.5) + + class timedelta: """Represent the difference between two datetime objects. @@ -399,7 +407,7 @@ class timedelta: # secondsfrac isn't referenced again if isinstance(microseconds, float): - microseconds = round(microseconds + usdouble) + microseconds = _round_half_up(microseconds + usdouble) seconds, microseconds = divmod(microseconds, 1000000) days, seconds = divmod(seconds, 24*3600) d += days @@ -410,7 +418,7 @@ class timedelta: days, seconds = divmod(seconds, 24*3600) d += days s += seconds - microseconds = round(microseconds + usdouble) + microseconds = _round_half_up(microseconds + usdouble) assert isinstance(s, int) assert isinstance(microseconds, int) assert abs(s) <= 3 * 24 * 3600 diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index babeb44c06..62f55272d9 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -662,28 +662,24 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): # Single-field rounding. eq(td(milliseconds=0.4/1000), td(0)) # rounds to 0 eq(td(milliseconds=-0.4/1000), td(0)) # rounds to 0 - eq(td(milliseconds=0.5/1000), td(microseconds=0)) - eq(td(milliseconds=-0.5/1000), td(microseconds=0)) + eq(td(milliseconds=0.5/1000), td(microseconds=1)) + eq(td(milliseconds=-0.5/1000), td(microseconds=-1)) eq(td(milliseconds=0.6/1000), td(microseconds=1)) eq(td(milliseconds=-0.6/1000), td(microseconds=-1)) - eq(td(seconds=0.5/10**6), td(microseconds=0)) - eq(td(seconds=-0.5/10**6), td(microseconds=0)) + eq(td(seconds=0.5/10**6), td(microseconds=1)) + eq(td(seconds=-0.5/10**6), td(microseconds=-1)) # Rounding due to contributions from more than one field. us_per_hour = 3600e6 us_per_day = us_per_hour * 24 eq(td(days=.4/us_per_day), td(0)) eq(td(hours=.2/us_per_hour), td(0)) - eq(td(days=.4/us_per_day, hours=.2/us_per_hour), td(microseconds=1)) + eq(td(days=.4/us_per_day, hours=.2/us_per_hour), td(microseconds=1), td) eq(td(days=-.4/us_per_day), td(0)) eq(td(hours=-.2/us_per_hour), td(0)) eq(td(days=-.4/us_per_day, hours=-.2/us_per_hour), td(microseconds=-1)) - # Test for a patch in Issue 8860 - eq(td(microseconds=0.5), 0.5*td(microseconds=1.0)) - eq(td(microseconds=0.5)//td.resolution, 0.5*td.resolution//td.resolution) - def test_massive_normalization(self): td = timedelta(microseconds=-1) self.assertEqual((td.days, td.seconds, td.microseconds), -- cgit v1.2.1 From 3a3cb362abc7a0e74b5e98b97c08ffac8803296f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 3 Sep 2015 01:38:44 +0200 Subject: Rewrite eintr_tester.py to avoid os.fork() eintr_tester.py calls signal.setitimer() to send signals to the current process every 100 ms. The test sometimes hangs on FreeBSD. Maybe there is a race condition in the child process after fork(). It's unsafe to run arbitrary code after fork(). This change replace os.fork() with a regular call to subprocess.Popen(). This change avoids the risk of having a child process which continue to execute eintr_tester.py instead of exiting. It also ensures that the child process doesn't inherit unexpected file descriptors by mistake. Since I'm unable to reproduce the issue on FreeBSD, I will have to watch FreeBSD buildbots to check if the issue is fixed or not. Remove previous attempt to debug: remove call to faulthandler.dump_traceback_later(). --- Lib/test/eintrdata/eintr_tester.py | 259 ++++++++++++++++++++++--------------- 1 file changed, 158 insertions(+), 101 deletions(-) (limited to 'Lib') diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 73711a47b0..0616df66c7 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -8,12 +8,13 @@ Signals are generated in-process using setitimer(ITIMER_REAL), which allows sub-second periodicity (contrarily to signal()). """ -import faulthandler import io import os import select import signal import socket +import subprocess +import sys import time import unittest @@ -29,7 +30,7 @@ class EINTRBaseTest(unittest.TestCase): # signal delivery periodicity signal_period = 0.1 # default sleep time for tests - should obviously have: - # sleep_time > signal_period + # sleep_time > signal_period sleep_time = 0.2 @classmethod @@ -37,17 +38,10 @@ class EINTRBaseTest(unittest.TestCase): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) - if hasattr(faulthandler, 'dump_traceback_later'): - # Most tests take less than 30 seconds, so 15 minutes should be - # enough. dump_traceback_later() is implemented with a thread, but - # pthread_sigmask() is used to mask all signaled on this thread. - faulthandler.dump_traceback_later(15 * 60, exit=True) @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) - if hasattr(faulthandler, 'cancel_dump_traceback_later'): - faulthandler.cancel_dump_traceback_later() @classmethod def tearDownClass(cls): @@ -59,18 +53,22 @@ class EINTRBaseTest(unittest.TestCase): # default sleep time time.sleep(cls.sleep_time) + def subprocess(self, *args, **kw): + cmd_args = (sys.executable, '-c') + args + return subprocess.Popen(cmd_args, **kw) + @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") class OSEINTRTest(EINTRBaseTest): """ EINTR tests for the os module. """ + def new_sleep_process(self): + code = 'import time; time.sleep(%r)' % self.sleep_time + return self.subprocess(code) + def _test_wait_multiple(self, wait_func): num = 3 - for _ in range(num): - pid = os.fork() - if pid == 0: - self._sleep() - os._exit(0) + processes = [self.new_sleep_process() for _ in range(num)] for _ in range(num): wait_func() @@ -82,12 +80,8 @@ class OSEINTRTest(EINTRBaseTest): self._test_wait_multiple(lambda: os.wait3(0)) def _test_wait_single(self, wait_func): - pid = os.fork() - if pid == 0: - self._sleep() - os._exit(0) - else: - wait_func(pid) + proc = self.new_sleep_process() + wait_func(proc.pid) def test_waitpid(self): self._test_wait_single(lambda pid: os.waitpid(pid, 0)) @@ -105,19 +99,24 @@ class OSEINTRTest(EINTRBaseTest): # atomic datas = [b"hello", b"world", b"spam"] - pid = os.fork() - if pid == 0: - os.close(rd) - for data in datas: - # let the parent block on read() - self._sleep() - os.write(wr, data) - os._exit(0) - else: - self.addCleanup(os.waitpid, pid, 0) + code = '\n'.join(( + 'import os, sys, time', + '', + 'wr = int(sys.argv[1])', + 'datas = %r' % datas, + 'sleep_time = %r' % self.sleep_time, + '', + 'for data in datas:', + ' # let the parent block on read()', + ' time.sleep(sleep_time)', + ' os.write(wr, data)', + )) + + with self.subprocess(code, str(wr), pass_fds=[wr]) as proc: os.close(wr) for data in datas: self.assertEqual(data, os.read(rd, len(data))) + self.assertEqual(proc.wait(), 0) def test_write(self): rd, wr = os.pipe() @@ -127,23 +126,34 @@ class OSEINTRTest(EINTRBaseTest): # we must write enough data for the write() to block data = b"xyz" * support.PIPE_MAX_SIZE - pid = os.fork() - if pid == 0: - os.close(wr) - read_data = io.BytesIO() - # let the parent block on write() - self._sleep() - while len(read_data.getvalue()) < len(data): - chunk = os.read(rd, 2 * len(data)) - read_data.write(chunk) - self.assertEqual(read_data.getvalue(), data) - os._exit(0) - else: + code = '\n'.join(( + 'import io, os, sys, time', + '', + 'rd = int(sys.argv[1])', + 'sleep_time = %r' % self.sleep_time, + 'data = b"xyz" * %s' % support.PIPE_MAX_SIZE, + 'data_len = len(data)', + '', + '# let the parent block on write()', + 'time.sleep(sleep_time)', + '', + 'read_data = io.BytesIO()', + 'while len(read_data.getvalue()) < data_len:', + ' chunk = os.read(rd, 2 * data_len)', + ' read_data.write(chunk)', + '', + 'value = read_data.getvalue()', + 'if value != data:', + ' raise Exception("read error: %s vs %s bytes"', + ' % (len(value), data_len))', + )) + + with self.subprocess(code, str(rd), pass_fds=[rd]) as proc: os.close(rd) written = 0 while written < len(data): written += os.write(wr, memoryview(data)[written:]) - self.assertEqual(0, os.waitpid(pid, 0)[1]) + self.assertEqual(proc.wait(), 0) @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") @@ -159,19 +169,32 @@ class SocketEINTRTest(EINTRBaseTest): # single-byte payload guard us against partial recv datas = [b"x", b"y", b"z"] - pid = os.fork() - if pid == 0: - rd.close() - for data in datas: - # let the parent block on recv() - self._sleep() - wr.sendall(data) - os._exit(0) - else: - self.addCleanup(os.waitpid, pid, 0) + code = '\n'.join(( + 'import os, socket, sys, time', + '', + 'fd = int(sys.argv[1])', + 'family = %s' % int(wr.family), + 'sock_type = %s' % int(wr.type), + 'datas = %r' % datas, + 'sleep_time = %r' % self.sleep_time, + '', + 'wr = socket.fromfd(fd, family, sock_type)', + 'os.close(fd)', + '', + 'with wr:', + ' for data in datas:', + ' # let the parent block on recv()', + ' time.sleep(sleep_time)', + ' wr.sendall(data)', + )) + + fd = wr.fileno() + proc = self.subprocess(code, str(fd), pass_fds=[fd]) + with proc: wr.close() for data in datas: self.assertEqual(data, recv_func(rd, len(data))) + self.assertEqual(proc.wait(), 0) def test_recv(self): self._test_recv(socket.socket.recv) @@ -188,25 +211,43 @@ class SocketEINTRTest(EINTRBaseTest): # we must send enough data for the send() to block data = b"xyz" * (support.SOCK_MAX_SIZE // 3) - pid = os.fork() - if pid == 0: - wr.close() - # let the parent block on send() - self._sleep() - received_data = bytearray(len(data)) - n = 0 - while n < len(data): - n += rd.recv_into(memoryview(received_data)[n:]) - self.assertEqual(received_data, data) - os._exit(0) - else: + code = '\n'.join(( + 'import os, socket, sys, time', + '', + 'fd = int(sys.argv[1])', + 'family = %s' % int(rd.family), + 'sock_type = %s' % int(rd.type), + 'sleep_time = %r' % self.sleep_time, + 'data = b"xyz" * %s' % (support.SOCK_MAX_SIZE // 3), + 'data_len = len(data)', + '', + 'rd = socket.fromfd(fd, family, sock_type)', + 'os.close(fd)', + '', + 'with rd:', + ' # let the parent block on send()', + ' time.sleep(sleep_time)', + '', + ' received_data = bytearray(data_len)', + ' n = 0', + ' while n < data_len:', + ' n += rd.recv_into(memoryview(received_data)[n:])', + '', + 'if received_data != data:', + ' raise Exception("recv error: %s vs %s bytes"', + ' % (len(received_data), data_len))', + )) + + fd = rd.fileno() + proc = self.subprocess(code, str(fd), pass_fds=[fd]) + with proc: rd.close() written = 0 while written < len(data): sent = send_func(wr, memoryview(data)[written:]) # sendall() returns None written += len(data) if sent is None else sent - self.assertEqual(0, os.waitpid(pid, 0)[1]) + self.assertEqual(proc.wait(), 0) def test_send(self): self._test_send(socket.socket.send) @@ -223,45 +264,60 @@ class SocketEINTRTest(EINTRBaseTest): self.addCleanup(sock.close) sock.bind((support.HOST, 0)) - _, port = sock.getsockname() + port = sock.getsockname()[1] sock.listen() - pid = os.fork() - if pid == 0: - # let parent block on accept() - self._sleep() - with socket.create_connection((support.HOST, port)): - self._sleep() - os._exit(0) - else: - self.addCleanup(os.waitpid, pid, 0) + code = '\n'.join(( + 'import socket, time', + '', + 'host = %r' % support.HOST, + 'port = %s' % port, + 'sleep_time = %r' % self.sleep_time, + '', + '# let parent block on accept()', + 'time.sleep(sleep_time)', + 'with socket.create_connection((host, port)):', + ' time.sleep(sleep_time)', + )) + + with self.subprocess(code) as proc: client_sock, _ = sock.accept() client_sock.close() + self.assertEqual(proc.wait(), 0) @unittest.skipUnless(hasattr(os, 'mkfifo'), 'needs mkfifo()') def _test_open(self, do_open_close_reader, do_open_close_writer): + filename = support.TESTFN + # Use a fifo: until the child opens it for reading, the parent will # block when trying to open it for writing. - support.unlink(support.TESTFN) - os.mkfifo(support.TESTFN) - self.addCleanup(support.unlink, support.TESTFN) - - pid = os.fork() - if pid == 0: - # let the parent block - self._sleep() - do_open_close_reader(support.TESTFN) - os._exit(0) - else: - self.addCleanup(os.waitpid, pid, 0) - do_open_close_writer(support.TESTFN) + support.unlink(filename) + os.mkfifo(filename) + self.addCleanup(support.unlink, filename) + + code = '\n'.join(( + 'import os, time', + '', + 'path = %a' % filename, + 'sleep_time = %r' % self.sleep_time, + '', + '# let the parent block', + 'time.sleep(sleep_time)', + '', + do_open_close_reader, + )) + + with self.subprocess(code) as proc: + do_open_close_writer(filename) + + self.assertEqual(proc.wait(), 0) def test_open(self): - self._test_open(lambda path: open(path, 'r').close(), + self._test_open("open(path, 'r').close()", lambda path: open(path, 'w').close()) def test_os_open(self): - self._test_open(lambda path: os.close(os.open(path, os.O_RDONLY)), + self._test_open("os.close(os.open(path, os.O_RDONLY))", lambda path: os.close(os.open(path, os.O_WRONLY))) @@ -298,20 +354,21 @@ class SignalEINTRTest(EINTRBaseTest): old_handler = signal.signal(signum, lambda *args: None) self.addCleanup(signal.signal, signum, old_handler) + code = '\n'.join(( + 'import os, time', + 'pid = %s' % os.getpid(), + 'signum = %s' % int(signum), + 'sleep_time = %r' % self.sleep_time, + 'time.sleep(sleep_time)', + 'os.kill(pid, signum)', + )) + t0 = time.monotonic() - child_pid = os.fork() - if child_pid == 0: - # child - try: - self._sleep() - os.kill(pid, signum) - finally: - os._exit(0) - else: + with self.subprocess(code) as proc: # parent signal.sigwaitinfo([signum]) dt = time.monotonic() - t0 - os.waitpid(child_pid, 0) + self.assertEqual(proc.wait(), 0) self.assertGreaterEqual(dt, self.sleep_time) -- cgit v1.2.1 From ea28cadbcbdbda4b0e2ae610896914d6298ce2aa Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 3 Sep 2015 09:06:44 +0200 Subject: Issue #23517: fromtimestamp() and utcfromtimestamp() methods of datetime.datetime now round microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of rounding towards -Infinity (ROUND_FLOOR). --- Lib/datetime.py | 4 ++-- Lib/test/datetimetester.py | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) (limited to 'Lib') diff --git a/Lib/datetime.py b/Lib/datetime.py index d661460fa8..5ba2ddb7ab 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1384,7 +1384,7 @@ class datetime(date): converter = _time.localtime if tz is None else _time.gmtime t, frac = divmod(t, 1.0) - us = int(frac * 1e6) + us = _round_half_up(frac * 1e6) # If timestamp is less than one microsecond smaller than a # full second, us can be rounded up to 1000000. In this case, @@ -1404,7 +1404,7 @@ class datetime(date): def utcfromtimestamp(cls, t): """Construct a naive UTC datetime from a POSIX timestamp.""" t, frac = divmod(t, 1.0) - us = int(frac * 1e6) + us = _round_half_up(frac * 1e6) # If timestamp is less than one microsecond smaller than a # full second, us can be rounded up to 1000000. In this case, diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 62f55272d9..f516434bd6 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1847,6 +1847,7 @@ class TestDateTime(TestDate): zero = fts(0) self.assertEqual(zero.second, 0) self.assertEqual(zero.microsecond, 0) + one = fts(1e-6) try: minus_one = fts(-1e-6) except OSError: @@ -1857,22 +1858,22 @@ class TestDateTime(TestDate): self.assertEqual(minus_one.microsecond, 999999) t = fts(-1e-8) - self.assertEqual(t, minus_one) + self.assertEqual(t, zero) t = fts(-9e-7) self.assertEqual(t, minus_one) t = fts(-1e-7) - self.assertEqual(t, minus_one) + self.assertEqual(t, zero) t = fts(1e-7) self.assertEqual(t, zero) t = fts(9e-7) - self.assertEqual(t, zero) + self.assertEqual(t, one) t = fts(0.99999949) self.assertEqual(t.second, 0) self.assertEqual(t.microsecond, 999999) t = fts(0.9999999) - self.assertEqual(t.second, 0) - self.assertEqual(t.microsecond, 999999) + self.assertEqual(t.second, 1) + self.assertEqual(t.microsecond, 0) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, -- cgit v1.2.1 From b3f6f87571da72dab9d25df7c4088fd8cca28bb6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 4 Sep 2015 10:31:16 +0200 Subject: test_time: add tests on HALF_UP rounding mode for _PyTime_ObjectToTime_t() and _PyTime_ObjectToTimespec() --- Lib/test/test_time.py | 130 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 90 insertions(+), 40 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 5947f4041f..590425ae90 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -606,24 +606,49 @@ class TestPytime(unittest.TestCase): @support.cpython_only def test_time_t(self): from _testcapi import pytime_object_to_time_t + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, seconds in ( + # int + (-1, -1), + (0, 0), + (1, 1), + + # float + (-1.0, -1), + (1.0, 1), + ): + with self.subTest(obj=obj, round=rnd, seconds=seconds): + self.assertEqual(pytime_object_to_time_t(obj, rnd), + seconds) + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for obj, time_t, rnd in ( - # Round towards minus infinity (-inf) - (0, 0, _PyTime.ROUND_FLOOR), - (-1, -1, _PyTime.ROUND_FLOOR), - (-1.0, -1, _PyTime.ROUND_FLOOR), - (-1.9, -2, _PyTime.ROUND_FLOOR), - (1.0, 1, _PyTime.ROUND_FLOOR), - (1.9, 1, _PyTime.ROUND_FLOOR), - # Round towards infinity (+inf) - (0, 0, _PyTime.ROUND_CEILING), - (-1, -1, _PyTime.ROUND_CEILING), - (-1.0, -1, _PyTime.ROUND_CEILING), - (-1.9, -1, _PyTime.ROUND_CEILING), - (1.0, 1, _PyTime.ROUND_CEILING), - (1.9, 2, _PyTime.ROUND_CEILING), + (-1.9, -2, FLOOR), + (-1.9, -1, CEILING), + (-1.9, -2, HALF_UP), + + (1.9, 1, FLOOR), + (1.9, 2, CEILING), + (1.9, 2, HALF_UP), + + # half up + (-0.999, -1, HALF_UP), + (-0.510, -1, HALF_UP), + (-0.500, -1, HALF_UP), + (-0.490, 0, HALF_UP), + ( 0.490, 0, HALF_UP), + ( 0.500, 1, HALF_UP), + ( 0.510, 1, HALF_UP), + ( 0.999, 1, HALF_UP), ): self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) + # Test OverflowError rnd = _PyTime.ROUND_FLOOR for invalid in self.invalid_values: self.assertRaises(OverflowError, @@ -632,39 +657,64 @@ class TestPytime(unittest.TestCase): @support.cpython_only def test_timespec(self): from _testcapi import pytime_object_to_timespec + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, timespec in ( + # int + (0, (0, 0)), + (-1, (-1, 0)), + + # float + (-1.2, (-2, 800000000)), + (-1.0, (-1, 0)), + (-1e-9, (-1, 999999999)), + (1e-9, (0, 1)), + (1.0, (1, 0)), + ): + with self.subTest(obj=obj, round=rnd, timespec=timespec): + self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for obj, timespec, rnd in ( # Round towards minus infinity (-inf) - (0, (0, 0), _PyTime.ROUND_FLOOR), - (-1, (-1, 0), _PyTime.ROUND_FLOOR), - (-1.0, (-1, 0), _PyTime.ROUND_FLOOR), - (1e-9, (0, 1), _PyTime.ROUND_FLOOR), - (1e-10, (0, 0), _PyTime.ROUND_FLOOR), - (-1e-9, (-1, 999999999), _PyTime.ROUND_FLOOR), - (-1e-10, (-1, 999999999), _PyTime.ROUND_FLOOR), - (-1.2, (-2, 800000000), _PyTime.ROUND_FLOOR), - (0.9999999999, (0, 999999999), _PyTime.ROUND_FLOOR), - (1.1234567890, (1, 123456789), _PyTime.ROUND_FLOOR), - (1.1234567899, (1, 123456789), _PyTime.ROUND_FLOOR), - (-1.1234567890, (-2, 876543211), _PyTime.ROUND_FLOOR), - (-1.1234567891, (-2, 876543210), _PyTime.ROUND_FLOOR), + (-1e-10, (0, 0), CEILING), + (-1e-10, (-1, 999999999), FLOOR), + (-1e-10, (0, 0), HALF_UP), + (1e-10, (0, 0), FLOOR), + (1e-10, (0, 1), CEILING), + (1e-10, (0, 0), HALF_UP), + + (0.9999999999, (0, 999999999), FLOOR), + (0.9999999999, (1, 0), CEILING), + + (1.1234567890, (1, 123456789), FLOOR), + (1.1234567899, (1, 123456789), FLOOR), + (-1.1234567890, (-2, 876543211), FLOOR), + (-1.1234567891, (-2, 876543210), FLOOR), # Round towards infinity (+inf) - (0, (0, 0), _PyTime.ROUND_CEILING), - (-1, (-1, 0), _PyTime.ROUND_CEILING), - (-1.0, (-1, 0), _PyTime.ROUND_CEILING), - (1e-9, (0, 1), _PyTime.ROUND_CEILING), - (1e-10, (0, 1), _PyTime.ROUND_CEILING), - (-1e-9, (-1, 999999999), _PyTime.ROUND_CEILING), - (-1e-10, (0, 0), _PyTime.ROUND_CEILING), - (-1.2, (-2, 800000000), _PyTime.ROUND_CEILING), - (0.9999999999, (1, 0), _PyTime.ROUND_CEILING), - (1.1234567890, (1, 123456790), _PyTime.ROUND_CEILING), - (1.1234567899, (1, 123456790), _PyTime.ROUND_CEILING), - (-1.1234567890, (-2, 876543211), _PyTime.ROUND_CEILING), - (-1.1234567891, (-2, 876543211), _PyTime.ROUND_CEILING), + (1.1234567890, (1, 123456790), CEILING), + (1.1234567899, (1, 123456790), CEILING), + (-1.1234567890, (-2, 876543211), CEILING), + (-1.1234567891, (-2, 876543211), CEILING), + + # half up + (-0.6e-9, (-1, 999999999), HALF_UP), + # skipped, 0.5e-6 is inexact in base 2 + #(-0.5e-9, (-1, 999999999), HALF_UP), + (-0.4e-9, (0, 0), HALF_UP), + + (0.4e-9, (0, 0), HALF_UP), + (0.5e-9, (0, 1), HALF_UP), + (0.6e-9, (0, 1), HALF_UP), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) + # Test OverflowError rnd = _PyTime.ROUND_FLOOR for invalid in self.invalid_values: self.assertRaises(OverflowError, -- cgit v1.2.1 From 85ed934b48ca506799d4d531db87e5e3aea2fb96 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 4 Sep 2015 23:57:25 +0200 Subject: Issue #23517: Fix implementation of the ROUND_HALF_UP rounding mode in datetime.datetime.fromtimestamp() and datetime.datetime.utcfromtimestamp(). microseconds sign should be kept before rounding. --- Lib/datetime.py | 48 ++++++++++++++++++++-------------------------- Lib/test/datetimetester.py | 12 ++++++++++-- Lib/test/test_time.py | 13 +++++++------ 3 files changed, 38 insertions(+), 35 deletions(-) (limited to 'Lib') diff --git a/Lib/datetime.py b/Lib/datetime.py index 5ba2ddb7ab..3bf9edc5ad 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1374,28 +1374,34 @@ class datetime(date): return self._tzinfo @classmethod - def fromtimestamp(cls, t, tz=None): + def _fromtimestamp(cls, t, utc, tz): """Construct a datetime from a POSIX timestamp (like time.time()). A timezone info object may be passed in as well. """ - _check_tzinfo_arg(tz) - - converter = _time.localtime if tz is None else _time.gmtime - - t, frac = divmod(t, 1.0) + frac, t = _math.modf(t) us = _round_half_up(frac * 1e6) - - # If timestamp is less than one microsecond smaller than a - # full second, us can be rounded up to 1000000. In this case, - # roll over to seconds, otherwise, ValueError is raised - # by the constructor. - if us == 1000000: + if us >= 1000000: t += 1 - us = 0 + us -= 1000000 + elif us < 0: + t -= 1 + us += 1000000 + + converter = _time.gmtime if utc else _time.localtime y, m, d, hh, mm, ss, weekday, jday, dst = converter(t) ss = min(ss, 59) # clamp out leap seconds if the platform has them - result = cls(y, m, d, hh, mm, ss, us, tz) + return cls(y, m, d, hh, mm, ss, us, tz) + + @classmethod + def fromtimestamp(cls, t, tz=None): + """Construct a datetime from a POSIX timestamp (like time.time()). + + A timezone info object may be passed in as well. + """ + _check_tzinfo_arg(tz) + + result = cls._fromtimestamp(t, tz is not None, tz) if tz is not None: result = tz.fromutc(result) return result @@ -1403,19 +1409,7 @@ class datetime(date): @classmethod def utcfromtimestamp(cls, t): """Construct a naive UTC datetime from a POSIX timestamp.""" - t, frac = divmod(t, 1.0) - us = _round_half_up(frac * 1e6) - - # If timestamp is less than one microsecond smaller than a - # full second, us can be rounded up to 1000000. In this case, - # roll over to seconds, otherwise, ValueError is raised - # by the constructor. - if us == 1000000: - t += 1 - us = 0 - y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t) - ss = min(ss, 59) # clamp out leap seconds if the platform has them - return cls(y, m, d, hh, mm, ss, us) + return cls._fromtimestamp(t, True, None) @classmethod def now(cls, tz=None): diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index f516434bd6..58873af7f6 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -668,6 +668,8 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): eq(td(milliseconds=-0.6/1000), td(microseconds=-1)) eq(td(seconds=0.5/10**6), td(microseconds=1)) eq(td(seconds=-0.5/10**6), td(microseconds=-1)) + eq(td(seconds=1/2**7), td(microseconds=7813)) + eq(td(seconds=-1/2**7), td(microseconds=-7813)) # Rounding due to contributions from more than one field. us_per_hour = 3600e6 @@ -1842,8 +1844,8 @@ class TestDateTime(TestDate): 18000 + 3600 + 2*60 + 3 + 4*1e-6) def test_microsecond_rounding(self): - for fts in [self.theclass.fromtimestamp, - self.theclass.utcfromtimestamp]: + for fts in (datetime.fromtimestamp, + self.theclass.utcfromtimestamp): zero = fts(0) self.assertEqual(zero.second, 0) self.assertEqual(zero.microsecond, 0) @@ -1874,6 +1876,12 @@ class TestDateTime(TestDate): t = fts(0.9999999) self.assertEqual(t.second, 1) self.assertEqual(t.microsecond, 0) + t = fts(1/2**7) + self.assertEqual(t.second, 0) + self.assertEqual(t.microsecond, 7813) + t = fts(-1/2**7) + self.assertEqual(t.second, 59) + self.assertEqual(t.microsecond, 992187) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 590425ae90..75ab666faa 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -655,7 +655,7 @@ class TestPytime(unittest.TestCase): pytime_object_to_time_t, invalid, rnd) @support.cpython_only - def test_timespec(self): + def test_object_to_timespec(self): from _testcapi import pytime_object_to_timespec # Conversion giving the same result for all rounding methods @@ -666,7 +666,7 @@ class TestPytime(unittest.TestCase): (-1, (-1, 0)), # float - (-1.2, (-2, 800000000)), + (-1/2**7, (-1, 992187500)), (-1.0, (-1, 0)), (-1e-9, (-1, 999999999)), (1e-9, (0, 1)), @@ -693,7 +693,7 @@ class TestPytime(unittest.TestCase): (1.1234567890, (1, 123456789), FLOOR), (1.1234567899, (1, 123456789), FLOOR), - (-1.1234567890, (-2, 876543211), FLOOR), + (-1.1234567890, (-2, 876543210), FLOOR), (-1.1234567891, (-2, 876543210), FLOOR), # Round towards infinity (+inf) (1.1234567890, (1, 123456790), CEILING), @@ -1155,7 +1155,7 @@ class TestOldPyTime(unittest.TestCase): self.assertRaises(OverflowError, pytime_object_to_time_t, invalid, rnd) - def test_timeval(self): + def test_object_to_timeval(self): from _testcapi import pytime_object_to_timeval # Conversion giving the same result for all rounding methods @@ -1167,7 +1167,8 @@ class TestOldPyTime(unittest.TestCase): # float (-1.0, (-1, 0)), - (-1.2, (-2, 800000)), + (1/2**6, (0, 15625)), + (-1/2**6, (-1, 984375)), (-1e-6, (-1, 999999)), (1e-6, (0, 1)), ): @@ -1225,7 +1226,7 @@ class TestOldPyTime(unittest.TestCase): (-1.0, (-1, 0)), (-1e-9, (-1, 999999999)), (1e-9, (0, 1)), - (-1.2, (-2, 800000000)), + (-1/2**9, (-1, 998046875)), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), -- cgit v1.2.1 From 8f8a76ce7354a886f821b41f069221c5da40086f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 5 Sep 2015 10:50:20 +0200 Subject: Issue #23517: Skip a datetime test on Windows The test calls gmtime(-1)/localtime(-1) which is not supported on Windows. --- Lib/test/datetimetester.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 58873af7f6..b8c9b0db94 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1865,6 +1865,9 @@ class TestDateTime(TestDate): self.assertEqual(t, minus_one) t = fts(-1e-7) self.assertEqual(t, zero) + t = fts(-1/2**7) + self.assertEqual(t.second, 59) + self.assertEqual(t.microsecond, 992187) t = fts(1e-7) self.assertEqual(t, zero) @@ -1879,9 +1882,6 @@ class TestDateTime(TestDate): t = fts(1/2**7) self.assertEqual(t.second, 0) self.assertEqual(t.microsecond, 7813) - t = fts(-1/2**7) - self.assertEqual(t.second, 59) - self.assertEqual(t.microsecond, 992187) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, -- cgit v1.2.1 From abe3e1eb0d757e661158661bee7e1b678cb0380f Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Sun, 6 Sep 2015 13:07:21 -0400 Subject: Closes Issue#22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. --- Lib/datetime.py | 2 ++ Lib/test/datetimetester.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/datetime.py b/Lib/datetime.py index 3bf9edc5ad..6b2ac33a32 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1920,6 +1920,8 @@ class timezone(tzinfo): @staticmethod def _name_from_offset(delta): + if not delta: + return 'UTC' if delta < timedelta(0): sign = '-' delta = -delta diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index b8c9b0db94..f23add35db 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -258,7 +258,8 @@ class TestTimeZone(unittest.TestCase): with self.assertRaises(TypeError): self.EST.dst(5) def test_tzname(self): - self.assertEqual('UTC+00:00', timezone(ZERO).tzname(None)) + self.assertEqual('UTC', timezone.utc.tzname(None)) + self.assertEqual('UTC', timezone(ZERO).tzname(None)) self.assertEqual('UTC-05:00', timezone(-5 * HOUR).tzname(None)) self.assertEqual('UTC+09:30', timezone(9.5 * HOUR).tzname(None)) self.assertEqual('UTC-00:01', timezone(timedelta(minutes=-1)).tzname(None)) -- cgit v1.2.1 From bbd2c26bb917642217efd84fc4a3dd0b3e0f0a20 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 8 Sep 2015 23:58:54 +0200 Subject: Revert change 0eb8c182131e: """Issue #23517: datetime.timedelta constructor now rounds microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of rounding to nearest with ties going to nearest even integer (ROUND_HALF_EVEN).""" datetime.timedelta uses rounding mode ROUND_HALF_EVEN again. --- Lib/datetime.py | 4 ++-- Lib/test/datetimetester.py | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) (limited to 'Lib') diff --git a/Lib/datetime.py b/Lib/datetime.py index 6b2ac33a32..3c25ef84c6 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -407,7 +407,7 @@ class timedelta: # secondsfrac isn't referenced again if isinstance(microseconds, float): - microseconds = _round_half_up(microseconds + usdouble) + microseconds = round(microseconds + usdouble) seconds, microseconds = divmod(microseconds, 1000000) days, seconds = divmod(seconds, 24*3600) d += days @@ -418,7 +418,7 @@ class timedelta: days, seconds = divmod(seconds, 24*3600) d += days s += seconds - microseconds = _round_half_up(microseconds + usdouble) + microseconds = round(microseconds + usdouble) assert isinstance(s, int) assert isinstance(microseconds, int) assert abs(s) <= 3 * 24 * 3600 diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index f23add35db..d87b106cff 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -663,14 +663,16 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): # Single-field rounding. eq(td(milliseconds=0.4/1000), td(0)) # rounds to 0 eq(td(milliseconds=-0.4/1000), td(0)) # rounds to 0 - eq(td(milliseconds=0.5/1000), td(microseconds=1)) - eq(td(milliseconds=-0.5/1000), td(microseconds=-1)) + eq(td(milliseconds=0.5/1000), td(microseconds=0)) + eq(td(milliseconds=-0.5/1000), td(microseconds=-0)) eq(td(milliseconds=0.6/1000), td(microseconds=1)) eq(td(milliseconds=-0.6/1000), td(microseconds=-1)) - eq(td(seconds=0.5/10**6), td(microseconds=1)) - eq(td(seconds=-0.5/10**6), td(microseconds=-1)) - eq(td(seconds=1/2**7), td(microseconds=7813)) - eq(td(seconds=-1/2**7), td(microseconds=-7813)) + eq(td(milliseconds=1.5/1000), td(microseconds=2)) + eq(td(milliseconds=-1.5/1000), td(microseconds=-2)) + eq(td(seconds=0.5/10**6), td(microseconds=0)) + eq(td(seconds=-0.5/10**6), td(microseconds=-0)) + eq(td(seconds=1/2**7), td(microseconds=7812)) + eq(td(seconds=-1/2**7), td(microseconds=-7812)) # Rounding due to contributions from more than one field. us_per_hour = 3600e6 @@ -683,6 +685,10 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): eq(td(hours=-.2/us_per_hour), td(0)) eq(td(days=-.4/us_per_day, hours=-.2/us_per_hour), td(microseconds=-1)) + # Test for a patch in Issue 8860 + eq(td(microseconds=0.5), 0.5*td(microseconds=1.0)) + eq(td(microseconds=0.5)//td.resolution, 0.5*td.resolution//td.resolution) + def test_massive_normalization(self): td = timedelta(microseconds=-1) self.assertEqual((td.days, td.seconds, td.microseconds), -- cgit v1.2.1 From fb8809cbfa9e3302900ffb10b732626ce864157c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2015 01:02:23 +0200 Subject: Issue #23517: fromtimestamp() and utcfromtimestamp() methods of datetime.datetime now round microseconds to nearest with ties going to nearest even integer (ROUND_HALF_EVEN), as round(float), instead of rounding towards -Infinity (ROUND_FLOOR). pytime API: replace _PyTime_ROUND_HALF_UP with _PyTime_ROUND_HALF_EVEN. Fix also _PyTime_Divide() for negative numbers. _PyTime_AsTimeval_impl() now reuses _PyTime_Divide() instead of reimplementing rounding modes. --- Lib/datetime.py | 2 +- Lib/test/datetimetester.py | 4 +- Lib/test/test_time.py | 245 ++++++++++++++++++++------------------------- 3 files changed, 113 insertions(+), 138 deletions(-) (limited to 'Lib') diff --git a/Lib/datetime.py b/Lib/datetime.py index 3c25ef84c6..3f29bc4b7a 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1380,7 +1380,7 @@ class datetime(date): A timezone info object may be passed in as well. """ frac, t = _math.modf(t) - us = _round_half_up(frac * 1e6) + us = round(frac * 1e6) if us >= 1000000: t += 1 us -= 1000000 diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index d87b106cff..467fbe2f45 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1874,7 +1874,7 @@ class TestDateTime(TestDate): self.assertEqual(t, zero) t = fts(-1/2**7) self.assertEqual(t.second, 59) - self.assertEqual(t.microsecond, 992187) + self.assertEqual(t.microsecond, 992188) t = fts(1e-7) self.assertEqual(t, zero) @@ -1888,7 +1888,7 @@ class TestDateTime(TestDate): self.assertEqual(t.microsecond, 0) t = fts(1/2**7) self.assertEqual(t.second, 0) - self.assertEqual(t.microsecond, 7813) + self.assertEqual(t.microsecond, 7812) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index d68dc4f1c6..493b197ba8 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -30,11 +30,11 @@ class _PyTime(enum.IntEnum): ROUND_FLOOR = 0 # Round towards infinity (+inf) ROUND_CEILING = 1 - # Round to nearest with ties going away from zero - ROUND_HALF_UP = 2 + # Round to nearest with ties going to nearest even integer + ROUND_HALF_EVEN = 2 ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING, - _PyTime.ROUND_HALF_UP) + _PyTime.ROUND_HALF_EVEN) class TimeTestCase(unittest.TestCase): @@ -639,27 +639,26 @@ class TestPytime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP - for obj, time_t, rnd in ( + HALF_EVEN = _PyTime.ROUND_HALF_EVEN + for obj, seconds, rnd in ( (-1.9, -2, FLOOR), (-1.9, -1, CEILING), - (-1.9, -2, HALF_UP), + (-1.9, -2, HALF_EVEN), (1.9, 1, FLOOR), (1.9, 2, CEILING), - (1.9, 2, HALF_UP), - - # half up - (-0.999, -1, HALF_UP), - (-0.510, -1, HALF_UP), - (-0.500, -1, HALF_UP), - (-0.490, 0, HALF_UP), - ( 0.490, 0, HALF_UP), - ( 0.500, 1, HALF_UP), - ( 0.510, 1, HALF_UP), - ( 0.999, 1, HALF_UP), + (1.9, 2, HALF_EVEN), + + # half even + (-1.5, -2, HALF_EVEN), + (-0.9, -1, HALF_EVEN), + (-0.5, 0, HALF_EVEN), + ( 0.5, 0, HALF_EVEN), + ( 0.9, 1, HALF_EVEN), + ( 1.5, 2, HALF_EVEN), ): - self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) + with self.subTest(obj=obj, round=rnd, seconds=seconds): + self.assertEqual(pytime_object_to_time_t(obj, rnd), seconds) # Test OverflowError rnd = _PyTime.ROUND_FLOOR @@ -691,15 +690,15 @@ class TestPytime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, timespec, rnd in ( # Round towards minus infinity (-inf) (-1e-10, (0, 0), CEILING), (-1e-10, (-1, 999999999), FLOOR), - (-1e-10, (0, 0), HALF_UP), + (-1e-10, (0, 0), HALF_EVEN), (1e-10, (0, 0), FLOOR), (1e-10, (0, 1), CEILING), - (1e-10, (0, 0), HALF_UP), + (1e-10, (0, 0), HALF_EVEN), (0.9999999999, (0, 999999999), FLOOR), (0.9999999999, (1, 0), CEILING), @@ -714,15 +713,13 @@ class TestPytime(unittest.TestCase): (-1.1234567890, (-2, 876543211), CEILING), (-1.1234567891, (-2, 876543211), CEILING), - # half up - (-0.6e-9, (-1, 999999999), HALF_UP), - # skipped, 0.5e-6 is inexact in base 2 - #(-0.5e-9, (-1, 999999999), HALF_UP), - (-0.4e-9, (0, 0), HALF_UP), - - (0.4e-9, (0, 0), HALF_UP), - (0.5e-9, (0, 1), HALF_UP), - (0.6e-9, (0, 1), HALF_UP), + # half even + (-1.5e-9, (-1, 999999998), HALF_EVEN), + (-0.9e-9, (-1, 999999999), HALF_EVEN), + (-0.5e-9, (0, 0), HALF_EVEN), + (0.5e-9, (0, 0), HALF_EVEN), + (0.9e-9, (0, 1), HALF_EVEN), + (1.5e-9, (0, 2), HALF_EVEN), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) @@ -823,10 +820,10 @@ class TestPyTime_t(unittest.TestCase): (-7.0, -7 * SEC_TO_NS), # nanosecond are kept for value <= 2^23 seconds, - # except 2**23-1e-9 with HALF_UP (2**22 - 1e-9, 4194303999999999), (2**22, 4194304000000000), (2**22 + 1e-9, 4194304000000001), + (2**23 - 1e-9, 8388607999999999), (2**23, 8388608000000000), # start loosing precision for value > 2^23 seconds @@ -859,38 +856,31 @@ class TestPyTime_t(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, ts, rnd in ( # close to zero ( 1e-10, 0, FLOOR), ( 1e-10, 1, CEILING), - ( 1e-10, 0, HALF_UP), + ( 1e-10, 0, HALF_EVEN), (-1e-10, -1, FLOOR), (-1e-10, 0, CEILING), - (-1e-10, 0, HALF_UP), + (-1e-10, 0, HALF_EVEN), # test rounding of the last nanosecond ( 1.1234567899, 1123456789, FLOOR), ( 1.1234567899, 1123456790, CEILING), - ( 1.1234567899, 1123456790, HALF_UP), + ( 1.1234567899, 1123456790, HALF_EVEN), (-1.1234567899, -1123456790, FLOOR), (-1.1234567899, -1123456789, CEILING), - (-1.1234567899, -1123456790, HALF_UP), + (-1.1234567899, -1123456790, HALF_EVEN), # close to 1 second ( 0.9999999999, 999999999, FLOOR), ( 0.9999999999, 1000000000, CEILING), - ( 0.9999999999, 1000000000, HALF_UP), + ( 0.9999999999, 1000000000, HALF_EVEN), (-0.9999999999, -1000000000, FLOOR), (-0.9999999999, -999999999, CEILING), - (-0.9999999999, -1000000000, HALF_UP), - - # close to 2^23 seconds - (2**23 - 1e-9, 8388607999999999, FLOOR), - (2**23 - 1e-9, 8388607999999999, CEILING), - # Issue #23517: skip HALF_UP test because the result is different - # depending on the FPU and how the compiler optimize the code :-/ - #(2**23 - 1e-9, 8388608000000000, HALF_UP), + (-0.9999999999, -1000000000, HALF_EVEN), ): with self.subTest(obj=obj, round=rnd, timestamp=ts): self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) @@ -958,33 +948,23 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for ns, tv, rnd in ( # nanoseconds (1, (0, 0), FLOOR), (1, (0, 1), CEILING), - (1, (0, 0), HALF_UP), + (1, (0, 0), HALF_EVEN), (-1, (-1, 999999), FLOOR), (-1, (0, 0), CEILING), - (-1, (0, 0), HALF_UP), - - # seconds + nanoseconds - (1234567001, (1, 234567), FLOOR), - (1234567001, (1, 234568), CEILING), - (1234567001, (1, 234567), HALF_UP), - (-1234567001, (-2, 765432), FLOOR), - (-1234567001, (-2, 765433), CEILING), - (-1234567001, (-2, 765433), HALF_UP), - - # half up - (499, (0, 0), HALF_UP), - (500, (0, 1), HALF_UP), - (501, (0, 1), HALF_UP), - (999, (0, 1), HALF_UP), - (-499, (0, 0), HALF_UP), - (-500, (0, 0), HALF_UP), - (-501, (-1, 999999), HALF_UP), - (-999, (-1, 999999), HALF_UP), + (-1, (0, 0), HALF_EVEN), + + # half even + (-1500, (-1, 999998), HALF_EVEN), + (-999, (-1, 999999), HALF_EVEN), + (-500, (0, 0), HALF_EVEN), + (500, (0, 0), HALF_EVEN), + (999, (0, 1), HALF_EVEN), + (1500, (0, 2), HALF_EVEN), ): with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) @@ -1027,33 +1007,31 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for ns, ms, rnd in ( # nanoseconds (1, 0, FLOOR), (1, 1, CEILING), - (1, 0, HALF_UP), - (-1, 0, FLOOR), - (-1, -1, CEILING), - (-1, 0, HALF_UP), + (1, 0, HALF_EVEN), + (-1, -1, FLOOR), + (-1, 0, CEILING), + (-1, 0, HALF_EVEN), # seconds + nanoseconds (1234 * MS_TO_NS + 1, 1234, FLOOR), (1234 * MS_TO_NS + 1, 1235, CEILING), - (1234 * MS_TO_NS + 1, 1234, HALF_UP), - (-1234 * MS_TO_NS - 1, -1234, FLOOR), - (-1234 * MS_TO_NS - 1, -1235, CEILING), - (-1234 * MS_TO_NS - 1, -1234, HALF_UP), + (1234 * MS_TO_NS + 1, 1234, HALF_EVEN), + (-1234 * MS_TO_NS - 1, -1235, FLOOR), + (-1234 * MS_TO_NS - 1, -1234, CEILING), + (-1234 * MS_TO_NS - 1, -1234, HALF_EVEN), # half up - (499999, 0, HALF_UP), - (499999, 0, HALF_UP), - (500000, 1, HALF_UP), - (999999, 1, HALF_UP), - (-499999, 0, HALF_UP), - (-500000, -1, HALF_UP), - (-500001, -1, HALF_UP), - (-999999, -1, HALF_UP), + (-1500000, -2, HALF_EVEN), + (-999999, -1, HALF_EVEN), + (-500000, 0, HALF_EVEN), + (500000, 0, HALF_EVEN), + (999999, 1, HALF_EVEN), + (1500000, 2, HALF_EVEN), ): with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): self.assertEqual(PyTime_AsMilliseconds(ns, rnd), ms) @@ -1079,31 +1057,31 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for ns, ms, rnd in ( # nanoseconds (1, 0, FLOOR), (1, 1, CEILING), - (1, 0, HALF_UP), - (-1, 0, FLOOR), - (-1, -1, CEILING), - (-1, 0, HALF_UP), + (1, 0, HALF_EVEN), + (-1, -1, FLOOR), + (-1, 0, CEILING), + (-1, 0, HALF_EVEN), # seconds + nanoseconds (1234 * US_TO_NS + 1, 1234, FLOOR), (1234 * US_TO_NS + 1, 1235, CEILING), - (1234 * US_TO_NS + 1, 1234, HALF_UP), - (-1234 * US_TO_NS - 1, -1234, FLOOR), - (-1234 * US_TO_NS - 1, -1235, CEILING), - (-1234 * US_TO_NS - 1, -1234, HALF_UP), + (1234 * US_TO_NS + 1, 1234, HALF_EVEN), + (-1234 * US_TO_NS - 1, -1235, FLOOR), + (-1234 * US_TO_NS - 1, -1234, CEILING), + (-1234 * US_TO_NS - 1, -1234, HALF_EVEN), # half up - (1499, 1, HALF_UP), - (1500, 2, HALF_UP), - (1501, 2, HALF_UP), - (-1499, -1, HALF_UP), - (-1500, -2, HALF_UP), - (-1501, -2, HALF_UP), + (-1500, -2, HALF_EVEN), + (-999, -1, HALF_EVEN), + (-500, 0, HALF_EVEN), + (500, 0, HALF_EVEN), + (999, 1, HALF_EVEN), + (1500, 2, HALF_EVEN), ): with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): self.assertEqual(PyTime_AsMicroseconds(ns, rnd), ms) @@ -1142,23 +1120,23 @@ class TestOldPyTime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, time_t, rnd in ( (-1.9, -2, FLOOR), - (-1.9, -2, HALF_UP), + (-1.9, -2, HALF_EVEN), (-1.9, -1, CEILING), (1.9, 1, FLOOR), - (1.9, 2, HALF_UP), + (1.9, 2, HALF_EVEN), (1.9, 2, CEILING), - (-0.6, -1, HALF_UP), - (-0.5, -1, HALF_UP), - (-0.4, 0, HALF_UP), - - (0.4, 0, HALF_UP), - (0.5, 1, HALF_UP), - (0.6, 1, HALF_UP), + # half even + (-1.5, -2, HALF_EVEN), + (-0.9, -1, HALF_EVEN), + (-0.5, 0, HALF_EVEN), + ( 0.5, 0, HALF_EVEN), + ( 0.9, 1, HALF_EVEN), + ( 1.5, 2, HALF_EVEN), ): self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) @@ -1192,29 +1170,27 @@ class TestOldPyTime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, timeval, rnd in ( (-1e-7, (-1, 999999), FLOOR), (-1e-7, (0, 0), CEILING), - (-1e-7, (0, 0), HALF_UP), + (-1e-7, (0, 0), HALF_EVEN), (1e-7, (0, 0), FLOOR), (1e-7, (0, 1), CEILING), - (1e-7, (0, 0), HALF_UP), + (1e-7, (0, 0), HALF_EVEN), (0.9999999, (0, 999999), FLOOR), (0.9999999, (1, 0), CEILING), - (0.9999999, (1, 0), HALF_UP), - - (-0.6e-6, (-1, 999999), HALF_UP), - # skipped, -0.5e-6 is inexact in base 2 - #(-0.5e-6, (-1, 999999), HALF_UP), - (-0.4e-6, (0, 0), HALF_UP), - - (0.4e-6, (0, 0), HALF_UP), - # skipped, 0.5e-6 is inexact in base 2 - #(0.5e-6, (0, 1), HALF_UP), - (0.6e-6, (0, 1), HALF_UP), + (0.9999999, (1, 0), HALF_EVEN), + + # half even + (-1.5e-6, (-1, 999998), HALF_EVEN), + (-0.9e-6, (-1, 999999), HALF_EVEN), + (-0.5e-6, (0, 0), HALF_EVEN), + (0.5e-6, (0, 0), HALF_EVEN), + (0.9e-6, (0, 1), HALF_EVEN), + (1.5e-6, (0, 2), HALF_EVEN), ): with self.subTest(obj=obj, round=rnd, timeval=timeval): self.assertEqual(pytime_object_to_timeval(obj, rnd), timeval) @@ -1248,28 +1224,27 @@ class TestOldPyTime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, timespec, rnd in ( (-1e-10, (-1, 999999999), FLOOR), (-1e-10, (0, 0), CEILING), - (-1e-10, (0, 0), HALF_UP), + (-1e-10, (0, 0), HALF_EVEN), (1e-10, (0, 0), FLOOR), (1e-10, (0, 1), CEILING), - (1e-10, (0, 0), HALF_UP), + (1e-10, (0, 0), HALF_EVEN), (0.9999999999, (0, 999999999), FLOOR), (0.9999999999, (1, 0), CEILING), - (0.9999999999, (1, 0), HALF_UP), - - (-0.6e-9, (-1, 999999999), HALF_UP), - # skipped, 0.5e-6 is inexact in base 2 - #(-0.5e-9, (-1, 999999999), HALF_UP), - (-0.4e-9, (0, 0), HALF_UP), - - (0.4e-9, (0, 0), HALF_UP), - (0.5e-9, (0, 1), HALF_UP), - (0.6e-9, (0, 1), HALF_UP), + (0.9999999999, (1, 0), HALF_EVEN), + + # half even + (-1.5e-9, (-1, 999999998), HALF_EVEN), + (-0.9e-9, (-1, 999999999), HALF_EVEN), + (-0.5e-9, (0, 0), HALF_EVEN), + (0.5e-9, (0, 0), HALF_EVEN), + (0.9e-9, (0, 1), HALF_EVEN), + (1.5e-9, (0, 2), HALF_EVEN), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) -- cgit v1.2.1 From daa6152baf97381bc3f58c4fb2c7db73369bb46b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2015 01:09:21 +0200 Subject: cleanup datetime code remove scories of round half up code and debug code. --- Lib/datetime.py | 7 ------- Lib/test/datetimetester.py | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) (limited to 'Lib') diff --git a/Lib/datetime.py b/Lib/datetime.py index 3f29bc4b7a..b734a743d0 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -316,13 +316,6 @@ def _divide_and_round(a, b): return q -def _round_half_up(x): - """Round to nearest with ties going away from zero.""" - if x >= 0.0: - return _math.floor(x + 0.5) - else: - return _math.ceil(x - 0.5) - class timedelta: """Represent the difference between two datetime objects. diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 467fbe2f45..5d2df323a9 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -679,7 +679,7 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): us_per_day = us_per_hour * 24 eq(td(days=.4/us_per_day), td(0)) eq(td(hours=.2/us_per_hour), td(0)) - eq(td(days=.4/us_per_day, hours=.2/us_per_hour), td(microseconds=1), td) + eq(td(days=.4/us_per_day, hours=.2/us_per_hour), td(microseconds=1)) eq(td(days=-.4/us_per_day), td(0)) eq(td(hours=-.2/us_per_hour), td(0)) -- cgit v1.2.1 From 00d73959796abf8ab71adefa9bbc6f46d2ab9cba Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2015 22:32:48 +0200 Subject: test_time: rewrite PyTime API rounding tests Drop all hardcoded tests. Instead, reimplement each function in Python, usually using decimal.Decimal for the rounding mode. Add much more values to the dataset. Test various timestamp units from picroseconds to seconds, in integer and float. Enhance also _PyTime_AsSecondsDouble(). --- Lib/test/test_time.py | 796 +++++++++++++++----------------------------------- 1 file changed, 234 insertions(+), 562 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 493b197ba8..bd697aeb13 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1,6 +1,8 @@ from test import support +import decimal import enum import locale +import math import platform import sys import sysconfig @@ -21,9 +23,11 @@ SIZEOF_INT = sysconfig.get_config_var('SIZEOF_INT') or 4 TIME_MAXYEAR = (1 << 8 * SIZEOF_INT - 1) - 1 TIME_MINYEAR = -TIME_MAXYEAR - 1 +SEC_TO_US = 10 ** 6 US_TO_NS = 10 ** 3 MS_TO_NS = 10 ** 6 SEC_TO_NS = 10 ** 9 +NS_TO_SEC = 10 ** 9 class _PyTime(enum.IntEnum): # Round towards minus infinity (-inf) @@ -33,8 +37,13 @@ class _PyTime(enum.IntEnum): # Round to nearest with ties going to nearest even integer ROUND_HALF_EVEN = 2 -ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING, - _PyTime.ROUND_HALF_EVEN) +# Rounding modes supported by PyTime +ROUNDING_MODES = ( + # (PyTime rounding method, decimal rounding method) + (_PyTime.ROUND_FLOOR, decimal.ROUND_FLOOR), + (_PyTime.ROUND_CEILING, decimal.ROUND_CEILING), + (_PyTime.ROUND_HALF_EVEN, decimal.ROUND_HALF_EVEN), +) class TimeTestCase(unittest.TestCase): @@ -610,126 +619,6 @@ class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear, unittest.TestCase): class TestPytime(unittest.TestCase): - def setUp(self): - self.invalid_values = ( - -(2 ** 100), 2 ** 100, - -(2.0 ** 100.0), 2.0 ** 100.0, - ) - - @support.cpython_only - def test_time_t(self): - from _testcapi import pytime_object_to_time_t - - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, seconds in ( - # int - (-1, -1), - (0, 0), - (1, 1), - - # float - (-1.0, -1), - (1.0, 1), - ): - with self.subTest(obj=obj, round=rnd, seconds=seconds): - self.assertEqual(pytime_object_to_time_t(obj, rnd), - seconds) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, seconds, rnd in ( - (-1.9, -2, FLOOR), - (-1.9, -1, CEILING), - (-1.9, -2, HALF_EVEN), - - (1.9, 1, FLOOR), - (1.9, 2, CEILING), - (1.9, 2, HALF_EVEN), - - # half even - (-1.5, -2, HALF_EVEN), - (-0.9, -1, HALF_EVEN), - (-0.5, 0, HALF_EVEN), - ( 0.5, 0, HALF_EVEN), - ( 0.9, 1, HALF_EVEN), - ( 1.5, 2, HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, seconds=seconds): - self.assertEqual(pytime_object_to_time_t(obj, rnd), seconds) - - # Test OverflowError - rnd = _PyTime.ROUND_FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_time_t, invalid, rnd) - - @support.cpython_only - def test_object_to_timespec(self): - from _testcapi import pytime_object_to_timespec - - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, timespec in ( - # int - (0, (0, 0)), - (-1, (-1, 0)), - - # float - (-1/2**7, (-1, 992187500)), - (-1.0, (-1, 0)), - (-1e-9, (-1, 999999999)), - (1e-9, (0, 1)), - (1.0, (1, 0)), - ): - with self.subTest(obj=obj, round=rnd, timespec=timespec): - self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, timespec, rnd in ( - # Round towards minus infinity (-inf) - (-1e-10, (0, 0), CEILING), - (-1e-10, (-1, 999999999), FLOOR), - (-1e-10, (0, 0), HALF_EVEN), - (1e-10, (0, 0), FLOOR), - (1e-10, (0, 1), CEILING), - (1e-10, (0, 0), HALF_EVEN), - - (0.9999999999, (0, 999999999), FLOOR), - (0.9999999999, (1, 0), CEILING), - - (1.1234567890, (1, 123456789), FLOOR), - (1.1234567899, (1, 123456789), FLOOR), - (-1.1234567890, (-2, 876543210), FLOOR), - (-1.1234567891, (-2, 876543210), FLOOR), - # Round towards infinity (+inf) - (1.1234567890, (1, 123456790), CEILING), - (1.1234567899, (1, 123456790), CEILING), - (-1.1234567890, (-2, 876543211), CEILING), - (-1.1234567891, (-2, 876543211), CEILING), - - # half even - (-1.5e-9, (-1, 999999998), HALF_EVEN), - (-0.9e-9, (-1, 999999999), HALF_EVEN), - (-0.5e-9, (0, 0), HALF_EVEN), - (0.5e-9, (0, 0), HALF_EVEN), - (0.9e-9, (0, 1), HALF_EVEN), - (1.5e-9, (0, 2), HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, timespec=timespec): - self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) - - # Test OverflowError - rnd = _PyTime.ROUND_FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_timespec, invalid, rnd) - @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_localtime_timezone(self): @@ -784,476 +673,259 @@ class TestPytime(unittest.TestCase): self.assertIs(lt.tm_zone, None) -@unittest.skipUnless(_testcapi is not None, - 'need the _testcapi module') -class TestPyTime_t(unittest.TestCase): +@unittest.skipIf(_testcapi is None, 'need the _testcapi module') +class CPyTimeTestCase: """ - Test the _PyTime_t API. + Base class to test the C _PyTime_t API. """ + OVERFLOW_SECONDS = None + + def _rounding_values(self, use_float): + "Build timestamps used to test rounding." + + units = [1, US_TO_NS, MS_TO_NS, SEC_TO_NS] + if use_float: + # picoseconds are only tested to pytime_converter accepting floats + units.append(1e-3) + + values = ( + # small values + 1, 2, 5, 7, 123, 456, 1234, + # 10^k - 1 + 9, + 99, + 999, + 9999, + 99999, + 999999, + # test half even rounding near 0.5, 1.5, 2.5, 3.5, 4.5 + 499, 500, 501, + 1499, 1500, 1501, + 2500, + 3500, + 4500, + ) + + ns_timestamps = [0] + for unit in units: + for value in values: + ns = value * unit + ns_timestamps.extend((-ns, ns)) + for pow2 in (0, 5, 10, 15, 22, 23, 24, 30, 33): + ns = (2 ** pow2) * SEC_TO_NS + ns_timestamps.extend(( + -ns-1, -ns, -ns+1, + ns-1, ns, ns+1 + )) + for seconds in (_testcapi.INT_MIN, _testcapi.INT_MAX): + ns_timestamps.append(seconds * SEC_TO_NS) + if use_float: + # numbers with an extract representation in IEEE 754 (base 2) + for pow2 in (3, 7, 10, 15): + ns = 2.0 ** (-pow2) + ns_timestamps.extend((-ns, ns)) + + # seconds close to _PyTime_t type limit + ns = (2 ** 63 // SEC_TO_NS) * SEC_TO_NS + ns_timestamps.extend((-ns, ns)) + + return ns_timestamps + + def _check_rounding(self, pytime_converter, expected_func, + use_float, unit_to_sec, value_filter=None): + + def convert_values(ns_timestamps): + if use_float: + unit_to_ns = SEC_TO_NS / float(unit_to_sec) + values = [ns / unit_to_ns for ns in ns_timestamps] + else: + unit_to_ns = SEC_TO_NS // unit_to_sec + values = [ns // unit_to_ns for ns in ns_timestamps] + + if value_filter: + values = filter(value_filter, values) + + # remove duplicates and sort + return sorted(set(values)) + + # test rounding + ns_timestamps = self._rounding_values(use_float) + valid_values = convert_values(ns_timestamps) + for time_rnd, decimal_rnd in ROUNDING_MODES : + context = decimal.getcontext() + context.rounding = decimal_rnd + + for value in valid_values: + expected = expected_func(value) + self.assertEqual(pytime_converter(value, time_rnd), + expected, + {'value': value, 'rounding': decimal_rnd}) + + # test overflow + ns = self.OVERFLOW_SECONDS * SEC_TO_NS + ns_timestamps = (-ns, ns) + overflow_values = convert_values(ns_timestamps) + for time_rnd, _ in ROUNDING_MODES : + for value in overflow_values: + with self.assertRaises(OverflowError): + pytime_converter(value, time_rnd) + + def check_int_rounding(self, pytime_converter, expected_func, unit_to_sec=1, + value_filter=None): + self._check_rounding(pytime_converter, expected_func, + False, unit_to_sec, value_filter) + + def check_float_rounding(self, pytime_converter, expected_func, unit_to_sec=1): + self._check_rounding(pytime_converter, expected_func, + True, unit_to_sec) + + def decimal_round(self, x): + d = decimal.Decimal(x) + d = d.quantize(1) + return int(d) + + +class TestCPyTime(CPyTimeTestCase, unittest.TestCase): + """ + Test the C _PyTime_t API. + """ + # _PyTime_t is a 64-bit signed integer + OVERFLOW_SECONDS = math.ceil((2**63 + 1) / SEC_TO_NS) + def test_FromSeconds(self): from _testcapi import PyTime_FromSeconds - for seconds in (0, 3, -456, _testcapi.INT_MAX, _testcapi.INT_MIN): - with self.subTest(seconds=seconds): - self.assertEqual(PyTime_FromSeconds(seconds), - seconds * SEC_TO_NS) + + # PyTime_FromSeconds() expects a C int, reject values out of range + def c_int_filter(secs): + return (_testcapi.INT_MIN <= secs <= _testcapi.INT_MAX) + + self.check_int_rounding(lambda secs, rnd: PyTime_FromSeconds(secs), + lambda secs: secs * SEC_TO_NS, + value_filter=c_int_filter) def test_FromSecondsObject(self): from _testcapi import PyTime_FromSecondsObject - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, ts in ( - # integers - (0, 0), - (1, SEC_TO_NS), - (-3, -3 * SEC_TO_NS), - - # float: subseconds - (0.0, 0), - (1e-9, 1), - (1e-6, 10 ** 3), - (1e-3, 10 ** 6), - - # float: seconds - (2.0, 2 * SEC_TO_NS), - (123.0, 123 * SEC_TO_NS), - (-7.0, -7 * SEC_TO_NS), - - # nanosecond are kept for value <= 2^23 seconds, - (2**22 - 1e-9, 4194303999999999), - (2**22, 4194304000000000), - (2**22 + 1e-9, 4194304000000001), - (2**23 - 1e-9, 8388607999999999), - (2**23, 8388608000000000), - - # start loosing precision for value > 2^23 seconds - (2**23 + 1e-9, 8388608000000002), - - # nanoseconds are lost for value > 2^23 seconds - (2**24 - 1e-9, 16777215999999998), - (2**24, 16777216000000000), - (2**24 + 1e-9, 16777216000000000), - (2**25 - 1e-9, 33554432000000000), - (2**25 , 33554432000000000), - (2**25 + 1e-9, 33554432000000000), - - # close to 2^63 nanoseconds (_PyTime_t limit) - (9223372036, 9223372036 * SEC_TO_NS), - (9223372036.0, 9223372036 * SEC_TO_NS), - (-9223372036, -9223372036 * SEC_TO_NS), - (-9223372036.0, -9223372036 * SEC_TO_NS), - ): - with self.subTest(obj=obj, round=rnd, timestamp=ts): - self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) - - with self.subTest(round=rnd): - with self.assertRaises(OverflowError): - PyTime_FromSecondsObject(9223372037, rnd) - PyTime_FromSecondsObject(9223372037.0, rnd) - PyTime_FromSecondsObject(-9223372037, rnd) - PyTime_FromSecondsObject(-9223372037.0, rnd) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, ts, rnd in ( - # close to zero - ( 1e-10, 0, FLOOR), - ( 1e-10, 1, CEILING), - ( 1e-10, 0, HALF_EVEN), - (-1e-10, -1, FLOOR), - (-1e-10, 0, CEILING), - (-1e-10, 0, HALF_EVEN), - - # test rounding of the last nanosecond - ( 1.1234567899, 1123456789, FLOOR), - ( 1.1234567899, 1123456790, CEILING), - ( 1.1234567899, 1123456790, HALF_EVEN), - (-1.1234567899, -1123456790, FLOOR), - (-1.1234567899, -1123456789, CEILING), - (-1.1234567899, -1123456790, HALF_EVEN), - - # close to 1 second - ( 0.9999999999, 999999999, FLOOR), - ( 0.9999999999, 1000000000, CEILING), - ( 0.9999999999, 1000000000, HALF_EVEN), - (-0.9999999999, -1000000000, FLOOR), - (-0.9999999999, -999999999, CEILING), - (-0.9999999999, -1000000000, HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, timestamp=ts): - self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) + self.check_int_rounding( + PyTime_FromSecondsObject, + lambda secs: secs * SEC_TO_NS) + + self.check_float_rounding( + PyTime_FromSecondsObject, + lambda ns: self.decimal_round(ns * SEC_TO_NS)) def test_AsSecondsDouble(self): from _testcapi import PyTime_AsSecondsDouble - for nanoseconds, seconds in ( - # near 1 nanosecond - ( 0, 0.0), - ( 1, 1e-9), - (-1, -1e-9), - - # near 1 second - (SEC_TO_NS + 1, 1.0 + 1e-9), - (SEC_TO_NS, 1.0), - (SEC_TO_NS - 1, 1.0 - 1e-9), - - # a few seconds - (123 * SEC_TO_NS, 123.0), - (-567 * SEC_TO_NS, -567.0), - - # nanosecond are kept for value <= 2^23 seconds - (4194303999999999, 2**22 - 1e-9), - (4194304000000000, 2**22), - (4194304000000001, 2**22 + 1e-9), - - # start loosing precision for value > 2^23 seconds - (8388608000000002, 2**23 + 1e-9), - - # nanoseconds are lost for value > 2^23 seconds - (16777215999999998, 2**24 - 1e-9), - (16777215999999999, 2**24 - 1e-9), - (16777216000000000, 2**24 ), - (16777216000000001, 2**24 ), - (16777216000000002, 2**24 + 2e-9), - - (33554432000000000, 2**25 ), - (33554432000000002, 2**25 ), - (33554432000000004, 2**25 + 4e-9), - - # close to 2^63 nanoseconds (_PyTime_t limit) - (9223372036 * SEC_TO_NS, 9223372036.0), - (-9223372036 * SEC_TO_NS, -9223372036.0), - ): - with self.subTest(nanoseconds=nanoseconds, seconds=seconds): - self.assertEqual(PyTime_AsSecondsDouble(nanoseconds), - seconds) - - def test_timeval(self): + def float_converter(ns): + if abs(ns) % SEC_TO_NS == 0: + return float(ns // SEC_TO_NS) + else: + return float(ns) / SEC_TO_NS + + self.check_int_rounding(lambda ns, rnd: PyTime_AsSecondsDouble(ns), + float_converter, + NS_TO_SEC) + + def create_decimal_converter(self, denominator): + denom = decimal.Decimal(denominator) + + def converter(value): + d = decimal.Decimal(value) / denom + return self.decimal_round(d) + + return converter + + def test_AsTimeval(self): from _testcapi import PyTime_AsTimeval - for rnd in ALL_ROUNDING_METHODS: - for ns, tv in ( - # microseconds - (0, (0, 0)), - (1000, (0, 1)), - (-1000, (-1, 999999)), - - # seconds - (2 * SEC_TO_NS, (2, 0)), - (-3 * SEC_TO_NS, (-3, 0)), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) - - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for ns, tv, rnd in ( - # nanoseconds - (1, (0, 0), FLOOR), - (1, (0, 1), CEILING), - (1, (0, 0), HALF_EVEN), - (-1, (-1, 999999), FLOOR), - (-1, (0, 0), CEILING), - (-1, (0, 0), HALF_EVEN), - - # half even - (-1500, (-1, 999998), HALF_EVEN), - (-999, (-1, 999999), HALF_EVEN), - (-500, (0, 0), HALF_EVEN), - (500, (0, 0), HALF_EVEN), - (999, (0, 1), HALF_EVEN), - (1500, (0, 2), HALF_EVEN), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) + + us_converter = self.create_decimal_converter(US_TO_NS) + + def timeval_converter(ns): + us = us_converter(ns) + return divmod(us, SEC_TO_US) + + self.check_int_rounding(PyTime_AsTimeval, + timeval_converter, + NS_TO_SEC) @unittest.skipUnless(hasattr(_testcapi, 'PyTime_AsTimespec'), 'need _testcapi.PyTime_AsTimespec') - def test_timespec(self): + def test_AsTimespec(self): from _testcapi import PyTime_AsTimespec - for ns, ts in ( - # nanoseconds - (0, (0, 0)), - (1, (0, 1)), - (-1, (-1, 999999999)), - - # seconds - (2 * SEC_TO_NS, (2, 0)), - (-3 * SEC_TO_NS, (-3, 0)), - - # seconds + nanoseconds - (1234567890, (1, 234567890)), - (-1234567890, (-2, 765432110)), - ): - with self.subTest(nanoseconds=ns, timespec=ts): - self.assertEqual(PyTime_AsTimespec(ns), ts) - - def test_milliseconds(self): + + def timespec_converter(ns): + return divmod(ns, SEC_TO_NS) + + self.check_int_rounding(lambda ns, rnd: PyTime_AsTimespec(ns), + timespec_converter, + NS_TO_SEC) + + def test_AsMilliseconds(self): from _testcapi import PyTime_AsMilliseconds - for rnd in ALL_ROUNDING_METHODS: - for ns, tv in ( - # milliseconds - (1 * MS_TO_NS, 1), - (-2 * MS_TO_NS, -2), - - # seconds - (2 * SEC_TO_NS, 2000), - (-3 * SEC_TO_NS, -3000), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsMilliseconds(ns, rnd), tv) - - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for ns, ms, rnd in ( - # nanoseconds - (1, 0, FLOOR), - (1, 1, CEILING), - (1, 0, HALF_EVEN), - (-1, -1, FLOOR), - (-1, 0, CEILING), - (-1, 0, HALF_EVEN), - - # seconds + nanoseconds - (1234 * MS_TO_NS + 1, 1234, FLOOR), - (1234 * MS_TO_NS + 1, 1235, CEILING), - (1234 * MS_TO_NS + 1, 1234, HALF_EVEN), - (-1234 * MS_TO_NS - 1, -1235, FLOOR), - (-1234 * MS_TO_NS - 1, -1234, CEILING), - (-1234 * MS_TO_NS - 1, -1234, HALF_EVEN), - - # half up - (-1500000, -2, HALF_EVEN), - (-999999, -1, HALF_EVEN), - (-500000, 0, HALF_EVEN), - (500000, 0, HALF_EVEN), - (999999, 1, HALF_EVEN), - (1500000, 2, HALF_EVEN), - ): - with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): - self.assertEqual(PyTime_AsMilliseconds(ns, rnd), ms) - - def test_microseconds(self): + + self.check_int_rounding(PyTime_AsMilliseconds, + self.create_decimal_converter(MS_TO_NS), + NS_TO_SEC) + + def test_AsMicroseconds(self): from _testcapi import PyTime_AsMicroseconds - for rnd in ALL_ROUNDING_METHODS: - for ns, tv in ( - # microseconds - (1 * US_TO_NS, 1), - (-2 * US_TO_NS, -2), - - # milliseconds - (1 * MS_TO_NS, 1000), - (-2 * MS_TO_NS, -2000), - - # seconds - (2 * SEC_TO_NS, 2000000), - (-3 * SEC_TO_NS, -3000000), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsMicroseconds(ns, rnd), tv) - - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for ns, ms, rnd in ( - # nanoseconds - (1, 0, FLOOR), - (1, 1, CEILING), - (1, 0, HALF_EVEN), - (-1, -1, FLOOR), - (-1, 0, CEILING), - (-1, 0, HALF_EVEN), - - # seconds + nanoseconds - (1234 * US_TO_NS + 1, 1234, FLOOR), - (1234 * US_TO_NS + 1, 1235, CEILING), - (1234 * US_TO_NS + 1, 1234, HALF_EVEN), - (-1234 * US_TO_NS - 1, -1235, FLOOR), - (-1234 * US_TO_NS - 1, -1234, CEILING), - (-1234 * US_TO_NS - 1, -1234, HALF_EVEN), - - # half up - (-1500, -2, HALF_EVEN), - (-999, -1, HALF_EVEN), - (-500, 0, HALF_EVEN), - (500, 0, HALF_EVEN), - (999, 1, HALF_EVEN), - (1500, 2, HALF_EVEN), - ): - with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): - self.assertEqual(PyTime_AsMicroseconds(ns, rnd), ms) - - -@unittest.skipUnless(_testcapi is not None, - 'need the _testcapi module') -class TestOldPyTime(unittest.TestCase): + + self.check_int_rounding(PyTime_AsMicroseconds, + self.create_decimal_converter(US_TO_NS), + NS_TO_SEC) + + +class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): """ - Test the old _PyTime_t API: _PyTime_ObjectToXXX() functions. + Test the old C _PyTime_t API: _PyTime_ObjectToXXX() functions. """ - def setUp(self): - self.invalid_values = ( - -(2 ** 100), 2 ** 100, - -(2.0 ** 100.0), 2.0 ** 100.0, - ) - @support.cpython_only - def test_time_t(self): + # time_t is a 32-bit or 64-bit signed integer + OVERFLOW_SECONDS = 2 ** 64 + + def test_object_to_time_t(self): from _testcapi import pytime_object_to_time_t - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, time_t in ( - # int - (0, 0), - (-1, -1), - - # float - (1.0, 1), - (-1.0, -1), - ): - self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) - - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, time_t, rnd in ( - (-1.9, -2, FLOOR), - (-1.9, -2, HALF_EVEN), - (-1.9, -1, CEILING), - - (1.9, 1, FLOOR), - (1.9, 2, HALF_EVEN), - (1.9, 2, CEILING), - - # half even - (-1.5, -2, HALF_EVEN), - (-0.9, -1, HALF_EVEN), - (-0.5, 0, HALF_EVEN), - ( 0.5, 0, HALF_EVEN), - ( 0.9, 1, HALF_EVEN), - ( 1.5, 2, HALF_EVEN), - ): - self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) - - # Test OverflowError - rnd = _PyTime.ROUND_FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_time_t, invalid, rnd) + self.check_int_rounding(pytime_object_to_time_t, + lambda secs: secs) + + self.check_float_rounding(pytime_object_to_time_t, + self.decimal_round) + + def create_converter(self, sec_to_unit): + def converter(secs): + floatpart, intpart = math.modf(secs) + intpart = int(intpart) + floatpart *= sec_to_unit + floatpart = self.decimal_round(floatpart) + if floatpart < 0: + floatpart += sec_to_unit + intpart -= 1 + elif floatpart >= sec_to_unit: + floatpart -= sec_to_unit + intpart += 1 + return (intpart, floatpart) + return converter def test_object_to_timeval(self): from _testcapi import pytime_object_to_timeval - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, timeval in ( - # int - (0, (0, 0)), - (-1, (-1, 0)), - - # float - (-1.0, (-1, 0)), - (1/2**6, (0, 15625)), - (-1/2**6, (-1, 984375)), - (-1e-6, (-1, 999999)), - (1e-6, (0, 1)), - ): - with self.subTest(obj=obj, round=rnd, timeval=timeval): - self.assertEqual(pytime_object_to_timeval(obj, rnd), - timeval) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, timeval, rnd in ( - (-1e-7, (-1, 999999), FLOOR), - (-1e-7, (0, 0), CEILING), - (-1e-7, (0, 0), HALF_EVEN), - - (1e-7, (0, 0), FLOOR), - (1e-7, (0, 1), CEILING), - (1e-7, (0, 0), HALF_EVEN), - - (0.9999999, (0, 999999), FLOOR), - (0.9999999, (1, 0), CEILING), - (0.9999999, (1, 0), HALF_EVEN), - - # half even - (-1.5e-6, (-1, 999998), HALF_EVEN), - (-0.9e-6, (-1, 999999), HALF_EVEN), - (-0.5e-6, (0, 0), HALF_EVEN), - (0.5e-6, (0, 0), HALF_EVEN), - (0.9e-6, (0, 1), HALF_EVEN), - (1.5e-6, (0, 2), HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, timeval=timeval): - self.assertEqual(pytime_object_to_timeval(obj, rnd), timeval) - - rnd = _PyTime.ROUND_FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_timeval, invalid, rnd) - - @support.cpython_only - def test_timespec(self): + self.check_int_rounding(pytime_object_to_timeval, + lambda secs: (secs, 0)) + + self.check_float_rounding(pytime_object_to_timeval, + self.create_converter(SEC_TO_US)) + + def test_object_to_timespec(self): from _testcapi import pytime_object_to_timespec - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, timespec in ( - # int - (0, (0, 0)), - (-1, (-1, 0)), - - # float - (-1.0, (-1, 0)), - (-1e-9, (-1, 999999999)), - (1e-9, (0, 1)), - (-1/2**9, (-1, 998046875)), - ): - with self.subTest(obj=obj, round=rnd, timespec=timespec): - self.assertEqual(pytime_object_to_timespec(obj, rnd), - timespec) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, timespec, rnd in ( - (-1e-10, (-1, 999999999), FLOOR), - (-1e-10, (0, 0), CEILING), - (-1e-10, (0, 0), HALF_EVEN), - - (1e-10, (0, 0), FLOOR), - (1e-10, (0, 1), CEILING), - (1e-10, (0, 0), HALF_EVEN), - - (0.9999999999, (0, 999999999), FLOOR), - (0.9999999999, (1, 0), CEILING), - (0.9999999999, (1, 0), HALF_EVEN), - - # half even - (-1.5e-9, (-1, 999999998), HALF_EVEN), - (-0.9e-9, (-1, 999999999), HALF_EVEN), - (-0.5e-9, (0, 0), HALF_EVEN), - (0.5e-9, (0, 0), HALF_EVEN), - (0.9e-9, (0, 1), HALF_EVEN), - (1.5e-9, (0, 2), HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, timespec=timespec): - self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) - - # Test OverflowError - rnd = FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_timespec, invalid, rnd) + self.check_int_rounding(pytime_object_to_timespec, + lambda secs: (secs, 0)) + + self.check_float_rounding(pytime_object_to_timespec, + self.create_converter(SEC_TO_NS)) + if __name__ == "__main__": -- cgit v1.2.1 From 18124b247e83d18d2e2332877c0a481a90a97b0c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 09:10:14 +0200 Subject: Fix test_time on Windows * Filter values which would overflow on conversion to the C long type (for timeval.tv_sec). * Adjust also the message of OverflowError on PyTime conversions * test_time: add debug information if a timestamp conversion fails --- Lib/test/test_time.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index bd697aeb13..7f1613b7ce 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -756,10 +756,15 @@ class CPyTimeTestCase: context.rounding = decimal_rnd for value in valid_values: - expected = expected_func(value) - self.assertEqual(pytime_converter(value, time_rnd), + debug_info = {'value': value, 'rounding': decimal_rnd} + try: + result = pytime_converter(value, time_rnd) + expected = expected_func(value) + except Exception as exc: + self.fail("Error on timestamp conversion: %s" % debug_info) + self.assertEqual(result, expected, - {'value': value, 'rounding': decimal_rnd}) + debug_info) # test overflow ns = self.OVERFLOW_SECONDS * SEC_TO_NS @@ -770,14 +775,15 @@ class CPyTimeTestCase: with self.assertRaises(OverflowError): pytime_converter(value, time_rnd) - def check_int_rounding(self, pytime_converter, expected_func, unit_to_sec=1, - value_filter=None): + def check_int_rounding(self, pytime_converter, expected_func, + unit_to_sec=1, value_filter=None): self._check_rounding(pytime_converter, expected_func, False, unit_to_sec, value_filter) - def check_float_rounding(self, pytime_converter, expected_func, unit_to_sec=1): + def check_float_rounding(self, pytime_converter, expected_func, + unit_to_sec=1, value_filter=None): self._check_rounding(pytime_converter, expected_func, - True, unit_to_sec) + True, unit_to_sec, value_filter) def decimal_round(self, x): d = decimal.Decimal(x) @@ -845,9 +851,19 @@ class TestCPyTime(CPyTimeTestCase, unittest.TestCase): us = us_converter(ns) return divmod(us, SEC_TO_US) + if sys.platform == 'win32': + from _testcapi import LONG_MIN, LONG_MAX + + # On Windows, timeval.tv_sec type is a C long + def seconds_filter(secs): + return LONG_MIN <= secs <= LONG_MAX + else: + seconds_filter = None + self.check_int_rounding(PyTime_AsTimeval, timeval_converter, - NS_TO_SEC) + NS_TO_SEC, + value_filter=seconds_filter) @unittest.skipUnless(hasattr(_testcapi, 'PyTime_AsTimespec'), 'need _testcapi.PyTime_AsTimespec') @@ -927,6 +943,5 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): self.create_converter(SEC_TO_NS)) - if __name__ == "__main__": unittest.main() -- cgit v1.2.1 From c29ee114ea953e5db07fa4c2fe95455c438bf2d9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 10:10:39 +0200 Subject: Fix test_time on platform with 32-bit time_t type Filter values which would overflow when converted to a C time_t type. --- Lib/test/test_time.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 7f1613b7ce..891c99d009 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -680,6 +680,15 @@ class CPyTimeTestCase: """ OVERFLOW_SECONDS = None + def setUp(self): + from _testcapi import SIZEOF_TIME_T + bits = SIZEOF_TIME_T * 8 - 1 + self.time_t_min = -2 ** bits + self.time_t_max = 2 ** bits - 1 + + def time_t_filter(self, seconds): + return (self.time_t_min <= seconds <= self.time_t_max) + def _rounding_values(self, use_float): "Build timestamps used to test rounding." @@ -858,7 +867,7 @@ class TestCPyTime(CPyTimeTestCase, unittest.TestCase): def seconds_filter(secs): return LONG_MIN <= secs <= LONG_MAX else: - seconds_filter = None + seconds_filter = self.time_t_filter self.check_int_rounding(PyTime_AsTimeval, timeval_converter, @@ -875,7 +884,8 @@ class TestCPyTime(CPyTimeTestCase, unittest.TestCase): self.check_int_rounding(lambda ns, rnd: PyTime_AsTimespec(ns), timespec_converter, - NS_TO_SEC) + NS_TO_SEC, + value_filter=self.time_t_filter) def test_AsMilliseconds(self): from _testcapi import PyTime_AsMilliseconds @@ -904,7 +914,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): from _testcapi import pytime_object_to_time_t self.check_int_rounding(pytime_object_to_time_t, - lambda secs: secs) + lambda secs: secs, + value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_time_t, self.decimal_round) @@ -928,7 +939,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): from _testcapi import pytime_object_to_timeval self.check_int_rounding(pytime_object_to_timeval, - lambda secs: (secs, 0)) + lambda secs: (secs, 0), + value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_timeval, self.create_converter(SEC_TO_US)) @@ -937,7 +949,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): from _testcapi import pytime_object_to_timespec self.check_int_rounding(pytime_object_to_timespec, - lambda secs: (secs, 0)) + lambda secs: (secs, 0), + value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_timespec, self.create_converter(SEC_TO_NS)) -- cgit v1.2.1 From 99d1d4f7eb6d1a53991cb7691324f87ab0361f29 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 11:45:06 +0200 Subject: Fix test_time on platform with 32-bit time_t type Filter also values for check_float_rounding(). --- Lib/test/test_time.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 891c99d009..30b01d5f4b 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -918,7 +918,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_time_t, - self.decimal_round) + self.decimal_round, + value_filter=self.time_t_filter) def create_converter(self, sec_to_unit): def converter(secs): @@ -943,7 +944,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_timeval, - self.create_converter(SEC_TO_US)) + self.create_converter(SEC_TO_US), + value_filter=self.time_t_filter) def test_object_to_timespec(self): from _testcapi import pytime_object_to_timespec @@ -953,7 +955,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_timespec, - self.create_converter(SEC_TO_NS)) + self.create_converter(SEC_TO_NS), + value_filter=self.time_t_filter) if __name__ == "__main__": -- cgit v1.2.1 From 57057ccfd38eaa647ad2983e024e34833e0205e1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 15:55:07 +0200 Subject: pytime: add _PyTime_check_mul_overflow() macro to avoid undefined behaviour Overflow test in test_FromSecondsObject() fails on FreeBSD 10.0 buildbot which uses clang. clang implements more aggressive optimization which gives different result than GCC on undefined behaviours. Check if a multiplication will overflow, instead of checking if a multiplicatin had overflowed, to avoid undefined behaviour. Add also debug information if the test on overflow fails. --- Lib/test/test_time.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 30b01d5f4b..f883c45d04 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -781,7 +781,8 @@ class CPyTimeTestCase: overflow_values = convert_values(ns_timestamps) for time_rnd, _ in ROUNDING_MODES : for value in overflow_values: - with self.assertRaises(OverflowError): + debug_info = {'value': value, 'rounding': time_rnd} + with self.assertRaises(OverflowError, msg=debug_info): pytime_converter(value, time_rnd) def check_int_rounding(self, pytime_converter, expected_func, -- cgit v1.2.1 From 35ac81784d9d2be83e29b164761a7304810ed7c7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2015 12:15:59 +0200 Subject: Issue #25122: try to debug test_eintr hang on FreeBSD * Add verbose mode to test_eintr * Always enable verbose mode in test_eintr * Use faulthandler.dump_traceback_later() with a timeout of 15 minutes in eintr_tester.py --- Lib/test/eintrdata/eintr_tester.py | 8 ++++++++ Lib/test/test_eintr.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 0616df66c7..b96f09b691 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -8,6 +8,7 @@ Signals are generated in-process using setitimer(ITIMER_REAL), which allows sub-second periodicity (contrarily to signal()). """ +import faulthandler import io import os import select @@ -36,12 +37,19 @@ class EINTRBaseTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) + if hasattr(faulthandler, 'dump_traceback_later'): + # Most tests take less than 30 seconds, so 15 minutes should be + # enough. dump_traceback_later() is implemented with a thread, but + # pthread_sigmask() is used to mask all signaled on this thread. + faulthandler.dump_traceback_later(5 * 60, exit=True) signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) + if hasattr(faulthandler, 'cancel_dump_traceback_later'): + faulthandler.cancel_dump_traceback_later() @classmethod def tearDownClass(cls): diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index 111ead365c..1d1d16e0eb 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -1,5 +1,7 @@ import os import signal +import subprocess +import sys import unittest from test import support @@ -14,7 +16,15 @@ class EINTRTests(unittest.TestCase): # Run the tester in a sub-process, to make sure there is only one # thread (for reliable signal delivery). tester = support.findfile("eintr_tester.py", subdir="eintrdata") - script_helper.assert_python_ok(tester) + + # FIXME: Issue #25122, always run in verbose mode to debug hang on FreeBSD + if True: #support.verbose: + args = [sys.executable, tester] + with subprocess.Popen(args, stdout=sys.stderr) as proc: + exitcode = proc.wait() + self.assertEqual(exitcode, 0) + else: + script_helper.assert_python_ok(tester) if __name__ == "__main__": -- cgit v1.2.1 From b4ff0cb43b4f6e80eaa87b51c459cee15969acc3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2015 22:38:09 +0200 Subject: Issue #25122: Fix test_eintr, kill child process on error Some test_eintr hangs on waiting for the child process completion if an error occurred on the parent. Kill the child process on error (in the parent) to avoid the hang. --- Lib/test/eintrdata/eintr_tester.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) (limited to 'Lib') diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index b96f09b691..e1330a71b1 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -8,6 +8,7 @@ Signals are generated in-process using setitimer(ITIMER_REAL), which allows sub-second periodicity (contrarily to signal()). """ +import contextlib import faulthandler import io import os @@ -21,6 +22,16 @@ import unittest from test import support +@contextlib.contextmanager +def kill_on_error(proc): + """Context manager killing the subprocess if a Python exception is raised.""" + with proc: + try: + yield proc + except: + proc.kill() + raise + @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") class EINTRBaseTest(unittest.TestCase): @@ -38,7 +49,7 @@ class EINTRBaseTest(unittest.TestCase): def setUpClass(cls): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) if hasattr(faulthandler, 'dump_traceback_later'): - # Most tests take less than 30 seconds, so 15 minutes should be + # Most tests take less than 30 seconds, so 5 minutes should be # enough. dump_traceback_later() is implemented with a thread, but # pthread_sigmask() is used to mask all signaled on this thread. faulthandler.dump_traceback_later(5 * 60, exit=True) @@ -120,7 +131,8 @@ class OSEINTRTest(EINTRBaseTest): ' os.write(wr, data)', )) - with self.subprocess(code, str(wr), pass_fds=[wr]) as proc: + proc = self.subprocess(code, str(wr), pass_fds=[wr]) + with kill_on_error(proc): os.close(wr) for data in datas: self.assertEqual(data, os.read(rd, len(data))) @@ -156,7 +168,8 @@ class OSEINTRTest(EINTRBaseTest): ' % (len(value), data_len))', )) - with self.subprocess(code, str(rd), pass_fds=[rd]) as proc: + proc = self.subprocess(code, str(rd), pass_fds=[rd]) + with kill_on_error(proc): os.close(rd) written = 0 while written < len(data): @@ -198,7 +211,7 @@ class SocketEINTRTest(EINTRBaseTest): fd = wr.fileno() proc = self.subprocess(code, str(fd), pass_fds=[fd]) - with proc: + with kill_on_error(proc): wr.close() for data in datas: self.assertEqual(data, recv_func(rd, len(data))) @@ -248,7 +261,7 @@ class SocketEINTRTest(EINTRBaseTest): fd = rd.fileno() proc = self.subprocess(code, str(fd), pass_fds=[fd]) - with proc: + with kill_on_error(proc): rd.close() written = 0 while written < len(data): @@ -288,7 +301,8 @@ class SocketEINTRTest(EINTRBaseTest): ' time.sleep(sleep_time)', )) - with self.subprocess(code) as proc: + proc = self.subprocess(code) + with kill_on_error(proc): client_sock, _ = sock.accept() client_sock.close() self.assertEqual(proc.wait(), 0) @@ -315,7 +329,8 @@ class SocketEINTRTest(EINTRBaseTest): do_open_close_reader, )) - with self.subprocess(code) as proc: + proc = self.subprocess(code) + with kill_on_error(proc): do_open_close_writer(filename) self.assertEqual(proc.wait(), 0) @@ -372,7 +387,8 @@ class SignalEINTRTest(EINTRBaseTest): )) t0 = time.monotonic() - with self.subprocess(code) as proc: + proc = self.subprocess(code) + with kill_on_error(proc): # parent signal.sigwaitinfo([signum]) dt = time.monotonic() - t0 -- cgit v1.2.1 From a7bf9bfc49ab04ec7b49a9c2e4f63cadc0b5ff38 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2015 22:55:52 +0200 Subject: Issue #25122: test_eintr: don't redirect stdout to stderr sys.stderr is sometimes a StringIO. The redirection was just a hack to see eintr_tester.py output in red in the buildbot output. --- Lib/test/test_eintr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index 1d1d16e0eb..a1907e8d98 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -20,7 +20,7 @@ class EINTRTests(unittest.TestCase): # FIXME: Issue #25122, always run in verbose mode to debug hang on FreeBSD if True: #support.verbose: args = [sys.executable, tester] - with subprocess.Popen(args, stdout=sys.stderr) as proc: + with subprocess.Popen(args) as proc: exitcode = proc.wait() self.assertEqual(exitcode, 0) else: -- cgit v1.2.1 From 71de2cd1d08dd90eddc94cb2f8b8d28b3971ee82 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2015 23:59:00 +0200 Subject: Issue #25122: optimize test_eintr Fix test_write(): copy support.PIPE_MAX_SIZE bytes, not support.PIPE_MAX_SIZE*3 bytes. --- Lib/test/eintrdata/eintr_tester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index e1330a71b1..24e809a2a4 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -144,14 +144,14 @@ class OSEINTRTest(EINTRBaseTest): # rd closed explicitly by parent # we must write enough data for the write() to block - data = b"xyz" * support.PIPE_MAX_SIZE + data = b"x" * support.PIPE_MAX_SIZE code = '\n'.join(( 'import io, os, sys, time', '', 'rd = int(sys.argv[1])', 'sleep_time = %r' % self.sleep_time, - 'data = b"xyz" * %s' % support.PIPE_MAX_SIZE, + 'data = b"x" * %s' % support.PIPE_MAX_SIZE, 'data_len = len(data)', '', '# let the parent block on write()', -- cgit v1.2.1 From 8ce7582397f333026a7c2d2314aa799a851b299d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Sep 2015 09:23:28 +0200 Subject: Issue #25122: add debug traces to test_eintr.test_open() --- Lib/test/eintrdata/eintr_tester.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'Lib') diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 24e809a2a4..c3c08fe8af 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -326,12 +326,26 @@ class SocketEINTRTest(EINTRBaseTest): '# let the parent block', 'time.sleep(sleep_time)', '', + 'print("try to open %a fifo for reading, pid %s, ppid %s"' + ' % (path, os.getpid(), os.getppid()),' + ' flush=True)', + '', do_open_close_reader, + '', + 'print("%a fifo opened for reading and closed, pid %s, ppid %s"' + ' % (path, os.getpid(), os.getppid()),' + ' flush=True)', )) proc = self.subprocess(code) with kill_on_error(proc): + print("try to open %a fifo for writing, pid %s" + % (filename, os.getpid()), + flush=True) do_open_close_writer(filename) + print("%a fifo opened for writing and closed, pid %s" + % (filename, os.getpid()), + flush=True) self.assertEqual(proc.wait(), 0) -- cgit v1.2.1 From b437f937dcddac9b8790beefab5ccc60d2c6584c Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Thu, 17 Sep 2015 21:49:12 -0700 Subject: Close issue24840: Enum._value_ is queried for bool(); original patch by Mike Lundy --- Lib/enum.py | 3 +++ Lib/test/test_enum.py | 7 +++++++ 2 files changed, 10 insertions(+) (limited to 'Lib') diff --git a/Lib/enum.py b/Lib/enum.py index c28f3452a7..616b2eac15 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -476,6 +476,9 @@ class Enum(metaclass=EnumMeta): def __str__(self): return "%s.%s" % (self.__class__.__name__, self._name_) + def __bool__(self): + return bool(self._value_) + def __dir__(self): added_behavior = [ m diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 4b5d0d07bc..0f7b769b7a 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -270,6 +270,13 @@ class TestEnum(unittest.TestCase): class Wrong(Enum): _any_name_ = 9 + def test_bool(self): + class Logic(Enum): + true = True + false = False + self.assertTrue(Logic.true) + self.assertFalse(Logic.false) + def test_contains(self): Season = self.Season self.assertIn(Season.AUTUMN, Season) -- cgit v1.2.1 From 6c4c7964cb8671a9aadf025b154ec637d5c358f2 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Thu, 17 Sep 2015 22:03:52 -0700 Subject: Close issue25147: use C implementation of OrderedDict --- Lib/enum.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/enum.py b/Lib/enum.py index 616b2eac15..6284b9bb7e 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1,7 +1,12 @@ import sys -from collections import OrderedDict from types import MappingProxyType, DynamicClassAttribute +try: + from _collections import OrderedDict +except ImportError: + from collections import OrderedDict + + __all__ = ['Enum', 'IntEnum', 'unique'] -- cgit v1.2.1 From 9bbaf4cf4713106e95ed88846df2334dad3c200d Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Thu, 17 Sep 2015 22:55:40 -0700 Subject: Issue 25147: add reason for using _collections --- Lib/enum.py | 1 + 1 file changed, 1 insertion(+) (limited to 'Lib') diff --git a/Lib/enum.py b/Lib/enum.py index 6284b9bb7e..8d04e2d718 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1,6 +1,7 @@ import sys from types import MappingProxyType, DynamicClassAttribute +# try _collections first to reduce startup cost try: from _collections import OrderedDict except ImportError: -- cgit v1.2.1 From 3d4a8cc6aedbae874c9f7d1fd821e2665d0f864b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2015 11:23:42 +0200 Subject: Issue #25122: Fix test_eintr.test_open() on FreeBSD Skip test_open() and test_os_open(): both tests uses a FIFO and signals, but there is a bug in the FreeBSD kernel which blocks the test. Skip the tests until the bug is fixed in FreeBSD kernel. Remove also debug traces from test_eintr: * stop using faulthandler to have a timeout * remove print() Write also open and close on two lines in test_open() and test_os_open() tests. If these tests block again, we can know if the test is stuck at open or close. test_eintr: don't always run the test in debug mode. --- Lib/test/eintrdata/eintr_tester.py | 44 +++++++++++++++----------------------- Lib/test/test_eintr.py | 3 +-- 2 files changed, 18 insertions(+), 29 deletions(-) (limited to 'Lib') diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index c3c08fe8af..c2407ca354 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -9,7 +9,6 @@ sub-second periodicity (contrarily to signal()). """ import contextlib -import faulthandler import io import os import select @@ -48,19 +47,12 @@ class EINTRBaseTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) - if hasattr(faulthandler, 'dump_traceback_later'): - # Most tests take less than 30 seconds, so 5 minutes should be - # enough. dump_traceback_later() is implemented with a thread, but - # pthread_sigmask() is used to mask all signaled on this thread. - faulthandler.dump_traceback_later(5 * 60, exit=True) signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) - if hasattr(faulthandler, 'cancel_dump_traceback_later'): - faulthandler.cancel_dump_traceback_later() @classmethod def tearDownClass(cls): @@ -307,6 +299,11 @@ class SocketEINTRTest(EINTRBaseTest): client_sock.close() self.assertEqual(proc.wait(), 0) + # Issue #25122: There is a race condition in the FreeBSD kernel on + # handling signals in the FIFO device. Skip the test until the bug is + # fixed in the kernel. Bet that the bug will be fixed in FreeBSD 11. + # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=203162 + @support.requires_freebsd_version(11) @unittest.skipUnless(hasattr(os, 'mkfifo'), 'needs mkfifo()') def _test_open(self, do_open_close_reader, do_open_close_writer): filename = support.TESTFN @@ -326,36 +323,29 @@ class SocketEINTRTest(EINTRBaseTest): '# let the parent block', 'time.sleep(sleep_time)', '', - 'print("try to open %a fifo for reading, pid %s, ppid %s"' - ' % (path, os.getpid(), os.getppid()),' - ' flush=True)', - '', do_open_close_reader, - '', - 'print("%a fifo opened for reading and closed, pid %s, ppid %s"' - ' % (path, os.getpid(), os.getppid()),' - ' flush=True)', )) proc = self.subprocess(code) with kill_on_error(proc): - print("try to open %a fifo for writing, pid %s" - % (filename, os.getpid()), - flush=True) do_open_close_writer(filename) - print("%a fifo opened for writing and closed, pid %s" - % (filename, os.getpid()), - flush=True) - self.assertEqual(proc.wait(), 0) + def python_open(self, path): + fp = open(path, 'w') + fp.close() + def test_open(self): - self._test_open("open(path, 'r').close()", - lambda path: open(path, 'w').close()) + self._test_open("fp = open(path, 'r')\nfp.close()", + self.python_open) + + def os_open(self, path): + fd = os.open(path, os.O_WRONLY) + os.close(fd) def test_os_open(self): - self._test_open("os.close(os.open(path, os.O_RDONLY))", - lambda path: os.close(os.open(path, os.O_WRONLY))) + self._test_open("fd = os.open(path, os.O_RDONLY)\nos.close(fd)", + self.os_open) @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index a1907e8d98..aabad835a0 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -17,8 +17,7 @@ class EINTRTests(unittest.TestCase): # thread (for reliable signal delivery). tester = support.findfile("eintr_tester.py", subdir="eintrdata") - # FIXME: Issue #25122, always run in verbose mode to debug hang on FreeBSD - if True: #support.verbose: + if support.verbose: args = [sys.executable, tester] with subprocess.Popen(args) as proc: exitcode = proc.wait() -- cgit v1.2.1 From 70a10eeb74bf4142712fabf90b678f10d0e2f9a6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2015 14:52:15 +0200 Subject: Oops, fix test_microsecond_rounding() Test self.theclass, not datetime. Regression introduced by manual tests. --- Lib/test/datetimetester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 5d2df323a9..a6c742100d 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1851,8 +1851,8 @@ class TestDateTime(TestDate): 18000 + 3600 + 2*60 + 3 + 4*1e-6) def test_microsecond_rounding(self): - for fts in (datetime.fromtimestamp, - self.theclass.utcfromtimestamp): + for fts in [self.theclass.fromtimestamp, + self.theclass.utcfromtimestamp]: zero = fts(0) self.assertEqual(zero.second, 0) self.assertEqual(zero.microsecond, 0) -- cgit v1.2.1 From 949cf4e005029d649c9250c7d8ef2af968a2ae4c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2015 16:24:31 +0200 Subject: Issue #25003: Skip test_os.URandomFDTests on Solaris 11.3 and newer When os.urandom() is implemented with the getrandom() function, it doesn't use a file descriptor. --- Lib/test/test_os.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index b73b64ba1f..d4b9e9cc63 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1225,13 +1225,15 @@ class URandomTests(unittest.TestCase): self.assertNotEqual(data1, data2) -HAVE_GETENTROPY = (sysconfig.get_config_var('HAVE_GETENTROPY') == 1) -HAVE_GETRANDOM = (sysconfig.get_config_var('HAVE_GETRANDOM_SYSCALL') == 1) - -@unittest.skipIf(HAVE_GETENTROPY, - "getentropy() does not use a file descriptor") -@unittest.skipIf(HAVE_GETRANDOM, - "getrandom() does not use a file descriptor") +# os.urandom() doesn't use a file descriptor when it is implemented with the +# getentropy() function, the getrandom() function or the getrandom() syscall +OS_URANDOM_DONT_USE_FD = ( + sysconfig.get_config_var('HAVE_GETENTROPY') == 1 + or sysconfig.get_config_var('HAVE_GETRANDOM') == 1 + or sysconfig.get_config_var('HAVE_GETRANDOM_SYSCALL') == 1) + +@unittest.skipIf(OS_URANDOM_DONT_USE_FD , + "os.random() does not use a file descriptor") class URandomFDTests(unittest.TestCase): @unittest.skipUnless(resource, "test requires the resource module") def test_urandom_failure(self): -- cgit v1.2.1 From d46a2718abd0798929ada1b1affc122daf3e004e Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 19 Sep 2015 09:05:42 -0700 Subject: Add a fast path (no iterator creation) for a common case for repeating deques of size 1 --- Lib/test/test_deque.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 87187161ab..c61e80bc2e 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -654,6 +654,15 @@ class TestBasic(unittest.TestCase): self.assertNotEqual(id(d), id(e)) self.assertEqual(list(d), list(e)) + for i in range(5): + for maxlen in range(-1, 6): + s = [random.random() for j in range(i)] + d = deque(s) if maxlen == -1 else deque(s, maxlen) + e = d.copy() + self.assertEqual(d, e) + self.assertEqual(d.maxlen, e.maxlen) + self.assertTrue(all(x is y for x, y in zip(d, e))) + def test_copy_method(self): mut = [10] d = deque([mut]) -- cgit v1.2.1 From 1371e8b9345d3500bc536150e6bfd684dc6c9c9d Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Sat, 19 Sep 2015 14:51:32 -0400 Subject: Issue #24965: Implement PEP 498 "Literal String Interpolation". Documentation is still needed, I'll open an issue for that. --- Lib/test/test_fstring.py | 715 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 715 insertions(+) create mode 100644 Lib/test/test_fstring.py (limited to 'Lib') diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py new file mode 100644 index 0000000000..a6ff9cf934 --- /dev/null +++ b/Lib/test/test_fstring.py @@ -0,0 +1,715 @@ +import ast +import types +import decimal +import unittest + +a_global = 'global variable' + +# You could argue that I'm too strict in looking for specific error +# values with assertRaisesRegex, but without it it's way too easy to +# make a syntax error in the test strings. Especially with all of the +# triple quotes, raw strings, backslashes, etc. I think it's a +# worthwhile tradeoff. When I switched to this method, I found many +# examples where I wasn't testing what I thought I was. + +class TestCase(unittest.TestCase): + def assertAllRaise(self, exception_type, regex, error_strings): + for str in error_strings: + with self.subTest(str=str): + with self.assertRaisesRegex(exception_type, regex): + eval(str) + + def test__format__lookup(self): + # Make sure __format__ is looked up on the type, not the instance. + class X: + def __format__(self, spec): + return 'class' + + x = X() + + # Add a bound __format__ method to the 'y' instance, but not + # the 'x' instance. + y = X() + y.__format__ = types.MethodType(lambda self, spec: 'instance', y) + + self.assertEqual(f'{y}', format(y)) + self.assertEqual(f'{y}', 'class') + self.assertEqual(format(x), format(y)) + + # __format__ is not called this way, but still make sure it + # returns what we expect (so we can make sure we're bypassing + # it). + self.assertEqual(x.__format__(''), 'class') + self.assertEqual(y.__format__(''), 'instance') + + # This is how __format__ is actually called. + self.assertEqual(type(x).__format__(x, ''), 'class') + self.assertEqual(type(y).__format__(y, ''), 'class') + + def test_ast(self): + # Inspired by http://bugs.python.org/issue24975 + class X: + def __init__(self): + self.called = False + def __call__(self): + self.called = True + return 4 + x = X() + expr = """ +a = 10 +f'{a * x()}'""" + t = ast.parse(expr) + c = compile(t, '', 'exec') + + # Make sure x was not called. + self.assertFalse(x.called) + + # Actually run the code. + exec(c) + + # Make sure x was called. + self.assertTrue(x.called) + + def test_literal_eval(self): + # With no expressions, an f-string is okay. + self.assertEqual(ast.literal_eval("f'x'"), 'x') + self.assertEqual(ast.literal_eval("f'x' 'y'"), 'xy') + + # But this should raise an error. + with self.assertRaisesRegex(ValueError, 'malformed node or string'): + ast.literal_eval("f'x{3}'") + + # As should this, which uses a different ast node + with self.assertRaisesRegex(ValueError, 'malformed node or string'): + ast.literal_eval("f'{3}'") + + def test_ast_compile_time_concat(self): + x = [''] + + expr = """x[0] = 'foo' f'{3}'""" + t = ast.parse(expr) + c = compile(t, '', 'exec') + exec(c) + self.assertEqual(x[0], 'foo3') + + def test_literal(self): + self.assertEqual(f'', '') + self.assertEqual(f'a', 'a') + self.assertEqual(f' ', ' ') + self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}', + '\N{GREEK CAPITAL LETTER DELTA}') + self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}', + '\u0394') + self.assertEqual(f'\N{True}', '\u22a8') + self.assertEqual(rf'\N{True}', r'\NTrue') + + def test_escape_order(self): + # note that hex(ord('{')) == 0x7b, so this + # string becomes f'a{4*10}b' + self.assertEqual(f'a\u007b4*10}b', 'a40b') + self.assertEqual(f'a\x7b4*10}b', 'a40b') + self.assertEqual(f'a\x7b4*10\N{RIGHT CURLY BRACKET}b', 'a40b') + self.assertEqual(f'{"a"!\N{LATIN SMALL LETTER R}}', "'a'") + self.assertEqual(f'{10\x3a02X}', '0A') + self.assertEqual(f'{10:02\N{LATIN CAPITAL LETTER X}}', '0A') + + self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed", + [r"""f'a{\u007b4*10}b'""", # mis-matched brackets + ]) + self.assertAllRaise(SyntaxError, 'unexpected character after line continuation character', + [r"""f'{"a"\!r}'""", + r"""f'{a\!r}'""", + ]) + + def test_unterminated_string(self): + self.assertAllRaise(SyntaxError, 'f-string: unterminated string', + [r"""f'{"x'""", + r"""f'{"x}'""", + r"""f'{("x'""", + r"""f'{("x}'""", + ]) + + def test_mismatched_parens(self): + self.assertAllRaise(SyntaxError, 'f-string: mismatched', + ["f'{((}'", + ]) + + def test_double_braces(self): + self.assertEqual(f'{{', '{') + self.assertEqual(f'a{{', 'a{') + self.assertEqual(f'{{b', '{b') + self.assertEqual(f'a{{b', 'a{b') + self.assertEqual(f'}}', '}') + self.assertEqual(f'a}}', 'a}') + self.assertEqual(f'}}b', '}b') + self.assertEqual(f'a}}b', 'a}b') + + self.assertEqual(f'{{{10}', '{10') + self.assertEqual(f'}}{10}', '}10') + self.assertEqual(f'}}{{{10}', '}{10') + self.assertEqual(f'}}a{{{10}', '}a{10') + + self.assertEqual(f'{10}{{', '10{') + self.assertEqual(f'{10}}}', '10}') + self.assertEqual(f'{10}}}{{', '10}{') + self.assertEqual(f'{10}}}a{{' '}', '10}a{}') + + # Inside of strings, don't interpret doubled brackets. + self.assertEqual(f'{"{{}}"}', '{{}}') + + self.assertAllRaise(TypeError, 'unhashable type', + ["f'{ {{}} }'", # dict in a set + ]) + + def test_compile_time_concat(self): + x = 'def' + self.assertEqual('abc' f'## {x}ghi', 'abc## defghi') + self.assertEqual('abc' f'{x}' 'ghi', 'abcdefghi') + self.assertEqual('abc' f'{x}' 'gh' f'i{x:4}', 'abcdefghidef ') + self.assertEqual('{x}' f'{x}', '{x}def') + self.assertEqual('{x' f'{x}', '{xdef') + self.assertEqual('{x}' f'{x}', '{x}def') + self.assertEqual('{{x}}' f'{x}', '{{x}}def') + self.assertEqual('{{x' f'{x}', '{{xdef') + self.assertEqual('x}}' f'{x}', 'x}}def') + self.assertEqual(f'{x}' 'x}}', 'defx}}') + self.assertEqual(f'{x}' '', 'def') + self.assertEqual('' f'{x}' '', 'def') + self.assertEqual('' f'{x}', 'def') + self.assertEqual(f'{x}' '2', 'def2') + self.assertEqual('1' f'{x}' '2', '1def2') + self.assertEqual('1' f'{x}', '1def') + self.assertEqual(f'{x}' f'-{x}', 'def-def') + self.assertEqual('' f'', '') + self.assertEqual('' f'' '', '') + self.assertEqual('' f'' '' f'', '') + self.assertEqual(f'', '') + self.assertEqual(f'' '', '') + self.assertEqual(f'' '' f'', '') + self.assertEqual(f'' '' f'' '', '') + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3' f'}'", # can't concat to get a valid f-string + ]) + + def test_comments(self): + # These aren't comments, since they're in strings. + d = {'#': 'hash'} + self.assertEqual(f'{"#"}', '#') + self.assertEqual(f'{d["#"]}', 'hash') + + self.assertAllRaise(SyntaxError, "f-string cannot include '#'", + ["f'{1#}'", # error because the expression becomes "(1#)" + "f'{3(#)}'", + ]) + + def test_many_expressions(self): + # Create a string with many expressions in it. Note that + # because we have a space in here as a literal, we're actually + # going to use twice as many ast nodes: one for each literal + # plus one for each expression. + def build_fstr(n, extra=''): + return "f'" + ('{x} ' * n) + extra + "'" + + x = 'X' + width = 1 + + # Test around 256. + for i in range(250, 260): + self.assertEqual(eval(build_fstr(i)), (x+' ')*i) + + # Test concatenating 2 largs fstrings. + self.assertEqual(eval(build_fstr(255)*256), (x+' ')*(255*256)) + + s = build_fstr(253, '{x:{width}} ') + self.assertEqual(eval(s), (x+' ')*254) + + # Test lots of expressions and constants, concatenated. + s = "f'{1}' 'x' 'y'" * 1024 + self.assertEqual(eval(s), '1xy' * 1024) + + def test_format_specifier_expressions(self): + width = 10 + precision = 4 + value = decimal.Decimal('12.34567') + self.assertEqual(f'result: {value:{width}.{precision}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result: 12.35') + self.assertEqual(f'{10:#{1}0x}', ' 0xa') + self.assertEqual(f'{10:{"#"}1{0}{"x"}}', ' 0xa') + self.assertEqual(f'{-10:-{"#"}1{0}x}', ' -0xa') + self.assertEqual(f'{-10:{"-"}#{1}0{"x"}}', ' -0xa') + self.assertEqual(f'{10:#{3 != {4:5} and width}x}', ' 0xa') + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["""f'{"s"!r{":10"}}'""", + + # This looks like a nested format spec. + ]) + + self.assertAllRaise(SyntaxError, "invalid syntax", + [# Invalid sytax inside a nested spec. + "f'{4:{/5}}'", + ]) + + self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply", + [# Can't nest format specifiers. + "f'result: {value:{width:{0}}.{precision:1}}'", + ]) + + self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character', + [# No expansion inside conversion or for + # the : or ! itself. + """f'{"s"!{"r"}}'""", + ]) + + def test_side_effect_order(self): + class X: + def __init__(self): + self.i = 0 + def __format__(self, spec): + self.i += 1 + return str(self.i) + + x = X() + self.assertEqual(f'{x} {x}', '1 2') + + def test_missing_expression(self): + self.assertAllRaise(SyntaxError, 'f-string: empty expression not allowed', + ["f'{}'", + "f'{ }'" + "f' {} '", + "f'{!r}'", + "f'{ !r}'", + "f'{10:{ }}'", + "f' { } '", + r"f'{\n}'", + r"f'{\n \n}'", + ]) + + def test_parens_in_expressions(self): + self.assertEqual(f'{3,}', '(3,)') + + # Add these because when an expression is evaluated, parens + # are added around it. But we shouldn't go from an invalid + # expression to a valid one. The added parens are just + # supposed to allow whitespace (including newlines). + self.assertAllRaise(SyntaxError, 'invalid syntax', + ["f'{,}'", + "f'{,}'", # this is (,), which is an error + ]) + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3)+(4}'", + ]) + + self.assertAllRaise(SyntaxError, 'EOL while scanning string literal', + ["f'{\n}'", + ]) + + def test_newlines_in_expressions(self): + self.assertEqual(f'{0}', '0') + self.assertEqual(f'{0\n}', '0') + self.assertEqual(f'{0\r}', '0') + self.assertEqual(f'{\n0\n}', '0') + self.assertEqual(f'{\r0\r}', '0') + self.assertEqual(f'{\n0\r}', '0') + self.assertEqual(f'{\n0}', '0') + self.assertEqual(f'{3+\n4}', '7') + self.assertEqual(f'{3+\\\n4}', '7') + self.assertEqual(rf'''{3+ +4}''', '7') + self.assertEqual(f'''{3+\ +4}''', '7') + + self.assertAllRaise(SyntaxError, 'f-string: empty expression not allowed', + [r"f'{\n}'", + ]) + + def test_lambda(self): + x = 5 + self.assertEqual(f'{(lambda y:x*y)("8")!r}', "'88888'") + self.assertEqual(f'{(lambda y:x*y)("8")!r:10}', "'88888' ") + self.assertEqual(f'{(lambda y:x*y)("8"):10}', "88888 ") + + # lambda doesn't work without parens, because the colon + # makes the parser think it's a format_spec + self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', + ["f'{lambda x:x}'", + ]) + + def test_yield(self): + # Not terribly useful, but make sure the yield turns + # a function into a generator + def fn(y): + f'y:{yield y*2}' + + g = fn(4) + self.assertEqual(next(g), 8) + + def test_yield_send(self): + def fn(x): + yield f'x:{yield (lambda i: x * i)}' + + g = fn(10) + the_lambda = next(g) + self.assertEqual(the_lambda(4), 40) + self.assertEqual(g.send('string'), 'x:string') + + def test_expressions_with_triple_quoted_strings(self): + self.assertEqual(f"{'''x'''}", 'x') + self.assertEqual(f"{'''eric's'''}", "eric's") + self.assertEqual(f'{"""eric\'s"""}', "eric's") + self.assertEqual(f"{'''eric\"s'''}", 'eric"s') + self.assertEqual(f'{"""eric"s"""}', 'eric"s') + + # Test concatenation within an expression + self.assertEqual(f'{"x" """eric"s""" "y"}', 'xeric"sy') + self.assertEqual(f'{"x" """eric"s"""}', 'xeric"s') + self.assertEqual(f'{"""eric"s""" "y"}', 'eric"sy') + self.assertEqual(f'{"""x""" """eric"s""" "y"}', 'xeric"sy') + self.assertEqual(f'{"""x""" """eric"s""" """y"""}', 'xeric"sy') + self.assertEqual(f'{r"""x""" """eric"s""" """y"""}', 'xeric"sy') + + def test_multiple_vars(self): + x = 98 + y = 'abc' + self.assertEqual(f'{x}{y}', '98abc') + + self.assertEqual(f'X{x}{y}', 'X98abc') + self.assertEqual(f'{x}X{y}', '98Xabc') + self.assertEqual(f'{x}{y}X', '98abcX') + + self.assertEqual(f'X{x}Y{y}', 'X98Yabc') + self.assertEqual(f'X{x}{y}Y', 'X98abcY') + self.assertEqual(f'{x}X{y}Y', '98XabcY') + + self.assertEqual(f'X{x}Y{y}Z', 'X98YabcZ') + + def test_closure(self): + def outer(x): + def inner(): + return f'x:{x}' + return inner + + self.assertEqual(outer('987')(), 'x:987') + self.assertEqual(outer(7)(), 'x:7') + + def test_arguments(self): + y = 2 + def f(x, width): + return f'x={x*y:{width}}' + + self.assertEqual(f('foo', 10), 'x=foofoo ') + x = 'bar' + self.assertEqual(f(10, 10), 'x= 20') + + def test_locals(self): + value = 123 + self.assertEqual(f'v:{value}', 'v:123') + + def test_missing_variable(self): + with self.assertRaises(NameError): + f'v:{value}' + + def test_missing_format_spec(self): + class O: + def __format__(self, spec): + if not spec: + return '*' + return spec + + self.assertEqual(f'{O():x}', 'x') + self.assertEqual(f'{O()}', '*') + self.assertEqual(f'{O():}', '*') + + self.assertEqual(f'{3:}', '3') + self.assertEqual(f'{3!s:}', '3') + + def test_global(self): + self.assertEqual(f'g:{a_global}', 'g:global variable') + self.assertEqual(f'g:{a_global!r}', "g:'global variable'") + + a_local = 'local variable' + self.assertEqual(f'g:{a_global} l:{a_local}', + 'g:global variable l:local variable') + self.assertEqual(f'g:{a_global!r}', + "g:'global variable'") + self.assertEqual(f'g:{a_global} l:{a_local!r}', + "g:global variable l:'local variable'") + + self.assertIn("module 'unittest' from", f'{unittest}') + + def test_shadowed_global(self): + a_global = 'really a local' + self.assertEqual(f'g:{a_global}', 'g:really a local') + self.assertEqual(f'g:{a_global!r}', "g:'really a local'") + + a_local = 'local variable' + self.assertEqual(f'g:{a_global} l:{a_local}', + 'g:really a local l:local variable') + self.assertEqual(f'g:{a_global!r}', + "g:'really a local'") + self.assertEqual(f'g:{a_global} l:{a_local!r}', + "g:really a local l:'local variable'") + + def test_call(self): + def foo(x): + return 'x=' + str(x) + + self.assertEqual(f'{foo(10)}', 'x=10') + + def test_nested_fstrings(self): + y = 5 + self.assertEqual(f'{f"{0}"*3}', '000') + self.assertEqual(f'{f"{y}"*3}', '555') + self.assertEqual(f'{f"{\'x\'}"*3}', 'xxx') + + self.assertEqual(f"{r'x' f'{\"s\"}'}", 'xs') + self.assertEqual(f"{r'x'rf'{\"s\"}'}", 'xs') + + def test_invalid_string_prefixes(self): + self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', + ["fu''", + "uf''", + "Fu''", + "fU''", + "Uf''", + "uF''", + "ufr''", + "urf''", + "fur''", + "fru''", + "rfu''", + "ruf''", + "FUR''", + "Fur''", + ]) + + def test_leading_trailing_spaces(self): + self.assertEqual(f'{ 3}', '3') + self.assertEqual(f'{ 3}', '3') + self.assertEqual(f'{\t3}', '3') + self.assertEqual(f'{\t\t3}', '3') + self.assertEqual(f'{3 }', '3') + self.assertEqual(f'{3 }', '3') + self.assertEqual(f'{3\t}', '3') + self.assertEqual(f'{3\t\t}', '3') + + self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]}}', + 'expr={1: 2}') + self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]} }', + 'expr={1: 2}') + + def test_character_name(self): + self.assertEqual(f'{4}\N{GREEK CAPITAL LETTER DELTA}{3}', + '4\N{GREEK CAPITAL LETTER DELTA}3') + self.assertEqual(f'{{}}\N{GREEK CAPITAL LETTER DELTA}{3}', + '{}\N{GREEK CAPITAL LETTER DELTA}3') + + def test_not_equal(self): + # There's a special test for this because there's a special + # case in the f-string parser to look for != as not ending an + # expression. Normally it would, while looking for !s or !r. + + self.assertEqual(f'{3!=4}', 'True') + self.assertEqual(f'{3!=4:}', 'True') + self.assertEqual(f'{3!=4!s}', 'True') + self.assertEqual(f'{3!=4!s:.3}', 'Tru') + + def test_conversions(self): + self.assertEqual(f'{3.14:10.10}', ' 3.14') + self.assertEqual(f'{3.14!s:10.10}', '3.14 ') + self.assertEqual(f'{3.14!r:10.10}', '3.14 ') + self.assertEqual(f'{3.14!a:10.10}', '3.14 ') + + self.assertEqual(f'{"a"}', 'a') + self.assertEqual(f'{"a"!r}', "'a'") + self.assertEqual(f'{"a"!a}', "'a'") + + # Not a conversion. + self.assertEqual(f'{"a!r"}', "a!r") + + # Not a conversion, but show that ! is allowed in a format spec. + self.assertEqual(f'{3.14:!<10.10}', '3.14!!!!!!') + + self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"}', '\u0394') + self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"!r}', "'\u0394'") + self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"!a}', "'\\u0394'") + + self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character', + ["f'{3!g}'", + "f'{3!A}'", + "f'{3!A}'", + "f'{3!A}'", + "f'{3!!}'", + "f'{3!:}'", + "f'{3!\N{GREEK CAPITAL LETTER DELTA}}'", + "f'{3! s}'", # no space before conversion char + "f'{x!\\x00:.<10}'", + ]) + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{x!s{y}}'", + "f'{3!ss}'", + "f'{3!ss:}'", + "f'{3!ss:s}'", + ]) + + def test_assignment(self): + self.assertAllRaise(SyntaxError, 'invalid syntax', + ["f'' = 3", + "f'{0}' = x", + "f'{x}' = x", + ]) + + def test_del(self): + self.assertAllRaise(SyntaxError, 'invalid syntax', + ["del f''", + "del '' f''", + ]) + + def test_mismatched_braces(self): + self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed", + ["f'{{}'", + "f'{{}}}'", + "f'}'", + "f'x}'", + "f'x}x'", + + # Can't have { or } in a format spec. + "f'{3:}>10}'", + r"f'{3:\\}>10}'", + "f'{3:}}>10}'", + ]) + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3:{{>10}'", + "f'{3'", + "f'{3!'", + "f'{3:'", + "f'{3!s'", + "f'{3!s:'", + "f'{3!s:3'", + "f'x{'", + "f'x{x'", + "f'{3:s'", + "f'{{{'", + "f'{{}}{'", + "f'{'", + ]) + + self.assertAllRaise(SyntaxError, 'invalid syntax', + [r"f'{3:\\{>10}'", + ]) + + # But these are just normal strings. + self.assertEqual(f'{"{"}', '{') + self.assertEqual(f'{"}"}', '}') + self.assertEqual(f'{3:{"}"}>10}', '}}}}}}}}}3') + self.assertEqual(f'{2:{"{"}>10}', '{{{{{{{{{2') + + def test_if_conditional(self): + # There's special logic in compile.c to test if the + # conditional for an if (and while) are constants. Exercise + # that code. + + def test_fstring(x, expected): + flag = 0 + if f'{x}': + flag = 1 + else: + flag = 2 + self.assertEqual(flag, expected) + + def test_concat_empty(x, expected): + flag = 0 + if '' f'{x}': + flag = 1 + else: + flag = 2 + self.assertEqual(flag, expected) + + def test_concat_non_empty(x, expected): + flag = 0 + if ' ' f'{x}': + flag = 1 + else: + flag = 2 + self.assertEqual(flag, expected) + + test_fstring('', 2) + test_fstring(' ', 1) + + test_concat_empty('', 2) + test_concat_empty(' ', 1) + + test_concat_non_empty('', 1) + test_concat_non_empty(' ', 1) + + def test_empty_format_specifier(self): + x = 'test' + self.assertEqual(f'{x}', 'test') + self.assertEqual(f'{x:}', 'test') + self.assertEqual(f'{x!s:}', 'test') + self.assertEqual(f'{x!r:}', "'test'") + + def test_str_format_differences(self): + d = {'a': 'string', + 0: 'integer', + } + a = 0 + self.assertEqual(f'{d[0]}', 'integer') + self.assertEqual(f'{d["a"]}', 'string') + self.assertEqual(f'{d[a]}', 'integer') + self.assertEqual('{d[a]}'.format(d=d), 'string') + self.assertEqual('{d[0]}'.format(d=d), 'integer') + + def test_invalid_expressions(self): + self.assertAllRaise(SyntaxError, 'invalid syntax', + [r"f'{a[4)}'", + r"f'{a(4]}'", + ]) + + def test_loop(self): + for i in range(1000): + self.assertEqual(f'i:{i}', 'i:' + str(i)) + + def test_dict(self): + d = {'"': 'dquote', + "'": 'squote', + 'foo': 'bar', + } + self.assertEqual(f'{d["\'"]}', 'squote') + self.assertEqual(f"{d['\"']}", 'dquote') + + self.assertEqual(f'''{d["'"]}''', 'squote') + self.assertEqual(f"""{d['"']}""", 'dquote') + + self.assertEqual(f'{d["foo"]}', 'bar') + self.assertEqual(f"{d['foo']}", 'bar') + self.assertEqual(f'{d[\'foo\']}', 'bar') + self.assertEqual(f"{d[\"foo\"]}", 'bar') + + def test_escaped_quotes(self): + d = {'"': 'a', + "'": 'b'} + + self.assertEqual(fr"{d['\"']}", 'a') + self.assertEqual(fr'{d["\'"]}', 'b') + self.assertEqual(fr"{'\"'}", '"') + self.assertEqual(fr'{"\'"}', "'") + self.assertEqual(f'{"\\"3"}', '"3') + + self.assertAllRaise(SyntaxError, 'f-string: unterminated string', + [r'''f'{"""\\}' ''', # Backslash at end of expression + ]) + self.assertAllRaise(SyntaxError, 'unexpected character after line continuation', + [r"rf'{3\}'", + ]) + + +if __name__ == '__main__': + unittest.main() -- cgit v1.2.1 From 6a834db051122b0f9b27d6bb6b14e1572a24df10 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Sat, 19 Sep 2015 15:49:57 -0400 Subject: Temporary hack for issue #25180: exclude test_fstring.py from the unparse round-tripping, while I figure out how to properly fix it. --- Lib/test/test_tools/test_unparse.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_tools/test_unparse.py b/Lib/test/test_tools/test_unparse.py index 976a6c59ae..920fcbd6ee 100644 --- a/Lib/test/test_tools/test_unparse.py +++ b/Lib/test/test_tools/test_unparse.py @@ -264,6 +264,8 @@ class DirectoryTestCase(ASTTestCase): for d in self.test_directories: test_dir = os.path.join(basepath, d) for n in os.listdir(test_dir): + if n == 'test_fstring.py': + continue if n.endswith('.py') and not n.startswith('bad'): names.append(os.path.join(test_dir, n)) -- cgit v1.2.1 From 90b73b362ee0ba0d42649d580c8a674d2b139b6f Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Sun, 20 Sep 2015 15:09:15 -0400 Subject: Issue 25180: Fix Tools/parser/unparse.py for f-strings. Patch by Martin Panter. --- Lib/test/test_tools/test_unparse.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_tools/test_unparse.py b/Lib/test/test_tools/test_unparse.py index 920fcbd6ee..4b47916636 100644 --- a/Lib/test/test_tools/test_unparse.py +++ b/Lib/test/test_tools/test_unparse.py @@ -134,6 +134,15 @@ class ASTTestCase(unittest.TestCase): class UnparseTestCase(ASTTestCase): # Tests for specific bugs found in earlier versions of unparse + def test_fstrings(self): + # See issue 25180 + self.check_roundtrip(r"""f'{f"{0}"*3}'""") + self.check_roundtrip(r"""f'{f"{y}"*3}'""") + self.check_roundtrip(r"""f'{f"{\'x\'}"*3}'""") + + self.check_roundtrip(r'''f"{r'x' f'{\"s\"}'}"''') + self.check_roundtrip(r'''f"{r'x'rf'{\"s\"}'}"''') + def test_del_statement(self): self.check_roundtrip("del x, y, z") @@ -264,8 +273,6 @@ class DirectoryTestCase(ASTTestCase): for d in self.test_directories: test_dir = os.path.join(basepath, d) for n in os.listdir(test_dir): - if n == 'test_fstring.py': - continue if n.endswith('.py') and not n.startswith('bad'): names.append(os.path.join(test_dir, n)) -- cgit v1.2.1 From 622f7e304f1ae867beb75a15ba84e45b0ab9996a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Sep 2015 22:29:43 +0200 Subject: Merge 3.5 (Issue #23630, fix test_asyncio) --- Lib/test/test_asyncio/test_events.py | 1 + 1 file changed, 1 insertion(+) (limited to 'Lib') diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 3488101689..9801d22223 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -764,6 +764,7 @@ class EventLoopTestsMixin: for host in hosts] self.loop.getaddrinfo = getaddrinfo_task self.loop._start_serving = mock.Mock() + self.loop._stop_serving = mock.Mock() f = self.loop.create_server(lambda: MyProto(self.loop), hosts, 80) server = self.loop.run_until_complete(f) self.addCleanup(server.close) -- cgit v1.2.1 From 7347e7d89c5bb7725e2c588586402f7a9f42a44c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Sep 2015 23:06:27 +0200 Subject: Issue #24870: Optimize the ASCII decoder for error handlers: surrogateescape, ignore and replace. Initial patch written by Naoki Inada. The decoder is now up to 60 times as fast for these error handlers. Add also unit tests for the ASCII decoder. --- Lib/test/test_codecs.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index a4a6f95ca2..e0e31199cc 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -27,6 +27,7 @@ def coding_checker(self, coder): self.assertEqual(coder(input), (expect, len(input))) return check + class Queue(object): """ queue: write bytes at one end, read bytes from the other end @@ -47,6 +48,7 @@ class Queue(object): self._buffer = self._buffer[size:] return s + class MixInCheckStateHandling: def check_state_handling_decode(self, encoding, u, s): for i in range(len(s)+1): @@ -80,6 +82,7 @@ class MixInCheckStateHandling: part2 = d.encode(u[i:], True) self.assertEqual(s, part1+part2) + class ReadTest(MixInCheckStateHandling): def check_partial(self, input, partialresults): # get a StreamReader for the encoding and feed the bytestring version @@ -383,6 +386,7 @@ class ReadTest(MixInCheckStateHandling): self.assertEqual(test_sequence.decode(self.encoding, "backslashreplace"), before + backslashreplace + after) + class UTF32Test(ReadTest, unittest.TestCase): encoding = "utf-32" if sys.byteorder == 'little': @@ -478,6 +482,7 @@ class UTF32Test(ReadTest, unittest.TestCase): self.assertEqual('\U00010000' * 1024, codecs.utf_32_decode(encoded_be)[0]) + class UTF32LETest(ReadTest, unittest.TestCase): encoding = "utf-32-le" ill_formed_sequence = b"\x80\xdc\x00\x00" @@ -523,6 +528,7 @@ class UTF32LETest(ReadTest, unittest.TestCase): self.assertEqual('\U00010000' * 1024, codecs.utf_32_le_decode(encoded)[0]) + class UTF32BETest(ReadTest, unittest.TestCase): encoding = "utf-32-be" ill_formed_sequence = b"\x00\x00\xdc\x80" @@ -797,6 +803,7 @@ class UTF8Test(ReadTest, unittest.TestCase): with self.assertRaises(UnicodeDecodeError): b"abc\xed\xa0z".decode("utf-8", "surrogatepass") + @unittest.skipUnless(sys.platform == 'win32', 'cp65001 is a Windows-only codec') class CP65001Test(ReadTest, unittest.TestCase): @@ -1136,6 +1143,7 @@ class EscapeDecodeTest(unittest.TestCase): self.assertEqual(decode(br"[\x0]\x0", "ignore"), (b"[]", 8)) self.assertEqual(decode(br"[\x0]\x0", "replace"), (b"[?]?", 8)) + class RecodingTest(unittest.TestCase): def test_recoding(self): f = io.BytesIO() @@ -1255,6 +1263,7 @@ for i in punycode_testcases: if len(i)!=2: print(repr(i)) + class PunycodeTest(unittest.TestCase): def test_encode(self): for uni, puny in punycode_testcases: @@ -1274,6 +1283,7 @@ class PunycodeTest(unittest.TestCase): puny = puny.decode("ascii").encode("ascii") self.assertEqual(uni, puny.decode("punycode")) + class UnicodeInternalTest(unittest.TestCase): @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t') def test_bug1251300(self): @@ -1528,6 +1538,7 @@ class NameprepTest(unittest.TestCase): except Exception as e: raise support.TestFailed("Test 3.%d: %s" % (pos+1, str(e))) + class IDNACodecTest(unittest.TestCase): def test_builtin_decode(self): self.assertEqual(str(b"python.org", "idna"), "python.org") @@ -1614,6 +1625,7 @@ class IDNACodecTest(unittest.TestCase): self.assertRaises(Exception, b"python.org".decode, "idna", errors) + class CodecsModuleTest(unittest.TestCase): def test_decode(self): @@ -1722,6 +1734,7 @@ class CodecsModuleTest(unittest.TestCase): self.assertRaises(UnicodeError, codecs.decode, b'abc', 'undefined', errors) + class StreamReaderTest(unittest.TestCase): def setUp(self): @@ -1732,6 +1745,7 @@ class StreamReaderTest(unittest.TestCase): f = self.reader(self.stream) self.assertEqual(f.readlines(), ['\ud55c\n', '\uae00']) + class EncodedFileTest(unittest.TestCase): def test_basic(self): @@ -1862,6 +1876,7 @@ broken_unicode_with_stateful = [ "unicode_internal" ] + class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): def test_basics(self): s = "abc123" # all codecs should be able to encode these @@ -2024,6 +2039,7 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): self.check_state_handling_decode(encoding, u, u.encode(encoding)) self.check_state_handling_encode(encoding, u, u.encode(encoding)) + class CharmapTest(unittest.TestCase): def test_decode_with_string_map(self): self.assertEqual( @@ -2274,6 +2290,7 @@ class WithStmtTest(unittest.TestCase): info.streamwriter, 'strict') as srw: self.assertEqual(srw.read(), "\xfc") + class TypesTest(unittest.TestCase): def test_decode_unicode(self): # Most decoders don't accept unicode input @@ -2564,6 +2581,7 @@ else: bytes_transform_encodings.append("bz2_codec") transform_aliases["bz2_codec"] = ["bz2"] + class TransformCodecTest(unittest.TestCase): def test_basics(self): @@ -3041,5 +3059,19 @@ class CodePageTest(unittest.TestCase): self.assertEqual(decoded, ('abc', 3)) +class ASCIITest(unittest.TestCase): + def test_decode(self): + for data, error_handler, expected in ( + (b'[\x80\xff]', 'ignore', '[]'), + (b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'), + (b'[\x80\xff]', 'surrogateescape', '[\udc80\udcff]'), + (b'[\x80\xff]', 'backslashreplace', '[\\x80\\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.decode('ascii', error_handler), + expected) + + if __name__ == "__main__": unittest.main() -- cgit v1.2.1 From 8e7b550d5f80778a41c7c6d073f19aa3633fe2d2 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Wed, 23 Sep 2015 07:49:00 -0400 Subject: Move f-string compilation of the expression earlier, before the conversion character and format_spec are checked. This allows for error messages that more closely match what a user would expect. --- Lib/test/test_fstring.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index a6ff9cf934..2a1a3cd36a 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -287,6 +287,15 @@ f'{a * x()}'""" "f' { } '", r"f'{\n}'", r"f'{\n \n}'", + + # Catch the empty expression before the + # invalid conversion. + "f'{!x}'", + "f'{ !xr}'", + "f'{!x:}'", + "f'{!x:a}'", + "f'{ !xr:}'", + "f'{ !xr:a}'", ]) def test_parens_in_expressions(self): -- cgit v1.2.1 From 60a1c43d9bd9add8244d9d49f8de9fa5f4e42d0b Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Wed, 23 Sep 2015 08:00:01 -0400 Subject: Added more f-string test for empty expressions. --- Lib/test/test_fstring.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 2a1a3cd36a..10c5849cf3 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -296,6 +296,9 @@ f'{a * x()}'""" "f'{!x:a}'", "f'{ !xr:}'", "f'{ !xr:a}'", + + "f'{!}'", + "f'{:}'", ]) def test_parens_in_expressions(self): -- cgit v1.2.1 From 598edf6049fbcb9cf65abe81546b09aca7c59b1c Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Wed, 23 Sep 2015 10:24:43 -0400 Subject: f-strings: More tests for empty expressions along with missing closing braces. --- Lib/test/test_fstring.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 10c5849cf3..d6f781c846 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -299,6 +299,13 @@ f'{a * x()}'""" "f'{!}'", "f'{:}'", + + # We find the empty expression before the + # missing closing brace. + "f'{!'", + "f'{!s:'", + "f'{:'", + "f'{:x'", ]) def test_parens_in_expressions(self): -- cgit v1.2.1 From 5e4b0df2ffc5a5db71c49b57c715aff5147e426c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Sep 2015 23:04:18 +0200 Subject: Issue #25220: Create Lib/test/libregrtest/ Start to split regrtest.py into smaller parts with the creation of Lib/test/libregrtest/cmdline.py. --- Lib/test/libregrtest/__init__.py | 1 + Lib/test/libregrtest/cmdline.py | 340 +++++++++++++++++++++++++++++++++++++++ Lib/test/regrtest.py | 332 +------------------------------------- Lib/test/test_regrtest.py | 4 +- 4 files changed, 345 insertions(+), 332 deletions(-) create mode 100644 Lib/test/libregrtest/__init__.py create mode 100644 Lib/test/libregrtest/cmdline.py (limited to 'Lib') diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py new file mode 100644 index 0000000000..554aeede4a --- /dev/null +++ b/Lib/test/libregrtest/__init__.py @@ -0,0 +1 @@ +from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py new file mode 100644 index 0000000000..94649965a5 --- /dev/null +++ b/Lib/test/libregrtest/cmdline.py @@ -0,0 +1,340 @@ +import argparse +import faulthandler +import os + +from test import support + +USAGE = """\ +python -m test [options] [test_name1 [test_name2 ...]] +python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] +""" + +DESCRIPTION = """\ +Run Python regression tests. + +If no arguments or options are provided, finds all files matching +the pattern "test_*" in the Lib/test subdirectory and runs +them in alphabetical order (but see -M and -u, below, for exceptions). + +For more rigorous testing, it is useful to use the following +command line: + +python -E -Wd -m test [options] [test_name1 ...] +""" + +EPILOG = """\ +Additional option details: + +-r randomizes test execution order. You can use --randseed=int to provide a +int seed value for the randomizer; this is useful for reproducing troublesome +test orders. + +-s On the first invocation of regrtest using -s, the first test file found +or the first test file given on the command line is run, and the name of +the next test is recorded in a file named pynexttest. If run from the +Python build directory, pynexttest is located in the 'build' subdirectory, +otherwise it is located in tempfile.gettempdir(). On subsequent runs, +the test in pynexttest is run, and the next test is written to pynexttest. +When the last test has been run, pynexttest is deleted. In this way it +is possible to single step through the test files. This is useful when +doing memory analysis on the Python interpreter, which process tends to +consume too many resources to run the full regression test non-stop. + +-S is used to continue running tests after an aborted run. It will +maintain the order a standard run (ie, this assumes -r is not used). +This is useful after the tests have prematurely stopped for some external +reason and you want to start running from where you left off rather +than starting from the beginning. + +-f reads the names of tests from the file given as f's argument, one +or more test names per line. Whitespace is ignored. Blank lines and +lines beginning with '#' are ignored. This is especially useful for +whittling down failures involving interactions among tests. + +-L causes the leaks(1) command to be run just before exit if it exists. +leaks(1) is available on Mac OS X and presumably on some other +FreeBSD-derived systems. + +-R runs each test several times and examines sys.gettotalrefcount() to +see if the test appears to be leaking references. The argument should +be of the form stab:run:fname where 'stab' is the number of times the +test is run to let gettotalrefcount settle down, 'run' is the number +of times further it is run and 'fname' is the name of the file the +reports are written to. These parameters all have defaults (5, 4 and +"reflog.txt" respectively), and the minimal invocation is '-R :'. + +-M runs tests that require an exorbitant amount of memory. These tests +typically try to ascertain containers keep working when containing more than +2 billion objects, which only works on 64-bit systems. There are also some +tests that try to exhaust the address space of the process, which only makes +sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, +which is a string in the form of '2.5Gb', determines howmuch memory the +tests will limit themselves to (but they may go slightly over.) The number +shouldn't be more memory than the machine has (including swap memory). You +should also keep in mind that swap memory is generally much, much slower +than RAM, and setting memlimit to all available RAM or higher will heavily +tax the machine. On the other hand, it is no use running these tests with a +limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect +to use more than memlimit memory will be skipped. The big-memory tests +generally run very, very long. + +-u is used to specify which special resource intensive tests to run, +such as those requiring large file support or network connectivity. +The argument is a comma-separated list of words indicating the +resources to test. Currently only the following are defined: + + all - Enable all special resources. + + none - Disable all special resources (this is the default). + + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) + + curses - Tests that use curses and will modify the terminal's + state and output modes. + + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2GB of disk space temporarily. + + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. + + decimal - Test the decimal module against a large suite that + verifies compliance with standards. + + cpu - Used for certain CPU-heavy tests. + + subprocess Run all tests for the subprocess module. + + urlfetch - It is okay to download files required on testing. + + gui - Run tests that require a running GUI. + +To enable all resources except one, use '-uall,-'. For +example, to run all the tests except for the gui tests, give the +option '-uall,-gui'. +""" + + +RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') + +class _ArgParser(argparse.ArgumentParser): + + def error(self, message): + super().error(message + "\nPass -h or --help for complete help.") + + +def _create_parser(): + # Set prog to prevent the uninformative "__main__.py" from displaying in + # error messages when using "python -m test ...". + parser = _ArgParser(prog='regrtest.py', + usage=USAGE, + description=DESCRIPTION, + epilog=EPILOG, + add_help=False, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # Arguments with this clause added to its help are described further in + # the epilog's "Additional option details" section. + more_details = ' See the section at bottom for more details.' + + group = parser.add_argument_group('General options') + # We add help explicitly to control what argument group it renders under. + group.add_argument('-h', '--help', action='help', + help='show this help message and exit') + group.add_argument('--timeout', metavar='TIMEOUT', type=float, + help='dump the traceback and exit if a test takes ' + 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'is negative or equals to zero') + group.add_argument('--wait', action='store_true', + help='wait for user input, e.g., allow a debugger ' + 'to be attached') + group.add_argument('--slaveargs', metavar='ARGS') + group.add_argument('-S', '--start', metavar='START', + help='the name of the test at which to start.' + + more_details) + + group = parser.add_argument_group('Verbosity') + group.add_argument('-v', '--verbose', action='count', + help='run tests in verbose mode with output to stdout') + group.add_argument('-w', '--verbose2', action='store_true', + help='re-run failed tests in verbose mode') + group.add_argument('-W', '--verbose3', action='store_true', + help='display test output on failure') + group.add_argument('-q', '--quiet', action='store_true', + help='no output unless one or more tests fail') + group.add_argument('-o', '--slow', action='store_true', dest='print_slow', + help='print the slowest 10 tests') + group.add_argument('--header', action='store_true', + help='print header with interpreter info') + + group = parser.add_argument_group('Selecting tests') + group.add_argument('-r', '--randomize', action='store_true', + help='randomize test execution order.' + more_details) + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a random seed to reproduce a previous ' + 'random run') + group.add_argument('-f', '--fromfile', metavar='FILE', + help='read names of tests to run from a file.' + + more_details) + group.add_argument('-x', '--exclude', action='store_true', + help='arguments are tests to *exclude*') + group.add_argument('-s', '--single', action='store_true', + help='single step through a set of tests.' + + more_details) + group.add_argument('-m', '--match', metavar='PAT', + dest='match_tests', + help='match test cases and methods with glob pattern PAT') + group.add_argument('-G', '--failfast', action='store_true', + help='fail as soon as a test fails (only with -v or -W)') + group.add_argument('-u', '--use', metavar='RES1,RES2,...', + action='append', type=resources_list, + help='specify which special resource intensive tests ' + 'to run.' + more_details) + group.add_argument('-M', '--memlimit', metavar='LIMIT', + help='run very large memory-consuming tests.' + + more_details) + group.add_argument('--testdir', metavar='DIR', + type=relative_filename, + help='execute test files in the specified directory ' + '(instead of the Python stdlib test suite)') + + group = parser.add_argument_group('Special runs') + group.add_argument('-l', '--findleaks', action='store_true', + help='if GC is available detect tests that leak memory') + group.add_argument('-L', '--runleaks', action='store_true', + help='run the leaks(1) command just before exit.' + + more_details) + group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', + type=huntrleaks, + help='search for reference leaks (needs debug build, ' + 'very slow).' + more_details) + group.add_argument('-j', '--multiprocess', metavar='PROCESSES', + dest='use_mp', type=int, + help='run PROCESSES processes at once') + group.add_argument('-T', '--coverage', action='store_true', + dest='trace', + help='turn on code coverage tracing using the trace ' + 'module') + group.add_argument('-D', '--coverdir', metavar='DIR', + type=relative_filename, + help='directory where coverage files are put') + group.add_argument('-N', '--nocoverdir', + action='store_const', const=None, dest='coverdir', + help='put coverage files alongside modules') + group.add_argument('-t', '--threshold', metavar='THRESHOLD', + type=int, + help='call gc.set_threshold(THRESHOLD)') + group.add_argument('-n', '--nowindows', action='store_true', + help='suppress error message boxes on Windows') + group.add_argument('-F', '--forever', action='store_true', + help='run the specified tests in a loop, until an ' + 'error happens') + + parser.add_argument('args', nargs=argparse.REMAINDER, + help=argparse.SUPPRESS) + + return parser + + +def relative_filename(string): + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + return os.path.join(support.SAVEDCWD, string) + + +def huntrleaks(string): + args = string.split(':') + if len(args) not in (2, 3): + raise argparse.ArgumentTypeError( + 'needs 2 or 3 colon-separated arguments') + nwarmup = int(args[0]) if args[0] else 5 + ntracked = int(args[1]) if args[1] else 4 + fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' + return nwarmup, ntracked, fname + + +def resources_list(string): + u = [x.lower() for x in string.split(',')] + for r in u: + if r == 'all' or r == 'none': + continue + if r[0] == '-': + r = r[1:] + if r not in RESOURCE_NAMES: + raise argparse.ArgumentTypeError('invalid resource: ' + r) + return u + + +def _parse_args(args, **kwargs): + # Defaults + ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, + exclude=False, single=False, randomize=False, fromfile=None, + findleaks=False, use_resources=None, trace=False, coverdir='coverage', + runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, + random_seed=None, use_mp=None, verbose3=False, forever=False, + header=False, failfast=False, match_tests=None) + for k, v in kwargs.items(): + if not hasattr(ns, k): + raise TypeError('%r is an invalid keyword argument ' + 'for this function' % k) + setattr(ns, k, v) + if ns.use_resources is None: + ns.use_resources = [] + + parser = _create_parser() + parser.parse_args(args=args, namespace=ns) + + if ns.single and ns.fromfile: + parser.error("-s and -f don't go together!") + if ns.use_mp and ns.trace: + parser.error("-T and -j don't go together!") + if ns.use_mp and ns.findleaks: + parser.error("-l and -j don't go together!") + if ns.use_mp and ns.memlimit: + parser.error("-M and -j don't go together!") + if ns.failfast and not (ns.verbose or ns.verbose3): + parser.error("-G/--failfast needs either -v or -W") + + if ns.quiet: + ns.verbose = 0 + if ns.timeout is not None: + if hasattr(faulthandler, 'dump_traceback_later'): + if ns.timeout <= 0: + ns.timeout = None + else: + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later") + ns.timeout = None + if ns.use_mp is not None: + if ns.use_mp <= 0: + # Use all cores + extras for tests that like to sleep + ns.use_mp = 2 + (os.cpu_count() or 1) + if ns.use_mp == 1: + ns.use_mp = None + if ns.use: + for a in ns.use: + for r in a: + if r == 'all': + ns.use_resources[:] = RESOURCE_NAMES + continue + if r == 'none': + del ns.use_resources[:] + continue + remove = False + if r[0] == '-': + remove = True + r = r[1:] + if remove: + if r in ns.use_resources: + ns.use_resources.remove(r) + elif r not in ns.use_resources: + ns.use_resources.append(r) + if ns.random_seed is not None: + ns.randomize = True + + return ns diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index b49e66be37..51e9e18029 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -6,119 +6,6 @@ Script to run Python regression tests. Run this script with -h or --help for documentation. """ -USAGE = """\ -python -m test [options] [test_name1 [test_name2 ...]] -python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] -""" - -DESCRIPTION = """\ -Run Python regression tests. - -If no arguments or options are provided, finds all files matching -the pattern "test_*" in the Lib/test subdirectory and runs -them in alphabetical order (but see -M and -u, below, for exceptions). - -For more rigorous testing, it is useful to use the following -command line: - -python -E -Wd -m test [options] [test_name1 ...] -""" - -EPILOG = """\ -Additional option details: - --r randomizes test execution order. You can use --randseed=int to provide a -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. - --s On the first invocation of regrtest using -s, the first test file found -or the first test file given on the command line is run, and the name of -the next test is recorded in a file named pynexttest. If run from the -Python build directory, pynexttest is located in the 'build' subdirectory, -otherwise it is located in tempfile.gettempdir(). On subsequent runs, -the test in pynexttest is run, and the next test is written to pynexttest. -When the last test has been run, pynexttest is deleted. In this way it -is possible to single step through the test files. This is useful when -doing memory analysis on the Python interpreter, which process tends to -consume too many resources to run the full regression test non-stop. - --S is used to continue running tests after an aborted run. It will -maintain the order a standard run (ie, this assumes -r is not used). -This is useful after the tests have prematurely stopped for some external -reason and you want to start running from where you left off rather -than starting from the beginning. - --f reads the names of tests from the file given as f's argument, one -or more test names per line. Whitespace is ignored. Blank lines and -lines beginning with '#' are ignored. This is especially useful for -whittling down failures involving interactions among tests. - --L causes the leaks(1) command to be run just before exit if it exists. -leaks(1) is available on Mac OS X and presumably on some other -FreeBSD-derived systems. - --R runs each test several times and examines sys.gettotalrefcount() to -see if the test appears to be leaking references. The argument should -be of the form stab:run:fname where 'stab' is the number of times the -test is run to let gettotalrefcount settle down, 'run' is the number -of times further it is run and 'fname' is the name of the file the -reports are written to. These parameters all have defaults (5, 4 and -"reflog.txt" respectively), and the minimal invocation is '-R :'. - --M runs tests that require an exorbitant amount of memory. These tests -typically try to ascertain containers keep working when containing more than -2 billion objects, which only works on 64-bit systems. There are also some -tests that try to exhaust the address space of the process, which only makes -sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, -which is a string in the form of '2.5Gb', determines howmuch memory the -tests will limit themselves to (but they may go slightly over.) The number -shouldn't be more memory than the machine has (including swap memory). You -should also keep in mind that swap memory is generally much, much slower -than RAM, and setting memlimit to all available RAM or higher will heavily -tax the machine. On the other hand, it is no use running these tests with a -limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect -to use more than memlimit memory will be skipped. The big-memory tests -generally run very, very long. - --u is used to specify which special resource intensive tests to run, -such as those requiring large file support or network connectivity. -The argument is a comma-separated list of words indicating the -resources to test. Currently only the following are defined: - - all - Enable all special resources. - - none - Disable all special resources (this is the default). - - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) - - curses - Tests that use curses and will modify the terminal's - state and output modes. - - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2GB of disk space temporarily. - - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. - - decimal - Test the decimal module against a large suite that - verifies compliance with standards. - - cpu - Used for certain CPU-heavy tests. - - subprocess Run all tests for the subprocess module. - - urlfetch - It is okay to download files required on testing. - - gui - Run tests that require a running GUI. - -To enable all resources except one, use '-uall,-'. For -example, to run all the tests except for the gui tests, give the -option '-uall,-gui'. -""" - # We import importlib *ASAP* in order to test #15386 import importlib @@ -153,6 +40,8 @@ try: except ImportError: multiprocessing = None +from test.libregrtest import _parse_args + # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some @@ -198,9 +87,6 @@ CHILD_ERROR = -5 # error in a child process from test import support -RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') - # When tests are run from the Python build directory, it is best practice # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. @@ -210,220 +96,6 @@ else: TEMPDIR = tempfile.gettempdir() TEMPDIR = os.path.abspath(TEMPDIR) -class _ArgParser(argparse.ArgumentParser): - - def error(self, message): - super().error(message + "\nPass -h or --help for complete help.") - -def _create_parser(): - # Set prog to prevent the uninformative "__main__.py" from displaying in - # error messages when using "python -m test ...". - parser = _ArgParser(prog='regrtest.py', - usage=USAGE, - description=DESCRIPTION, - epilog=EPILOG, - add_help=False, - formatter_class=argparse.RawDescriptionHelpFormatter) - - # Arguments with this clause added to its help are described further in - # the epilog's "Additional option details" section. - more_details = ' See the section at bottom for more details.' - - group = parser.add_argument_group('General options') - # We add help explicitly to control what argument group it renders under. - group.add_argument('-h', '--help', action='help', - help='show this help message and exit') - group.add_argument('--timeout', metavar='TIMEOUT', type=float, - help='dump the traceback and exit if a test takes ' - 'more than TIMEOUT seconds; disabled if TIMEOUT ' - 'is negative or equals to zero') - group.add_argument('--wait', action='store_true', - help='wait for user input, e.g., allow a debugger ' - 'to be attached') - group.add_argument('--slaveargs', metavar='ARGS') - group.add_argument('-S', '--start', metavar='START', - help='the name of the test at which to start.' + - more_details) - - group = parser.add_argument_group('Verbosity') - group.add_argument('-v', '--verbose', action='count', - help='run tests in verbose mode with output to stdout') - group.add_argument('-w', '--verbose2', action='store_true', - help='re-run failed tests in verbose mode') - group.add_argument('-W', '--verbose3', action='store_true', - help='display test output on failure') - group.add_argument('-q', '--quiet', action='store_true', - help='no output unless one or more tests fail') - group.add_argument('-o', '--slow', action='store_true', dest='print_slow', - help='print the slowest 10 tests') - group.add_argument('--header', action='store_true', - help='print header with interpreter info') - - group = parser.add_argument_group('Selecting tests') - group.add_argument('-r', '--randomize', action='store_true', - help='randomize test execution order.' + more_details) - group.add_argument('--randseed', metavar='SEED', - dest='random_seed', type=int, - help='pass a random seed to reproduce a previous ' - 'random run') - group.add_argument('-f', '--fromfile', metavar='FILE', - help='read names of tests to run from a file.' + - more_details) - group.add_argument('-x', '--exclude', action='store_true', - help='arguments are tests to *exclude*') - group.add_argument('-s', '--single', action='store_true', - help='single step through a set of tests.' + - more_details) - group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', - help='match test cases and methods with glob pattern PAT') - group.add_argument('-G', '--failfast', action='store_true', - help='fail as soon as a test fails (only with -v or -W)') - group.add_argument('-u', '--use', metavar='RES1,RES2,...', - action='append', type=resources_list, - help='specify which special resource intensive tests ' - 'to run.' + more_details) - group.add_argument('-M', '--memlimit', metavar='LIMIT', - help='run very large memory-consuming tests.' + - more_details) - group.add_argument('--testdir', metavar='DIR', - type=relative_filename, - help='execute test files in the specified directory ' - '(instead of the Python stdlib test suite)') - - group = parser.add_argument_group('Special runs') - group.add_argument('-l', '--findleaks', action='store_true', - help='if GC is available detect tests that leak memory') - group.add_argument('-L', '--runleaks', action='store_true', - help='run the leaks(1) command just before exit.' + - more_details) - group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', - type=huntrleaks, - help='search for reference leaks (needs debug build, ' - 'very slow).' + more_details) - group.add_argument('-j', '--multiprocess', metavar='PROCESSES', - dest='use_mp', type=int, - help='run PROCESSES processes at once') - group.add_argument('-T', '--coverage', action='store_true', - dest='trace', - help='turn on code coverage tracing using the trace ' - 'module') - group.add_argument('-D', '--coverdir', metavar='DIR', - type=relative_filename, - help='directory where coverage files are put') - group.add_argument('-N', '--nocoverdir', - action='store_const', const=None, dest='coverdir', - help='put coverage files alongside modules') - group.add_argument('-t', '--threshold', metavar='THRESHOLD', - type=int, - help='call gc.set_threshold(THRESHOLD)') - group.add_argument('-n', '--nowindows', action='store_true', - help='suppress error message boxes on Windows') - group.add_argument('-F', '--forever', action='store_true', - help='run the specified tests in a loop, until an ' - 'error happens') - - parser.add_argument('args', nargs=argparse.REMAINDER, - help=argparse.SUPPRESS) - - return parser - -def relative_filename(string): - # CWD is replaced with a temporary dir before calling main(), so we - # join it with the saved CWD so it ends up where the user expects. - return os.path.join(support.SAVEDCWD, string) - -def huntrleaks(string): - args = string.split(':') - if len(args) not in (2, 3): - raise argparse.ArgumentTypeError( - 'needs 2 or 3 colon-separated arguments') - nwarmup = int(args[0]) if args[0] else 5 - ntracked = int(args[1]) if args[1] else 4 - fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' - return nwarmup, ntracked, fname - -def resources_list(string): - u = [x.lower() for x in string.split(',')] - for r in u: - if r == 'all' or r == 'none': - continue - if r[0] == '-': - r = r[1:] - if r not in RESOURCE_NAMES: - raise argparse.ArgumentTypeError('invalid resource: ' + r) - return u - -def _parse_args(args, **kwargs): - # Defaults - ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, - exclude=False, single=False, randomize=False, fromfile=None, - findleaks=False, use_resources=None, trace=False, coverdir='coverage', - runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, - random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None) - for k, v in kwargs.items(): - if not hasattr(ns, k): - raise TypeError('%r is an invalid keyword argument ' - 'for this function' % k) - setattr(ns, k, v) - if ns.use_resources is None: - ns.use_resources = [] - - parser = _create_parser() - parser.parse_args(args=args, namespace=ns) - - if ns.single and ns.fromfile: - parser.error("-s and -f don't go together!") - if ns.use_mp and ns.trace: - parser.error("-T and -j don't go together!") - if ns.use_mp and ns.findleaks: - parser.error("-l and -j don't go together!") - if ns.use_mp and ns.memlimit: - parser.error("-M and -j don't go together!") - if ns.failfast and not (ns.verbose or ns.verbose3): - parser.error("-G/--failfast needs either -v or -W") - - if ns.quiet: - ns.verbose = 0 - if ns.timeout is not None: - if hasattr(faulthandler, 'dump_traceback_later'): - if ns.timeout <= 0: - ns.timeout = None - else: - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later") - ns.timeout = None - if ns.use_mp is not None: - if ns.use_mp <= 0: - # Use all cores + extras for tests that like to sleep - ns.use_mp = 2 + (os.cpu_count() or 1) - if ns.use_mp == 1: - ns.use_mp = None - if ns.use: - for a in ns.use: - for r in a: - if r == 'all': - ns.use_resources[:] = RESOURCE_NAMES - continue - if r == 'none': - del ns.use_resources[:] - continue - remove = False - if r[0] == '-': - remove = True - r = r[1:] - if remove: - if r in ns.use_resources: - ns.use_resources.remove(r) - elif r not in ns.use_resources: - ns.use_resources.append(r) - if ns.random_seed is not None: - ns.randomize = True - - return ns - - def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a398a4f836..cf4b84fe19 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,7 +7,7 @@ import faulthandler import getopt import os.path import unittest -from test import regrtest, support +from test import regrtest, support, libregrtest class ParseArgsTestCase(unittest.TestCase): @@ -148,7 +148,7 @@ class ParseArgsTestCase(unittest.TestCase): self.assertEqual(ns.use_resources, ['gui', 'network']) ns = regrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(regrtest.RESOURCE_NAMES) + expected = list(libregrtest.RESOURCE_NAMES) expected.remove('gui') ns = regrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) -- cgit v1.2.1 From 257d6edddac7b70f1dc8bd20d84c4ee1d6acdc9a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Sep 2015 23:16:47 +0200 Subject: Issue #25220: Backed out changeset eaf9a99b6bb8 --- Lib/test/libregrtest/__init__.py | 1 - Lib/test/libregrtest/cmdline.py | 340 --------------------------------------- Lib/test/regrtest.py | 332 +++++++++++++++++++++++++++++++++++++- Lib/test/test_regrtest.py | 4 +- 4 files changed, 332 insertions(+), 345 deletions(-) delete mode 100644 Lib/test/libregrtest/__init__.py delete mode 100644 Lib/test/libregrtest/cmdline.py (limited to 'Lib') diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py deleted file mode 100644 index 554aeede4a..0000000000 --- a/Lib/test/libregrtest/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py deleted file mode 100644 index 94649965a5..0000000000 --- a/Lib/test/libregrtest/cmdline.py +++ /dev/null @@ -1,340 +0,0 @@ -import argparse -import faulthandler -import os - -from test import support - -USAGE = """\ -python -m test [options] [test_name1 [test_name2 ...]] -python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] -""" - -DESCRIPTION = """\ -Run Python regression tests. - -If no arguments or options are provided, finds all files matching -the pattern "test_*" in the Lib/test subdirectory and runs -them in alphabetical order (but see -M and -u, below, for exceptions). - -For more rigorous testing, it is useful to use the following -command line: - -python -E -Wd -m test [options] [test_name1 ...] -""" - -EPILOG = """\ -Additional option details: - --r randomizes test execution order. You can use --randseed=int to provide a -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. - --s On the first invocation of regrtest using -s, the first test file found -or the first test file given on the command line is run, and the name of -the next test is recorded in a file named pynexttest. If run from the -Python build directory, pynexttest is located in the 'build' subdirectory, -otherwise it is located in tempfile.gettempdir(). On subsequent runs, -the test in pynexttest is run, and the next test is written to pynexttest. -When the last test has been run, pynexttest is deleted. In this way it -is possible to single step through the test files. This is useful when -doing memory analysis on the Python interpreter, which process tends to -consume too many resources to run the full regression test non-stop. - --S is used to continue running tests after an aborted run. It will -maintain the order a standard run (ie, this assumes -r is not used). -This is useful after the tests have prematurely stopped for some external -reason and you want to start running from where you left off rather -than starting from the beginning. - --f reads the names of tests from the file given as f's argument, one -or more test names per line. Whitespace is ignored. Blank lines and -lines beginning with '#' are ignored. This is especially useful for -whittling down failures involving interactions among tests. - --L causes the leaks(1) command to be run just before exit if it exists. -leaks(1) is available on Mac OS X and presumably on some other -FreeBSD-derived systems. - --R runs each test several times and examines sys.gettotalrefcount() to -see if the test appears to be leaking references. The argument should -be of the form stab:run:fname where 'stab' is the number of times the -test is run to let gettotalrefcount settle down, 'run' is the number -of times further it is run and 'fname' is the name of the file the -reports are written to. These parameters all have defaults (5, 4 and -"reflog.txt" respectively), and the minimal invocation is '-R :'. - --M runs tests that require an exorbitant amount of memory. These tests -typically try to ascertain containers keep working when containing more than -2 billion objects, which only works on 64-bit systems. There are also some -tests that try to exhaust the address space of the process, which only makes -sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, -which is a string in the form of '2.5Gb', determines howmuch memory the -tests will limit themselves to (but they may go slightly over.) The number -shouldn't be more memory than the machine has (including swap memory). You -should also keep in mind that swap memory is generally much, much slower -than RAM, and setting memlimit to all available RAM or higher will heavily -tax the machine. On the other hand, it is no use running these tests with a -limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect -to use more than memlimit memory will be skipped. The big-memory tests -generally run very, very long. - --u is used to specify which special resource intensive tests to run, -such as those requiring large file support or network connectivity. -The argument is a comma-separated list of words indicating the -resources to test. Currently only the following are defined: - - all - Enable all special resources. - - none - Disable all special resources (this is the default). - - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) - - curses - Tests that use curses and will modify the terminal's - state and output modes. - - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2GB of disk space temporarily. - - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. - - decimal - Test the decimal module against a large suite that - verifies compliance with standards. - - cpu - Used for certain CPU-heavy tests. - - subprocess Run all tests for the subprocess module. - - urlfetch - It is okay to download files required on testing. - - gui - Run tests that require a running GUI. - -To enable all resources except one, use '-uall,-'. For -example, to run all the tests except for the gui tests, give the -option '-uall,-gui'. -""" - - -RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') - -class _ArgParser(argparse.ArgumentParser): - - def error(self, message): - super().error(message + "\nPass -h or --help for complete help.") - - -def _create_parser(): - # Set prog to prevent the uninformative "__main__.py" from displaying in - # error messages when using "python -m test ...". - parser = _ArgParser(prog='regrtest.py', - usage=USAGE, - description=DESCRIPTION, - epilog=EPILOG, - add_help=False, - formatter_class=argparse.RawDescriptionHelpFormatter) - - # Arguments with this clause added to its help are described further in - # the epilog's "Additional option details" section. - more_details = ' See the section at bottom for more details.' - - group = parser.add_argument_group('General options') - # We add help explicitly to control what argument group it renders under. - group.add_argument('-h', '--help', action='help', - help='show this help message and exit') - group.add_argument('--timeout', metavar='TIMEOUT', type=float, - help='dump the traceback and exit if a test takes ' - 'more than TIMEOUT seconds; disabled if TIMEOUT ' - 'is negative or equals to zero') - group.add_argument('--wait', action='store_true', - help='wait for user input, e.g., allow a debugger ' - 'to be attached') - group.add_argument('--slaveargs', metavar='ARGS') - group.add_argument('-S', '--start', metavar='START', - help='the name of the test at which to start.' + - more_details) - - group = parser.add_argument_group('Verbosity') - group.add_argument('-v', '--verbose', action='count', - help='run tests in verbose mode with output to stdout') - group.add_argument('-w', '--verbose2', action='store_true', - help='re-run failed tests in verbose mode') - group.add_argument('-W', '--verbose3', action='store_true', - help='display test output on failure') - group.add_argument('-q', '--quiet', action='store_true', - help='no output unless one or more tests fail') - group.add_argument('-o', '--slow', action='store_true', dest='print_slow', - help='print the slowest 10 tests') - group.add_argument('--header', action='store_true', - help='print header with interpreter info') - - group = parser.add_argument_group('Selecting tests') - group.add_argument('-r', '--randomize', action='store_true', - help='randomize test execution order.' + more_details) - group.add_argument('--randseed', metavar='SEED', - dest='random_seed', type=int, - help='pass a random seed to reproduce a previous ' - 'random run') - group.add_argument('-f', '--fromfile', metavar='FILE', - help='read names of tests to run from a file.' + - more_details) - group.add_argument('-x', '--exclude', action='store_true', - help='arguments are tests to *exclude*') - group.add_argument('-s', '--single', action='store_true', - help='single step through a set of tests.' + - more_details) - group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', - help='match test cases and methods with glob pattern PAT') - group.add_argument('-G', '--failfast', action='store_true', - help='fail as soon as a test fails (only with -v or -W)') - group.add_argument('-u', '--use', metavar='RES1,RES2,...', - action='append', type=resources_list, - help='specify which special resource intensive tests ' - 'to run.' + more_details) - group.add_argument('-M', '--memlimit', metavar='LIMIT', - help='run very large memory-consuming tests.' + - more_details) - group.add_argument('--testdir', metavar='DIR', - type=relative_filename, - help='execute test files in the specified directory ' - '(instead of the Python stdlib test suite)') - - group = parser.add_argument_group('Special runs') - group.add_argument('-l', '--findleaks', action='store_true', - help='if GC is available detect tests that leak memory') - group.add_argument('-L', '--runleaks', action='store_true', - help='run the leaks(1) command just before exit.' + - more_details) - group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', - type=huntrleaks, - help='search for reference leaks (needs debug build, ' - 'very slow).' + more_details) - group.add_argument('-j', '--multiprocess', metavar='PROCESSES', - dest='use_mp', type=int, - help='run PROCESSES processes at once') - group.add_argument('-T', '--coverage', action='store_true', - dest='trace', - help='turn on code coverage tracing using the trace ' - 'module') - group.add_argument('-D', '--coverdir', metavar='DIR', - type=relative_filename, - help='directory where coverage files are put') - group.add_argument('-N', '--nocoverdir', - action='store_const', const=None, dest='coverdir', - help='put coverage files alongside modules') - group.add_argument('-t', '--threshold', metavar='THRESHOLD', - type=int, - help='call gc.set_threshold(THRESHOLD)') - group.add_argument('-n', '--nowindows', action='store_true', - help='suppress error message boxes on Windows') - group.add_argument('-F', '--forever', action='store_true', - help='run the specified tests in a loop, until an ' - 'error happens') - - parser.add_argument('args', nargs=argparse.REMAINDER, - help=argparse.SUPPRESS) - - return parser - - -def relative_filename(string): - # CWD is replaced with a temporary dir before calling main(), so we - # join it with the saved CWD so it ends up where the user expects. - return os.path.join(support.SAVEDCWD, string) - - -def huntrleaks(string): - args = string.split(':') - if len(args) not in (2, 3): - raise argparse.ArgumentTypeError( - 'needs 2 or 3 colon-separated arguments') - nwarmup = int(args[0]) if args[0] else 5 - ntracked = int(args[1]) if args[1] else 4 - fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' - return nwarmup, ntracked, fname - - -def resources_list(string): - u = [x.lower() for x in string.split(',')] - for r in u: - if r == 'all' or r == 'none': - continue - if r[0] == '-': - r = r[1:] - if r not in RESOURCE_NAMES: - raise argparse.ArgumentTypeError('invalid resource: ' + r) - return u - - -def _parse_args(args, **kwargs): - # Defaults - ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, - exclude=False, single=False, randomize=False, fromfile=None, - findleaks=False, use_resources=None, trace=False, coverdir='coverage', - runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, - random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None) - for k, v in kwargs.items(): - if not hasattr(ns, k): - raise TypeError('%r is an invalid keyword argument ' - 'for this function' % k) - setattr(ns, k, v) - if ns.use_resources is None: - ns.use_resources = [] - - parser = _create_parser() - parser.parse_args(args=args, namespace=ns) - - if ns.single and ns.fromfile: - parser.error("-s and -f don't go together!") - if ns.use_mp and ns.trace: - parser.error("-T and -j don't go together!") - if ns.use_mp and ns.findleaks: - parser.error("-l and -j don't go together!") - if ns.use_mp and ns.memlimit: - parser.error("-M and -j don't go together!") - if ns.failfast and not (ns.verbose or ns.verbose3): - parser.error("-G/--failfast needs either -v or -W") - - if ns.quiet: - ns.verbose = 0 - if ns.timeout is not None: - if hasattr(faulthandler, 'dump_traceback_later'): - if ns.timeout <= 0: - ns.timeout = None - else: - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later") - ns.timeout = None - if ns.use_mp is not None: - if ns.use_mp <= 0: - # Use all cores + extras for tests that like to sleep - ns.use_mp = 2 + (os.cpu_count() or 1) - if ns.use_mp == 1: - ns.use_mp = None - if ns.use: - for a in ns.use: - for r in a: - if r == 'all': - ns.use_resources[:] = RESOURCE_NAMES - continue - if r == 'none': - del ns.use_resources[:] - continue - remove = False - if r[0] == '-': - remove = True - r = r[1:] - if remove: - if r in ns.use_resources: - ns.use_resources.remove(r) - elif r not in ns.use_resources: - ns.use_resources.append(r) - if ns.random_seed is not None: - ns.randomize = True - - return ns diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 51e9e18029..b49e66be37 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -6,6 +6,119 @@ Script to run Python regression tests. Run this script with -h or --help for documentation. """ +USAGE = """\ +python -m test [options] [test_name1 [test_name2 ...]] +python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] +""" + +DESCRIPTION = """\ +Run Python regression tests. + +If no arguments or options are provided, finds all files matching +the pattern "test_*" in the Lib/test subdirectory and runs +them in alphabetical order (but see -M and -u, below, for exceptions). + +For more rigorous testing, it is useful to use the following +command line: + +python -E -Wd -m test [options] [test_name1 ...] +""" + +EPILOG = """\ +Additional option details: + +-r randomizes test execution order. You can use --randseed=int to provide a +int seed value for the randomizer; this is useful for reproducing troublesome +test orders. + +-s On the first invocation of regrtest using -s, the first test file found +or the first test file given on the command line is run, and the name of +the next test is recorded in a file named pynexttest. If run from the +Python build directory, pynexttest is located in the 'build' subdirectory, +otherwise it is located in tempfile.gettempdir(). On subsequent runs, +the test in pynexttest is run, and the next test is written to pynexttest. +When the last test has been run, pynexttest is deleted. In this way it +is possible to single step through the test files. This is useful when +doing memory analysis on the Python interpreter, which process tends to +consume too many resources to run the full regression test non-stop. + +-S is used to continue running tests after an aborted run. It will +maintain the order a standard run (ie, this assumes -r is not used). +This is useful after the tests have prematurely stopped for some external +reason and you want to start running from where you left off rather +than starting from the beginning. + +-f reads the names of tests from the file given as f's argument, one +or more test names per line. Whitespace is ignored. Blank lines and +lines beginning with '#' are ignored. This is especially useful for +whittling down failures involving interactions among tests. + +-L causes the leaks(1) command to be run just before exit if it exists. +leaks(1) is available on Mac OS X and presumably on some other +FreeBSD-derived systems. + +-R runs each test several times and examines sys.gettotalrefcount() to +see if the test appears to be leaking references. The argument should +be of the form stab:run:fname where 'stab' is the number of times the +test is run to let gettotalrefcount settle down, 'run' is the number +of times further it is run and 'fname' is the name of the file the +reports are written to. These parameters all have defaults (5, 4 and +"reflog.txt" respectively), and the minimal invocation is '-R :'. + +-M runs tests that require an exorbitant amount of memory. These tests +typically try to ascertain containers keep working when containing more than +2 billion objects, which only works on 64-bit systems. There are also some +tests that try to exhaust the address space of the process, which only makes +sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, +which is a string in the form of '2.5Gb', determines howmuch memory the +tests will limit themselves to (but they may go slightly over.) The number +shouldn't be more memory than the machine has (including swap memory). You +should also keep in mind that swap memory is generally much, much slower +than RAM, and setting memlimit to all available RAM or higher will heavily +tax the machine. On the other hand, it is no use running these tests with a +limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect +to use more than memlimit memory will be skipped. The big-memory tests +generally run very, very long. + +-u is used to specify which special resource intensive tests to run, +such as those requiring large file support or network connectivity. +The argument is a comma-separated list of words indicating the +resources to test. Currently only the following are defined: + + all - Enable all special resources. + + none - Disable all special resources (this is the default). + + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) + + curses - Tests that use curses and will modify the terminal's + state and output modes. + + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2GB of disk space temporarily. + + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. + + decimal - Test the decimal module against a large suite that + verifies compliance with standards. + + cpu - Used for certain CPU-heavy tests. + + subprocess Run all tests for the subprocess module. + + urlfetch - It is okay to download files required on testing. + + gui - Run tests that require a running GUI. + +To enable all resources except one, use '-uall,-'. For +example, to run all the tests except for the gui tests, give the +option '-uall,-gui'. +""" + # We import importlib *ASAP* in order to test #15386 import importlib @@ -40,8 +153,6 @@ try: except ImportError: multiprocessing = None -from test.libregrtest import _parse_args - # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some @@ -87,6 +198,9 @@ CHILD_ERROR = -5 # error in a child process from test import support +RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') + # When tests are run from the Python build directory, it is best practice # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. @@ -96,6 +210,220 @@ else: TEMPDIR = tempfile.gettempdir() TEMPDIR = os.path.abspath(TEMPDIR) +class _ArgParser(argparse.ArgumentParser): + + def error(self, message): + super().error(message + "\nPass -h or --help for complete help.") + +def _create_parser(): + # Set prog to prevent the uninformative "__main__.py" from displaying in + # error messages when using "python -m test ...". + parser = _ArgParser(prog='regrtest.py', + usage=USAGE, + description=DESCRIPTION, + epilog=EPILOG, + add_help=False, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # Arguments with this clause added to its help are described further in + # the epilog's "Additional option details" section. + more_details = ' See the section at bottom for more details.' + + group = parser.add_argument_group('General options') + # We add help explicitly to control what argument group it renders under. + group.add_argument('-h', '--help', action='help', + help='show this help message and exit') + group.add_argument('--timeout', metavar='TIMEOUT', type=float, + help='dump the traceback and exit if a test takes ' + 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'is negative or equals to zero') + group.add_argument('--wait', action='store_true', + help='wait for user input, e.g., allow a debugger ' + 'to be attached') + group.add_argument('--slaveargs', metavar='ARGS') + group.add_argument('-S', '--start', metavar='START', + help='the name of the test at which to start.' + + more_details) + + group = parser.add_argument_group('Verbosity') + group.add_argument('-v', '--verbose', action='count', + help='run tests in verbose mode with output to stdout') + group.add_argument('-w', '--verbose2', action='store_true', + help='re-run failed tests in verbose mode') + group.add_argument('-W', '--verbose3', action='store_true', + help='display test output on failure') + group.add_argument('-q', '--quiet', action='store_true', + help='no output unless one or more tests fail') + group.add_argument('-o', '--slow', action='store_true', dest='print_slow', + help='print the slowest 10 tests') + group.add_argument('--header', action='store_true', + help='print header with interpreter info') + + group = parser.add_argument_group('Selecting tests') + group.add_argument('-r', '--randomize', action='store_true', + help='randomize test execution order.' + more_details) + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a random seed to reproduce a previous ' + 'random run') + group.add_argument('-f', '--fromfile', metavar='FILE', + help='read names of tests to run from a file.' + + more_details) + group.add_argument('-x', '--exclude', action='store_true', + help='arguments are tests to *exclude*') + group.add_argument('-s', '--single', action='store_true', + help='single step through a set of tests.' + + more_details) + group.add_argument('-m', '--match', metavar='PAT', + dest='match_tests', + help='match test cases and methods with glob pattern PAT') + group.add_argument('-G', '--failfast', action='store_true', + help='fail as soon as a test fails (only with -v or -W)') + group.add_argument('-u', '--use', metavar='RES1,RES2,...', + action='append', type=resources_list, + help='specify which special resource intensive tests ' + 'to run.' + more_details) + group.add_argument('-M', '--memlimit', metavar='LIMIT', + help='run very large memory-consuming tests.' + + more_details) + group.add_argument('--testdir', metavar='DIR', + type=relative_filename, + help='execute test files in the specified directory ' + '(instead of the Python stdlib test suite)') + + group = parser.add_argument_group('Special runs') + group.add_argument('-l', '--findleaks', action='store_true', + help='if GC is available detect tests that leak memory') + group.add_argument('-L', '--runleaks', action='store_true', + help='run the leaks(1) command just before exit.' + + more_details) + group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', + type=huntrleaks, + help='search for reference leaks (needs debug build, ' + 'very slow).' + more_details) + group.add_argument('-j', '--multiprocess', metavar='PROCESSES', + dest='use_mp', type=int, + help='run PROCESSES processes at once') + group.add_argument('-T', '--coverage', action='store_true', + dest='trace', + help='turn on code coverage tracing using the trace ' + 'module') + group.add_argument('-D', '--coverdir', metavar='DIR', + type=relative_filename, + help='directory where coverage files are put') + group.add_argument('-N', '--nocoverdir', + action='store_const', const=None, dest='coverdir', + help='put coverage files alongside modules') + group.add_argument('-t', '--threshold', metavar='THRESHOLD', + type=int, + help='call gc.set_threshold(THRESHOLD)') + group.add_argument('-n', '--nowindows', action='store_true', + help='suppress error message boxes on Windows') + group.add_argument('-F', '--forever', action='store_true', + help='run the specified tests in a loop, until an ' + 'error happens') + + parser.add_argument('args', nargs=argparse.REMAINDER, + help=argparse.SUPPRESS) + + return parser + +def relative_filename(string): + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + return os.path.join(support.SAVEDCWD, string) + +def huntrleaks(string): + args = string.split(':') + if len(args) not in (2, 3): + raise argparse.ArgumentTypeError( + 'needs 2 or 3 colon-separated arguments') + nwarmup = int(args[0]) if args[0] else 5 + ntracked = int(args[1]) if args[1] else 4 + fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' + return nwarmup, ntracked, fname + +def resources_list(string): + u = [x.lower() for x in string.split(',')] + for r in u: + if r == 'all' or r == 'none': + continue + if r[0] == '-': + r = r[1:] + if r not in RESOURCE_NAMES: + raise argparse.ArgumentTypeError('invalid resource: ' + r) + return u + +def _parse_args(args, **kwargs): + # Defaults + ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, + exclude=False, single=False, randomize=False, fromfile=None, + findleaks=False, use_resources=None, trace=False, coverdir='coverage', + runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, + random_seed=None, use_mp=None, verbose3=False, forever=False, + header=False, failfast=False, match_tests=None) + for k, v in kwargs.items(): + if not hasattr(ns, k): + raise TypeError('%r is an invalid keyword argument ' + 'for this function' % k) + setattr(ns, k, v) + if ns.use_resources is None: + ns.use_resources = [] + + parser = _create_parser() + parser.parse_args(args=args, namespace=ns) + + if ns.single and ns.fromfile: + parser.error("-s and -f don't go together!") + if ns.use_mp and ns.trace: + parser.error("-T and -j don't go together!") + if ns.use_mp and ns.findleaks: + parser.error("-l and -j don't go together!") + if ns.use_mp and ns.memlimit: + parser.error("-M and -j don't go together!") + if ns.failfast and not (ns.verbose or ns.verbose3): + parser.error("-G/--failfast needs either -v or -W") + + if ns.quiet: + ns.verbose = 0 + if ns.timeout is not None: + if hasattr(faulthandler, 'dump_traceback_later'): + if ns.timeout <= 0: + ns.timeout = None + else: + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later") + ns.timeout = None + if ns.use_mp is not None: + if ns.use_mp <= 0: + # Use all cores + extras for tests that like to sleep + ns.use_mp = 2 + (os.cpu_count() or 1) + if ns.use_mp == 1: + ns.use_mp = None + if ns.use: + for a in ns.use: + for r in a: + if r == 'all': + ns.use_resources[:] = RESOURCE_NAMES + continue + if r == 'none': + del ns.use_resources[:] + continue + remove = False + if r[0] == '-': + remove = True + r = r[1:] + if remove: + if r in ns.use_resources: + ns.use_resources.remove(r) + elif r not in ns.use_resources: + ns.use_resources.append(r) + if ns.random_seed is not None: + ns.randomize = True + + return ns + + def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index cf4b84fe19..a398a4f836 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,7 +7,7 @@ import faulthandler import getopt import os.path import unittest -from test import regrtest, support, libregrtest +from test import regrtest, support class ParseArgsTestCase(unittest.TestCase): @@ -148,7 +148,7 @@ class ParseArgsTestCase(unittest.TestCase): self.assertEqual(ns.use_resources, ['gui', 'network']) ns = regrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(libregrtest.RESOURCE_NAMES) + expected = list(regrtest.RESOURCE_NAMES) expected.remove('gui') ns = regrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) -- cgit v1.2.1 From 1edcfe4d63b13305db3d98f611ec3df76e8e587e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 25 Sep 2015 13:05:13 -0700 Subject: Issue #25186: Remove duplicated function from importlib._bootstrap_external --- Lib/importlib/_bootstrap_external.py | 48 +++++++++++++++++------------------- 1 file changed, 22 insertions(+), 26 deletions(-) (limited to 'Lib') diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 3508ce96b0..abde6d9afe 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -360,14 +360,6 @@ def _calc_mode(path): return mode -def _verbose_message(message, *args, verbosity=1): - """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" - if sys.flags.verbose >= verbosity: - if not message.startswith(('#', 'import ')): - message = '# ' + message - print(message.format(*args), file=sys.stderr) - - def _check_name(method): """Decorator to verify that the module being requested matches the one the loader can handle. @@ -437,15 +429,15 @@ def _validate_bytecode_header(data, source_stats=None, name=None, path=None): raw_size = data[8:12] if magic != MAGIC_NUMBER: message = 'bad magic number in {!r}: {!r}'.format(name, magic) - _verbose_message(message) + _bootstrap._verbose_message(message) raise ImportError(message, **exc_details) elif len(raw_timestamp) != 4: message = 'reached EOF while reading timestamp in {!r}'.format(name) - _verbose_message(message) + _bootstrap._verbose_message(message) raise EOFError(message) elif len(raw_size) != 4: message = 'reached EOF while reading size of source in {!r}'.format(name) - _verbose_message(message) + _bootstrap._verbose_message(message) raise EOFError(message) if source_stats is not None: try: @@ -455,7 +447,7 @@ def _validate_bytecode_header(data, source_stats=None, name=None, path=None): else: if _r_long(raw_timestamp) != source_mtime: message = 'bytecode is stale for {!r}'.format(name) - _verbose_message(message) + _bootstrap._verbose_message(message) raise ImportError(message, **exc_details) try: source_size = source_stats['size'] & 0xFFFFFFFF @@ -472,7 +464,7 @@ def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None): """Compile bytecode as returned by _validate_bytecode_header().""" code = marshal.loads(data) if isinstance(code, _code_type): - _verbose_message('code object from {!r}', bytecode_path) + _bootstrap._verbose_message('code object from {!r}', bytecode_path) if source_path is not None: _imp._fix_co_filename(code, source_path) return code @@ -755,21 +747,21 @@ class SourceLoader(_LoaderBasics): except (ImportError, EOFError): pass else: - _verbose_message('{} matches {}', bytecode_path, - source_path) + _bootstrap._verbose_message('{} matches {}', bytecode_path, + source_path) return _compile_bytecode(bytes_data, name=fullname, bytecode_path=bytecode_path, source_path=source_path) source_bytes = self.get_data(source_path) code_object = self.source_to_code(source_bytes, source_path) - _verbose_message('code object from {}', source_path) + _bootstrap._verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and source_mtime is not None): data = _code_to_bytecode(code_object, source_mtime, len(source_bytes)) try: self._cache_bytecode(source_path, bytecode_path, data) - _verbose_message('wrote {!r}', bytecode_path) + _bootstrap._verbose_message('wrote {!r}', bytecode_path) except NotImplementedError: pass return code_object @@ -849,14 +841,16 @@ class SourceFileLoader(FileLoader, SourceLoader): except OSError as exc: # Could be a permission error, read-only filesystem: just forget # about writing the data. - _verbose_message('could not create {!r}: {!r}', parent, exc) + _bootstrap._verbose_message('could not create {!r}: {!r}', + parent, exc) return try: _write_atomic(path, data, _mode) - _verbose_message('created {!r}', path) + _bootstrap._verbose_message('created {!r}', path) except OSError as exc: # Same as above: just don't write the bytecode. - _verbose_message('could not create {!r}: {!r}', path, exc) + _bootstrap._verbose_message('could not create {!r}: {!r}', path, + exc) class SourcelessFileLoader(FileLoader, _LoaderBasics): @@ -901,14 +895,14 @@ class ExtensionFileLoader(FileLoader, _LoaderBasics): """Create an unitialized extension module""" module = _bootstrap._call_with_frames_removed( _imp.create_dynamic, spec) - _verbose_message('extension module {!r} loaded from {!r}', + _bootstrap._verbose_message('extension module {!r} loaded from {!r}', spec.name, self.path) return module def exec_module(self, module): """Initialize an extension module""" _bootstrap._call_with_frames_removed(_imp.exec_dynamic, module) - _verbose_message('extension module {!r} executed from {!r}', + _bootstrap._verbose_message('extension module {!r} executed from {!r}', self.name, self.path) def is_package(self, fullname): @@ -1023,7 +1017,8 @@ class _NamespaceLoader: """ # The import system never calls this method. - _verbose_message('namespace module loaded with path {!r}', self._path) + _bootstrap._verbose_message('namespace module loaded with path {!r}', + self._path) return _bootstrap._load_module_shim(self, fullname) @@ -1243,12 +1238,13 @@ class FileFinder: # Check for a file w/ a proper suffix exists. for suffix, loader_class in self._loaders: full_path = _path_join(self.path, tail_module + suffix) - _verbose_message('trying {}'.format(full_path), verbosity=2) + _bootstrap._verbose_message('trying {}', full_path, verbosity=2) if cache_module + suffix in cache: if _path_isfile(full_path): - return self._get_spec(loader_class, fullname, full_path, None, target) + return self._get_spec(loader_class, fullname, full_path, + None, target) if is_namespace: - _verbose_message('possible namespace for {}'.format(base_path)) + _bootstrap._verbose_message('possible namespace for {}', base_path) spec = _bootstrap.ModuleSpec(fullname, None) spec.submodule_search_locations = [base_path] return spec -- cgit v1.2.1 From 11d373e548de9451949e6e1dbbaacb2dae3ac77a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Sep 2015 09:43:45 +0200 Subject: Issue #25220: Create Lib/test/libregrtest/ Start to split regrtest.py into smaller parts with the creation of Lib/test/libregrtest/cmdline.py: code to handle the command line, especially parsing command line arguments. This part of the code is tested by test_regrtest. --- Lib/test/libregrtest/__init__.py | 1 + Lib/test/libregrtest/cmdline.py | 340 +++++++++++++++++++++++++++++++++++++++ Lib/test/regrtest.py | 332 +------------------------------------- Lib/test/test_regrtest.py | 4 +- 4 files changed, 345 insertions(+), 332 deletions(-) create mode 100644 Lib/test/libregrtest/__init__.py create mode 100644 Lib/test/libregrtest/cmdline.py (limited to 'Lib') diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py new file mode 100644 index 0000000000..554aeede4a --- /dev/null +++ b/Lib/test/libregrtest/__init__.py @@ -0,0 +1 @@ +from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py new file mode 100644 index 0000000000..94649965a5 --- /dev/null +++ b/Lib/test/libregrtest/cmdline.py @@ -0,0 +1,340 @@ +import argparse +import faulthandler +import os + +from test import support + +USAGE = """\ +python -m test [options] [test_name1 [test_name2 ...]] +python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] +""" + +DESCRIPTION = """\ +Run Python regression tests. + +If no arguments or options are provided, finds all files matching +the pattern "test_*" in the Lib/test subdirectory and runs +them in alphabetical order (but see -M and -u, below, for exceptions). + +For more rigorous testing, it is useful to use the following +command line: + +python -E -Wd -m test [options] [test_name1 ...] +""" + +EPILOG = """\ +Additional option details: + +-r randomizes test execution order. You can use --randseed=int to provide a +int seed value for the randomizer; this is useful for reproducing troublesome +test orders. + +-s On the first invocation of regrtest using -s, the first test file found +or the first test file given on the command line is run, and the name of +the next test is recorded in a file named pynexttest. If run from the +Python build directory, pynexttest is located in the 'build' subdirectory, +otherwise it is located in tempfile.gettempdir(). On subsequent runs, +the test in pynexttest is run, and the next test is written to pynexttest. +When the last test has been run, pynexttest is deleted. In this way it +is possible to single step through the test files. This is useful when +doing memory analysis on the Python interpreter, which process tends to +consume too many resources to run the full regression test non-stop. + +-S is used to continue running tests after an aborted run. It will +maintain the order a standard run (ie, this assumes -r is not used). +This is useful after the tests have prematurely stopped for some external +reason and you want to start running from where you left off rather +than starting from the beginning. + +-f reads the names of tests from the file given as f's argument, one +or more test names per line. Whitespace is ignored. Blank lines and +lines beginning with '#' are ignored. This is especially useful for +whittling down failures involving interactions among tests. + +-L causes the leaks(1) command to be run just before exit if it exists. +leaks(1) is available on Mac OS X and presumably on some other +FreeBSD-derived systems. + +-R runs each test several times and examines sys.gettotalrefcount() to +see if the test appears to be leaking references. The argument should +be of the form stab:run:fname where 'stab' is the number of times the +test is run to let gettotalrefcount settle down, 'run' is the number +of times further it is run and 'fname' is the name of the file the +reports are written to. These parameters all have defaults (5, 4 and +"reflog.txt" respectively), and the minimal invocation is '-R :'. + +-M runs tests that require an exorbitant amount of memory. These tests +typically try to ascertain containers keep working when containing more than +2 billion objects, which only works on 64-bit systems. There are also some +tests that try to exhaust the address space of the process, which only makes +sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, +which is a string in the form of '2.5Gb', determines howmuch memory the +tests will limit themselves to (but they may go slightly over.) The number +shouldn't be more memory than the machine has (including swap memory). You +should also keep in mind that swap memory is generally much, much slower +than RAM, and setting memlimit to all available RAM or higher will heavily +tax the machine. On the other hand, it is no use running these tests with a +limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect +to use more than memlimit memory will be skipped. The big-memory tests +generally run very, very long. + +-u is used to specify which special resource intensive tests to run, +such as those requiring large file support or network connectivity. +The argument is a comma-separated list of words indicating the +resources to test. Currently only the following are defined: + + all - Enable all special resources. + + none - Disable all special resources (this is the default). + + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) + + curses - Tests that use curses and will modify the terminal's + state and output modes. + + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2GB of disk space temporarily. + + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. + + decimal - Test the decimal module against a large suite that + verifies compliance with standards. + + cpu - Used for certain CPU-heavy tests. + + subprocess Run all tests for the subprocess module. + + urlfetch - It is okay to download files required on testing. + + gui - Run tests that require a running GUI. + +To enable all resources except one, use '-uall,-'. For +example, to run all the tests except for the gui tests, give the +option '-uall,-gui'. +""" + + +RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') + +class _ArgParser(argparse.ArgumentParser): + + def error(self, message): + super().error(message + "\nPass -h or --help for complete help.") + + +def _create_parser(): + # Set prog to prevent the uninformative "__main__.py" from displaying in + # error messages when using "python -m test ...". + parser = _ArgParser(prog='regrtest.py', + usage=USAGE, + description=DESCRIPTION, + epilog=EPILOG, + add_help=False, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # Arguments with this clause added to its help are described further in + # the epilog's "Additional option details" section. + more_details = ' See the section at bottom for more details.' + + group = parser.add_argument_group('General options') + # We add help explicitly to control what argument group it renders under. + group.add_argument('-h', '--help', action='help', + help='show this help message and exit') + group.add_argument('--timeout', metavar='TIMEOUT', type=float, + help='dump the traceback and exit if a test takes ' + 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'is negative or equals to zero') + group.add_argument('--wait', action='store_true', + help='wait for user input, e.g., allow a debugger ' + 'to be attached') + group.add_argument('--slaveargs', metavar='ARGS') + group.add_argument('-S', '--start', metavar='START', + help='the name of the test at which to start.' + + more_details) + + group = parser.add_argument_group('Verbosity') + group.add_argument('-v', '--verbose', action='count', + help='run tests in verbose mode with output to stdout') + group.add_argument('-w', '--verbose2', action='store_true', + help='re-run failed tests in verbose mode') + group.add_argument('-W', '--verbose3', action='store_true', + help='display test output on failure') + group.add_argument('-q', '--quiet', action='store_true', + help='no output unless one or more tests fail') + group.add_argument('-o', '--slow', action='store_true', dest='print_slow', + help='print the slowest 10 tests') + group.add_argument('--header', action='store_true', + help='print header with interpreter info') + + group = parser.add_argument_group('Selecting tests') + group.add_argument('-r', '--randomize', action='store_true', + help='randomize test execution order.' + more_details) + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a random seed to reproduce a previous ' + 'random run') + group.add_argument('-f', '--fromfile', metavar='FILE', + help='read names of tests to run from a file.' + + more_details) + group.add_argument('-x', '--exclude', action='store_true', + help='arguments are tests to *exclude*') + group.add_argument('-s', '--single', action='store_true', + help='single step through a set of tests.' + + more_details) + group.add_argument('-m', '--match', metavar='PAT', + dest='match_tests', + help='match test cases and methods with glob pattern PAT') + group.add_argument('-G', '--failfast', action='store_true', + help='fail as soon as a test fails (only with -v or -W)') + group.add_argument('-u', '--use', metavar='RES1,RES2,...', + action='append', type=resources_list, + help='specify which special resource intensive tests ' + 'to run.' + more_details) + group.add_argument('-M', '--memlimit', metavar='LIMIT', + help='run very large memory-consuming tests.' + + more_details) + group.add_argument('--testdir', metavar='DIR', + type=relative_filename, + help='execute test files in the specified directory ' + '(instead of the Python stdlib test suite)') + + group = parser.add_argument_group('Special runs') + group.add_argument('-l', '--findleaks', action='store_true', + help='if GC is available detect tests that leak memory') + group.add_argument('-L', '--runleaks', action='store_true', + help='run the leaks(1) command just before exit.' + + more_details) + group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', + type=huntrleaks, + help='search for reference leaks (needs debug build, ' + 'very slow).' + more_details) + group.add_argument('-j', '--multiprocess', metavar='PROCESSES', + dest='use_mp', type=int, + help='run PROCESSES processes at once') + group.add_argument('-T', '--coverage', action='store_true', + dest='trace', + help='turn on code coverage tracing using the trace ' + 'module') + group.add_argument('-D', '--coverdir', metavar='DIR', + type=relative_filename, + help='directory where coverage files are put') + group.add_argument('-N', '--nocoverdir', + action='store_const', const=None, dest='coverdir', + help='put coverage files alongside modules') + group.add_argument('-t', '--threshold', metavar='THRESHOLD', + type=int, + help='call gc.set_threshold(THRESHOLD)') + group.add_argument('-n', '--nowindows', action='store_true', + help='suppress error message boxes on Windows') + group.add_argument('-F', '--forever', action='store_true', + help='run the specified tests in a loop, until an ' + 'error happens') + + parser.add_argument('args', nargs=argparse.REMAINDER, + help=argparse.SUPPRESS) + + return parser + + +def relative_filename(string): + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + return os.path.join(support.SAVEDCWD, string) + + +def huntrleaks(string): + args = string.split(':') + if len(args) not in (2, 3): + raise argparse.ArgumentTypeError( + 'needs 2 or 3 colon-separated arguments') + nwarmup = int(args[0]) if args[0] else 5 + ntracked = int(args[1]) if args[1] else 4 + fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' + return nwarmup, ntracked, fname + + +def resources_list(string): + u = [x.lower() for x in string.split(',')] + for r in u: + if r == 'all' or r == 'none': + continue + if r[0] == '-': + r = r[1:] + if r not in RESOURCE_NAMES: + raise argparse.ArgumentTypeError('invalid resource: ' + r) + return u + + +def _parse_args(args, **kwargs): + # Defaults + ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, + exclude=False, single=False, randomize=False, fromfile=None, + findleaks=False, use_resources=None, trace=False, coverdir='coverage', + runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, + random_seed=None, use_mp=None, verbose3=False, forever=False, + header=False, failfast=False, match_tests=None) + for k, v in kwargs.items(): + if not hasattr(ns, k): + raise TypeError('%r is an invalid keyword argument ' + 'for this function' % k) + setattr(ns, k, v) + if ns.use_resources is None: + ns.use_resources = [] + + parser = _create_parser() + parser.parse_args(args=args, namespace=ns) + + if ns.single and ns.fromfile: + parser.error("-s and -f don't go together!") + if ns.use_mp and ns.trace: + parser.error("-T and -j don't go together!") + if ns.use_mp and ns.findleaks: + parser.error("-l and -j don't go together!") + if ns.use_mp and ns.memlimit: + parser.error("-M and -j don't go together!") + if ns.failfast and not (ns.verbose or ns.verbose3): + parser.error("-G/--failfast needs either -v or -W") + + if ns.quiet: + ns.verbose = 0 + if ns.timeout is not None: + if hasattr(faulthandler, 'dump_traceback_later'): + if ns.timeout <= 0: + ns.timeout = None + else: + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later") + ns.timeout = None + if ns.use_mp is not None: + if ns.use_mp <= 0: + # Use all cores + extras for tests that like to sleep + ns.use_mp = 2 + (os.cpu_count() or 1) + if ns.use_mp == 1: + ns.use_mp = None + if ns.use: + for a in ns.use: + for r in a: + if r == 'all': + ns.use_resources[:] = RESOURCE_NAMES + continue + if r == 'none': + del ns.use_resources[:] + continue + remove = False + if r[0] == '-': + remove = True + r = r[1:] + if remove: + if r in ns.use_resources: + ns.use_resources.remove(r) + elif r not in ns.use_resources: + ns.use_resources.append(r) + if ns.random_seed is not None: + ns.randomize = True + + return ns diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index b49e66be37..51e9e18029 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -6,119 +6,6 @@ Script to run Python regression tests. Run this script with -h or --help for documentation. """ -USAGE = """\ -python -m test [options] [test_name1 [test_name2 ...]] -python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] -""" - -DESCRIPTION = """\ -Run Python regression tests. - -If no arguments or options are provided, finds all files matching -the pattern "test_*" in the Lib/test subdirectory and runs -them in alphabetical order (but see -M and -u, below, for exceptions). - -For more rigorous testing, it is useful to use the following -command line: - -python -E -Wd -m test [options] [test_name1 ...] -""" - -EPILOG = """\ -Additional option details: - --r randomizes test execution order. You can use --randseed=int to provide a -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. - --s On the first invocation of regrtest using -s, the first test file found -or the first test file given on the command line is run, and the name of -the next test is recorded in a file named pynexttest. If run from the -Python build directory, pynexttest is located in the 'build' subdirectory, -otherwise it is located in tempfile.gettempdir(). On subsequent runs, -the test in pynexttest is run, and the next test is written to pynexttest. -When the last test has been run, pynexttest is deleted. In this way it -is possible to single step through the test files. This is useful when -doing memory analysis on the Python interpreter, which process tends to -consume too many resources to run the full regression test non-stop. - --S is used to continue running tests after an aborted run. It will -maintain the order a standard run (ie, this assumes -r is not used). -This is useful after the tests have prematurely stopped for some external -reason and you want to start running from where you left off rather -than starting from the beginning. - --f reads the names of tests from the file given as f's argument, one -or more test names per line. Whitespace is ignored. Blank lines and -lines beginning with '#' are ignored. This is especially useful for -whittling down failures involving interactions among tests. - --L causes the leaks(1) command to be run just before exit if it exists. -leaks(1) is available on Mac OS X and presumably on some other -FreeBSD-derived systems. - --R runs each test several times and examines sys.gettotalrefcount() to -see if the test appears to be leaking references. The argument should -be of the form stab:run:fname where 'stab' is the number of times the -test is run to let gettotalrefcount settle down, 'run' is the number -of times further it is run and 'fname' is the name of the file the -reports are written to. These parameters all have defaults (5, 4 and -"reflog.txt" respectively), and the minimal invocation is '-R :'. - --M runs tests that require an exorbitant amount of memory. These tests -typically try to ascertain containers keep working when containing more than -2 billion objects, which only works on 64-bit systems. There are also some -tests that try to exhaust the address space of the process, which only makes -sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, -which is a string in the form of '2.5Gb', determines howmuch memory the -tests will limit themselves to (but they may go slightly over.) The number -shouldn't be more memory than the machine has (including swap memory). You -should also keep in mind that swap memory is generally much, much slower -than RAM, and setting memlimit to all available RAM or higher will heavily -tax the machine. On the other hand, it is no use running these tests with a -limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect -to use more than memlimit memory will be skipped. The big-memory tests -generally run very, very long. - --u is used to specify which special resource intensive tests to run, -such as those requiring large file support or network connectivity. -The argument is a comma-separated list of words indicating the -resources to test. Currently only the following are defined: - - all - Enable all special resources. - - none - Disable all special resources (this is the default). - - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) - - curses - Tests that use curses and will modify the terminal's - state and output modes. - - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2GB of disk space temporarily. - - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. - - decimal - Test the decimal module against a large suite that - verifies compliance with standards. - - cpu - Used for certain CPU-heavy tests. - - subprocess Run all tests for the subprocess module. - - urlfetch - It is okay to download files required on testing. - - gui - Run tests that require a running GUI. - -To enable all resources except one, use '-uall,-'. For -example, to run all the tests except for the gui tests, give the -option '-uall,-gui'. -""" - # We import importlib *ASAP* in order to test #15386 import importlib @@ -153,6 +40,8 @@ try: except ImportError: multiprocessing = None +from test.libregrtest import _parse_args + # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some @@ -198,9 +87,6 @@ CHILD_ERROR = -5 # error in a child process from test import support -RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') - # When tests are run from the Python build directory, it is best practice # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. @@ -210,220 +96,6 @@ else: TEMPDIR = tempfile.gettempdir() TEMPDIR = os.path.abspath(TEMPDIR) -class _ArgParser(argparse.ArgumentParser): - - def error(self, message): - super().error(message + "\nPass -h or --help for complete help.") - -def _create_parser(): - # Set prog to prevent the uninformative "__main__.py" from displaying in - # error messages when using "python -m test ...". - parser = _ArgParser(prog='regrtest.py', - usage=USAGE, - description=DESCRIPTION, - epilog=EPILOG, - add_help=False, - formatter_class=argparse.RawDescriptionHelpFormatter) - - # Arguments with this clause added to its help are described further in - # the epilog's "Additional option details" section. - more_details = ' See the section at bottom for more details.' - - group = parser.add_argument_group('General options') - # We add help explicitly to control what argument group it renders under. - group.add_argument('-h', '--help', action='help', - help='show this help message and exit') - group.add_argument('--timeout', metavar='TIMEOUT', type=float, - help='dump the traceback and exit if a test takes ' - 'more than TIMEOUT seconds; disabled if TIMEOUT ' - 'is negative or equals to zero') - group.add_argument('--wait', action='store_true', - help='wait for user input, e.g., allow a debugger ' - 'to be attached') - group.add_argument('--slaveargs', metavar='ARGS') - group.add_argument('-S', '--start', metavar='START', - help='the name of the test at which to start.' + - more_details) - - group = parser.add_argument_group('Verbosity') - group.add_argument('-v', '--verbose', action='count', - help='run tests in verbose mode with output to stdout') - group.add_argument('-w', '--verbose2', action='store_true', - help='re-run failed tests in verbose mode') - group.add_argument('-W', '--verbose3', action='store_true', - help='display test output on failure') - group.add_argument('-q', '--quiet', action='store_true', - help='no output unless one or more tests fail') - group.add_argument('-o', '--slow', action='store_true', dest='print_slow', - help='print the slowest 10 tests') - group.add_argument('--header', action='store_true', - help='print header with interpreter info') - - group = parser.add_argument_group('Selecting tests') - group.add_argument('-r', '--randomize', action='store_true', - help='randomize test execution order.' + more_details) - group.add_argument('--randseed', metavar='SEED', - dest='random_seed', type=int, - help='pass a random seed to reproduce a previous ' - 'random run') - group.add_argument('-f', '--fromfile', metavar='FILE', - help='read names of tests to run from a file.' + - more_details) - group.add_argument('-x', '--exclude', action='store_true', - help='arguments are tests to *exclude*') - group.add_argument('-s', '--single', action='store_true', - help='single step through a set of tests.' + - more_details) - group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', - help='match test cases and methods with glob pattern PAT') - group.add_argument('-G', '--failfast', action='store_true', - help='fail as soon as a test fails (only with -v or -W)') - group.add_argument('-u', '--use', metavar='RES1,RES2,...', - action='append', type=resources_list, - help='specify which special resource intensive tests ' - 'to run.' + more_details) - group.add_argument('-M', '--memlimit', metavar='LIMIT', - help='run very large memory-consuming tests.' + - more_details) - group.add_argument('--testdir', metavar='DIR', - type=relative_filename, - help='execute test files in the specified directory ' - '(instead of the Python stdlib test suite)') - - group = parser.add_argument_group('Special runs') - group.add_argument('-l', '--findleaks', action='store_true', - help='if GC is available detect tests that leak memory') - group.add_argument('-L', '--runleaks', action='store_true', - help='run the leaks(1) command just before exit.' + - more_details) - group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', - type=huntrleaks, - help='search for reference leaks (needs debug build, ' - 'very slow).' + more_details) - group.add_argument('-j', '--multiprocess', metavar='PROCESSES', - dest='use_mp', type=int, - help='run PROCESSES processes at once') - group.add_argument('-T', '--coverage', action='store_true', - dest='trace', - help='turn on code coverage tracing using the trace ' - 'module') - group.add_argument('-D', '--coverdir', metavar='DIR', - type=relative_filename, - help='directory where coverage files are put') - group.add_argument('-N', '--nocoverdir', - action='store_const', const=None, dest='coverdir', - help='put coverage files alongside modules') - group.add_argument('-t', '--threshold', metavar='THRESHOLD', - type=int, - help='call gc.set_threshold(THRESHOLD)') - group.add_argument('-n', '--nowindows', action='store_true', - help='suppress error message boxes on Windows') - group.add_argument('-F', '--forever', action='store_true', - help='run the specified tests in a loop, until an ' - 'error happens') - - parser.add_argument('args', nargs=argparse.REMAINDER, - help=argparse.SUPPRESS) - - return parser - -def relative_filename(string): - # CWD is replaced with a temporary dir before calling main(), so we - # join it with the saved CWD so it ends up where the user expects. - return os.path.join(support.SAVEDCWD, string) - -def huntrleaks(string): - args = string.split(':') - if len(args) not in (2, 3): - raise argparse.ArgumentTypeError( - 'needs 2 or 3 colon-separated arguments') - nwarmup = int(args[0]) if args[0] else 5 - ntracked = int(args[1]) if args[1] else 4 - fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' - return nwarmup, ntracked, fname - -def resources_list(string): - u = [x.lower() for x in string.split(',')] - for r in u: - if r == 'all' or r == 'none': - continue - if r[0] == '-': - r = r[1:] - if r not in RESOURCE_NAMES: - raise argparse.ArgumentTypeError('invalid resource: ' + r) - return u - -def _parse_args(args, **kwargs): - # Defaults - ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, - exclude=False, single=False, randomize=False, fromfile=None, - findleaks=False, use_resources=None, trace=False, coverdir='coverage', - runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, - random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None) - for k, v in kwargs.items(): - if not hasattr(ns, k): - raise TypeError('%r is an invalid keyword argument ' - 'for this function' % k) - setattr(ns, k, v) - if ns.use_resources is None: - ns.use_resources = [] - - parser = _create_parser() - parser.parse_args(args=args, namespace=ns) - - if ns.single and ns.fromfile: - parser.error("-s and -f don't go together!") - if ns.use_mp and ns.trace: - parser.error("-T and -j don't go together!") - if ns.use_mp and ns.findleaks: - parser.error("-l and -j don't go together!") - if ns.use_mp and ns.memlimit: - parser.error("-M and -j don't go together!") - if ns.failfast and not (ns.verbose or ns.verbose3): - parser.error("-G/--failfast needs either -v or -W") - - if ns.quiet: - ns.verbose = 0 - if ns.timeout is not None: - if hasattr(faulthandler, 'dump_traceback_later'): - if ns.timeout <= 0: - ns.timeout = None - else: - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later") - ns.timeout = None - if ns.use_mp is not None: - if ns.use_mp <= 0: - # Use all cores + extras for tests that like to sleep - ns.use_mp = 2 + (os.cpu_count() or 1) - if ns.use_mp == 1: - ns.use_mp = None - if ns.use: - for a in ns.use: - for r in a: - if r == 'all': - ns.use_resources[:] = RESOURCE_NAMES - continue - if r == 'none': - del ns.use_resources[:] - continue - remove = False - if r[0] == '-': - remove = True - r = r[1:] - if remove: - if r in ns.use_resources: - ns.use_resources.remove(r) - elif r not in ns.use_resources: - ns.use_resources.append(r) - if ns.random_seed is not None: - ns.randomize = True - - return ns - - def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a398a4f836..cf4b84fe19 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,7 +7,7 @@ import faulthandler import getopt import os.path import unittest -from test import regrtest, support +from test import regrtest, support, libregrtest class ParseArgsTestCase(unittest.TestCase): @@ -148,7 +148,7 @@ class ParseArgsTestCase(unittest.TestCase): self.assertEqual(ns.use_resources, ['gui', 'network']) ns = regrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(regrtest.RESOURCE_NAMES) + expected = list(libregrtest.RESOURCE_NAMES) expected.remove('gui') ns = regrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) -- cgit v1.2.1 From ce0d11c6b13b46068c20f05ef99d1f0d3251ec5e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Sep 2015 10:38:01 +0200 Subject: Issue #25220: Move most regrtest.py code to libregrtest --- Lib/test/libregrtest/__init__.py | 1 + Lib/test/libregrtest/cmdline.py | 2 +- Lib/test/libregrtest/main.py | 547 +++++++++++++++++ Lib/test/libregrtest/refleak.py | 165 +++++ Lib/test/libregrtest/runtest.py | 271 +++++++++ Lib/test/libregrtest/save_env.py | 284 +++++++++ Lib/test/regrtest.py | 1225 +------------------------------------- Lib/test/test_regrtest.py | 95 +-- 8 files changed, 1318 insertions(+), 1272 deletions(-) create mode 100644 Lib/test/libregrtest/main.py create mode 100644 Lib/test/libregrtest/refleak.py create mode 100644 Lib/test/libregrtest/runtest.py create mode 100644 Lib/test/libregrtest/save_env.py (limited to 'Lib') diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py index 554aeede4a..a882ed2960 100644 --- a/Lib/test/libregrtest/__init__.py +++ b/Lib/test/libregrtest/__init__.py @@ -1 +1,2 @@ from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES +from test.libregrtest.main import main_in_temp_cwd diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 94649965a5..19c52a2fe6 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -1,9 +1,9 @@ import argparse import faulthandler import os - from test import support + USAGE = """\ python -m test [options] [test_name1 [test_name2 ...]] python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py new file mode 100644 index 0000000000..5d34cffd49 --- /dev/null +++ b/Lib/test/libregrtest/main.py @@ -0,0 +1,547 @@ +import faulthandler +import json +import os +import re +import sys +import tempfile +import sysconfig +import signal +import random +import platform +import traceback +import unittest +from test.libregrtest.runtest import ( + findtests, runtest, run_test_in_subprocess, + STDTESTS, NOTTESTS, + PASSED, FAILED, ENV_CHANGED, SKIPPED, + RESOURCE_DENIED, INTERRUPTED, CHILD_ERROR) +from test.libregrtest.refleak import warm_caches +from test.libregrtest.cmdline import _parse_args +from test import support +try: + import threading +except ImportError: + threading = None + + +# Some times __path__ and __file__ are not absolute (e.g. while running from +# Lib/) and, if we change the CWD to run the tests in a temporary dir, some +# imports might fail. This affects only the modules imported before os.chdir(). +# These modules are searched first in sys.path[0] (so '' -- the CWD) and if +# they are found in the CWD their __file__ and __path__ will be relative (this +# happens before the chdir). All the modules imported after the chdir, are +# not found in the CWD, and since the other paths in sys.path[1:] are absolute +# (site.py absolutize them), the __file__ and __path__ will be absolute too. +# Therefore it is necessary to absolutize manually the __file__ and __path__ of +# the packages to prevent later imports to fail when the CWD is different. +for module in sys.modules.values(): + if hasattr(module, '__path__'): + module.__path__ = [os.path.abspath(path) for path in module.__path__] + if hasattr(module, '__file__'): + module.__file__ = os.path.abspath(module.__file__) + + +# MacOSX (a.k.a. Darwin) has a default stack size that is too small +# for deeply recursive regular expressions. We see this as crashes in +# the Python test suite when running test_re.py and test_sre.py. The +# fix is to set the stack limit to 2048. +# This approach may also be useful for other Unixy platforms that +# suffer from small default stack limits. +if sys.platform == 'darwin': + try: + import resource + except ImportError: + pass + else: + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + newsoft = min(hard, max(soft, 1024*2048)) + resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + + +# When tests are run from the Python build directory, it is best practice +# to keep the test files in a subfolder. This eases the cleanup of leftover +# files using the "make distclean" command. +if sysconfig.is_python_build(): + TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') +else: + TEMPDIR = tempfile.gettempdir() +TEMPDIR = os.path.abspath(TEMPDIR) + + +def main(tests=None, **kwargs): + """Execute a test suite. + + This also parses command-line options and modifies its behavior + accordingly. + + tests -- a list of strings containing test names (optional) + testdir -- the directory in which to look for tests (optional) + + Users other than the Python test suite will certainly want to + specify testdir; if it's omitted, the directory containing the + Python test suite is searched for. + + If the tests argument is omitted, the tests listed on the + command-line will be used. If that's empty, too, then all *.py + files beginning with test_ will be used. + + The other default arguments (verbose, quiet, exclude, + single, randomize, findleaks, use_resources, trace, coverdir, + print_slow, and random_seed) allow programmers calling main() + directly to set the values that would normally be set by flags + on the command line. + """ + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + + replace_stdout() + + support.record_original_stdout(sys.stdout) + + ns = _parse_args(sys.argv[1:], **kwargs) + + if ns.huntrleaks: + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() + if ns.memlimit is not None: + support.set_memlimit(ns.memlimit) + if ns.threshold is not None: + import gc + gc.set_threshold(ns.threshold) + if ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + if ns.wait: + input("Press any key to continue...") + + if ns.slaveargs is not None: + args, kwargs = json.loads(ns.slaveargs) + if kwargs.get('huntrleaks'): + unittest.BaseTestSuite._cleanup = False + try: + result = runtest(*args, **kwargs) + except KeyboardInterrupt: + result = INTERRUPTED, '' + except BaseException as e: + traceback.print_exc() + result = CHILD_ERROR, str(e) + sys.stdout.flush() + print() # Force a newline (just in case) + print(json.dumps(result)) + sys.exit(0) + + good = [] + bad = [] + skipped = [] + resource_denieds = [] + environment_changed = [] + interrupted = False + + if ns.findleaks: + try: + import gc + except ImportError: + print('No GC available, disabling findleaks.') + ns.findleaks = False + else: + # Uncomment the line below to report garbage that is not + # freeable by reference counting alone. By default only + # garbage that is not collectable by the GC is reported. + #gc.set_debug(gc.DEBUG_SAVEALL) + found_garbage = [] + + if ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + if ns.single: + filename = os.path.join(TEMPDIR, 'pynexttest') + try: + with open(filename, 'r') as fp: + next_test = fp.read().strip() + tests = [next_test] + except OSError: + pass + + if ns.fromfile: + tests = [] + with open(os.path.join(support.SAVEDCWD, ns.fromfile)) as fp: + count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') + for line in fp: + line = count_pat.sub('', line) + guts = line.split() # assuming no test has whitespace in its name + if guts and not guts[0].startswith('#'): + tests.extend(guts) + + # Strip .py extensions. + removepy(ns.args) + removepy(tests) + + stdtests = STDTESTS[:] + nottests = NOTTESTS.copy() + if ns.exclude: + for arg in ns.args: + if arg in stdtests: + stdtests.remove(arg) + nottests.add(arg) + ns.args = [] + + # For a partial run, we do not need to clutter the output. + if ns.verbose or ns.header or not (ns.quiet or ns.single or tests or ns.args): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("== ", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== ", "hash algorithm:", sys.hash_info.algorithm, + "64bit" if sys.maxsize > 2**32 else "32bit") + print("== ", os.getcwd()) + print("Testing with flags:", sys.flags) + + # if testdir is set, then we are not running the python tests suite, so + # don't add default tests to be executed or skipped (pass empty values) + if ns.testdir: + alltests = findtests(ns.testdir, list(), set()) + else: + alltests = findtests(ns.testdir, stdtests, nottests) + + selected = tests or ns.args or alltests + if ns.single: + selected = selected[:1] + try: + next_single_test = alltests[alltests.index(selected[0])+1] + except IndexError: + next_single_test = None + # Remove all the selected tests that precede start if it's set. + if ns.start: + try: + del selected[:selected.index(ns.start)] + except ValueError: + print("Couldn't find starting test (%s), using all tests" % ns.start) + if ns.randomize: + if ns.random_seed is None: + ns.random_seed = random.randrange(10000000) + random.seed(ns.random_seed) + print("Using random seed", ns.random_seed) + random.shuffle(selected) + if ns.trace: + import trace, tempfile + tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, + tempfile.gettempdir()], + trace=False, count=True) + + test_times = [] + support.verbose = ns.verbose # Tell tests to be moderately quiet + support.use_resources = ns.use_resources + save_modules = sys.modules.keys() + + def accumulate_result(test, result): + ok, test_time = result + test_times.append((test_time, test)) + if ok == PASSED: + good.append(test) + elif ok == FAILED: + bad.append(test) + elif ok == ENV_CHANGED: + environment_changed.append(test) + elif ok == SKIPPED: + skipped.append(test) + elif ok == RESOURCE_DENIED: + skipped.append(test) + resource_denieds.append(test) + + if ns.forever: + def test_forever(tests=list(selected)): + while True: + for test in tests: + yield test + if bad: + return + tests = test_forever() + test_count = '' + test_count_width = 3 + else: + tests = iter(selected) + test_count = '/{}'.format(len(selected)) + test_count_width = len(test_count) - 1 + + if ns.use_mp: + try: + from threading import Thread + except ImportError: + print("Multiprocess option requires thread support") + sys.exit(2) + from queue import Queue + debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") + output = Queue() + pending = MultiprocessTests(tests) + def work(): + # A worker thread. + try: + while True: + try: + test = next(pending) + except StopIteration: + output.put((None, None, None, None)) + return + retcode, stdout, stderr = run_test_in_subprocess(test, ns) + # Strip last refcount output line if it exists, since it + # comes from the shutdown of the interpreter in the subcommand. + stderr = debug_output_pat.sub("", stderr) + stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + return + if not result: + output.put((None, None, None, None)) + return + result = json.loads(result) + output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + except BaseException: + output.put((None, None, None, None)) + raise + workers = [Thread(target=work) for i in range(ns.use_mp)] + for worker in workers: + worker.start() + finished = 0 + test_index = 1 + try: + while finished < ns.use_mp: + test, stdout, stderr, result = output.get() + if test is None: + finished += 1 + continue + accumulate_result(test, result) + if not ns.quiet: + fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" + print(fmt.format( + test_count_width, test_index, test_count, + len(bad), test)) + if stdout: + print(stdout) + if stderr: + print(stderr, file=sys.stderr) + sys.stdout.flush() + sys.stderr.flush() + if result[0] == INTERRUPTED: + raise KeyboardInterrupt + if result[0] == CHILD_ERROR: + raise Exception("Child error on {}: {}".format(test, result[1])) + test_index += 1 + except KeyboardInterrupt: + interrupted = True + pending.interrupted = True + for worker in workers: + worker.join() + else: + for test_index, test in enumerate(tests, 1): + if not ns.quiet: + fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" + print(fmt.format( + test_count_width, test_index, test_count, len(bad), test)) + sys.stdout.flush() + if ns.trace: + # If we're tracing code coverage, then we don't exit with status + # if on a false return value from main. + tracer.runctx('runtest(test, ns.verbose, ns.quiet, timeout=ns.timeout)', + globals=globals(), locals=vars()) + else: + try: + result = runtest(test, ns.verbose, ns.quiet, + ns.huntrleaks, + output_on_failure=ns.verbose3, + timeout=ns.timeout, failfast=ns.failfast, + match_tests=ns.match_tests) + accumulate_result(test, result) + except KeyboardInterrupt: + interrupted = True + break + if ns.findleaks: + gc.collect() + if gc.garbage: + print("Warning: test created", len(gc.garbage), end=' ') + print("uncollectable object(s).") + # move the uncollectable objects somewhere so we don't see + # them again + found_garbage.extend(gc.garbage) + del gc.garbage[:] + # Unload the newly imported modules (best effort finalization) + for module in sys.modules.keys(): + if module not in save_modules and module.startswith("test."): + support.unload(module) + + if interrupted: + # print a newline after ^C + print() + print("Test suite interrupted by signal SIGINT.") + omitted = set(selected) - set(good) - set(bad) - set(skipped) + print(count(len(omitted), "test"), "omitted:") + printlist(omitted) + if good and not ns.quiet: + if not bad and not skipped and not interrupted and len(good) > 1: + print("All", end=' ') + print(count(len(good), "test"), "OK.") + if ns.print_slow: + test_times.sort(reverse=True) + print("10 slowest tests:") + for time, test in test_times[:10]: + print("%s: %.1fs" % (test, time)) + if bad: + print(count(len(bad), "test"), "failed:") + printlist(bad) + if environment_changed: + print("{} altered the execution environment:".format( + count(len(environment_changed), "test"))) + printlist(environment_changed) + if skipped and not ns.quiet: + print(count(len(skipped), "test"), "skipped:") + printlist(skipped) + + if ns.verbose2 and bad: + print("Re-running failed tests in verbose mode") + for test in bad[:]: + print("Re-running test %r in verbose mode" % test) + sys.stdout.flush() + try: + ns.verbose = True + ok = runtest(test, True, ns.quiet, ns.huntrleaks, + timeout=ns.timeout) + except KeyboardInterrupt: + # print a newline separate from the ^C + print() + break + else: + if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: + bad.remove(test) + else: + if bad: + print(count(len(bad), 'test'), "failed again:") + printlist(bad) + + if ns.single: + if next_single_test: + with open(filename, 'w') as fp: + fp.write(next_single_test + '\n') + else: + os.unlink(filename) + + if ns.trace: + r = tracer.results() + r.write_results(show_missing=True, summary=True, coverdir=ns.coverdir) + + if ns.runleaks: + os.system("leaks %d" % os.getpid()) + + sys.exit(len(bad) > 0 or interrupted) + + +# We do not use a generator so multiple threads can call next(). +class MultiprocessTests(object): + + """A thread-safe iterator over tests for multiprocess mode.""" + + def __init__(self, tests): + self.interrupted = False + self.lock = threading.Lock() + self.tests = tests + + def __iter__(self): + return self + + def __next__(self): + with self.lock: + if self.interrupted: + raise StopIteration('tests interrupted') + return next(self.tests) + + +def replace_stdout(): + """Set stdout encoder error handler to backslashreplace (as stderr error + handler) to avoid UnicodeEncodeError when printing a traceback""" + import atexit + + stdout = sys.stdout + sys.stdout = open(stdout.fileno(), 'w', + encoding=stdout.encoding, + errors="backslashreplace", + closefd=False, + newline='\n') + + def restore_stdout(): + sys.stdout.close() + sys.stdout = stdout + atexit.register(restore_stdout) + + +def removepy(names): + if not names: + return + for idx, name in enumerate(names): + basename, ext = os.path.splitext(name) + if ext == '.py': + names[idx] = basename + + +def count(n, word): + if n == 1: + return "%d %s" % (n, word) + else: + return "%d %ss" % (n, word) + + +def printlist(x, width=70, indent=4): + """Print the elements of iterable x to stdout. + + Optional arg width (default 70) is the maximum line length. + Optional arg indent (default 4) is the number of blanks with which to + begin each line. + """ + + from textwrap import fill + blanks = ' ' * indent + # Print the sorted list: 'x' may be a '--random' list or a set() + print(fill(' '.join(str(elt) for elt in sorted(x)), width, + initial_indent=blanks, subsequent_indent=blanks)) + + +def main_in_temp_cwd(): + """Run main() in a temporary working directory.""" + if sysconfig.is_python_build(): + try: + os.mkdir(TEMPDIR) + except FileExistsError: + pass + + # Define a writable temp dir that will be used as cwd while running + # the tests. The name of the dir includes the pid to allow parallel + # testing (see the -j option). + test_cwd = 'test_python_{}'.format(os.getpid()) + test_cwd = os.path.join(TEMPDIR, test_cwd) + + # Run the tests in a context manager that temporarily changes the CWD to a + # temporary and writable directory. If it's not possible to create or + # change the CWD, the original CWD will be used. The original CWD is + # available from support.SAVEDCWD. + with support.temp_cwd(test_cwd, quiet=True): + main() diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py new file mode 100644 index 0000000000..dd0d05d3ec --- /dev/null +++ b/Lib/test/libregrtest/refleak.py @@ -0,0 +1,165 @@ +import os +import re +import sys +import warnings +from inspect import isabstract +from test import support + + +def dash_R(the_module, test, indirect_test, huntrleaks): + """Run a test multiple times, looking for reference leaks. + + Returns: + False if the test didn't leak references; True if we detected refleaks. + """ + # This code is hackish and inelegant, but it seems to do the job. + import copyreg + import collections.abc + + if not hasattr(sys, 'gettotalrefcount'): + raise Exception("Tracking reference leaks requires a debug build " + "of Python") + + # Save current values for dash_R_cleanup() to restore. + fs = warnings.filters[:] + ps = copyreg.dispatch_table.copy() + pic = sys.path_importer_cache.copy() + try: + import zipimport + except ImportError: + zdc = None # Run unmodified on platforms without zipimport support + else: + zdc = zipimport._zip_directory_cache.copy() + abcs = {} + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: + if not isabstract(abc): + continue + for obj in abc.__subclasses__() + [abc]: + abcs[obj] = obj._abc_registry.copy() + + nwarmup, ntracked, fname = huntrleaks + fname = os.path.join(support.SAVEDCWD, fname) + repcount = nwarmup + ntracked + rc_deltas = [0] * repcount + alloc_deltas = [0] * repcount + + print("beginning", repcount, "repetitions", file=sys.stderr) + print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr) + sys.stderr.flush() + for i in range(repcount): + indirect_test() + alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) + sys.stderr.write('.') + sys.stderr.flush() + if i >= nwarmup: + rc_deltas[i] = rc_after - rc_before + alloc_deltas[i] = alloc_after - alloc_before + alloc_before, rc_before = alloc_after, rc_after + print(file=sys.stderr) + # These checkers return False on success, True on failure + def check_rc_deltas(deltas): + return any(deltas) + def check_alloc_deltas(deltas): + # At least 1/3rd of 0s + if 3 * deltas.count(0) < len(deltas): + return True + # Nothing else than 1s, 0s and -1s + if not set(deltas) <= {1,0,-1}: + return True + return False + failed = False + for deltas, item_name, checker in [ + (rc_deltas, 'references', check_rc_deltas), + (alloc_deltas, 'memory blocks', check_alloc_deltas)]: + if checker(deltas): + msg = '%s leaked %s %s, sum=%s' % ( + test, deltas[nwarmup:], item_name, sum(deltas)) + print(msg, file=sys.stderr) + sys.stderr.flush() + with open(fname, "a") as refrep: + print(msg, file=refrep) + refrep.flush() + failed = True + return failed + + +def dash_R_cleanup(fs, ps, pic, zdc, abcs): + import gc, copyreg + import _strptime, linecache + import urllib.parse, urllib.request, mimetypes, doctest + import struct, filecmp, collections.abc + from distutils.dir_util import _path_created + from weakref import WeakSet + + # Clear the warnings registry, so they can be displayed again + for mod in sys.modules.values(): + if hasattr(mod, '__warningregistry__'): + del mod.__warningregistry__ + + # Restore some original values. + warnings.filters[:] = fs + copyreg.dispatch_table.clear() + copyreg.dispatch_table.update(ps) + sys.path_importer_cache.clear() + sys.path_importer_cache.update(pic) + try: + import zipimport + except ImportError: + pass # Run unmodified on platforms without zipimport support + else: + zipimport._zip_directory_cache.clear() + zipimport._zip_directory_cache.update(zdc) + + # clear type cache + sys._clear_type_cache() + + # Clear ABC registries, restoring previously saved ABC registries. + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: + if not isabstract(abc): + continue + for obj in abc.__subclasses__() + [abc]: + obj._abc_registry = abcs.get(obj, WeakSet()).copy() + obj._abc_cache.clear() + obj._abc_negative_cache.clear() + + # Flush standard output, so that buffered data is sent to the OS and + # associated Python objects are reclaimed. + for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): + if stream is not None: + stream.flush() + + # Clear assorted module caches. + _path_created.clear() + re.purge() + _strptime._regex_cache.clear() + urllib.parse.clear_cache() + urllib.request.urlcleanup() + linecache.clearcache() + mimetypes._default_mime_types() + filecmp._cache.clear() + struct._clearcache() + doctest.master = None + try: + import ctypes + except ImportError: + # Don't worry about resetting the cache if ctypes is not supported + pass + else: + ctypes._reset_cache() + + # Collect cyclic trash and read memory statistics immediately after. + func1 = sys.getallocatedblocks + func2 = sys.gettotalrefcount + gc.collect() + return func1(), func2() + + +def warm_caches(): + # char cache + s = bytes(range(256)) + for i in range(256): + s[i:i+1] + # unicode cache + x = [chr(i) for i in range(256)] + # int cache + x = list(range(-5, 257)) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py new file mode 100644 index 0000000000..d8c0eb2a86 --- /dev/null +++ b/Lib/test/libregrtest/runtest.py @@ -0,0 +1,271 @@ +import faulthandler +import importlib +import io +import json +import os +import sys +import time +import traceback +import unittest +from test import support +from test.libregrtest.refleak import dash_R +from test.libregrtest.save_env import saved_test_environment + + +# Test result constants. +PASSED = 1 +FAILED = 0 +ENV_CHANGED = -1 +SKIPPED = -2 +RESOURCE_DENIED = -3 +INTERRUPTED = -4 +CHILD_ERROR = -5 # error in a child process + + +def run_test_in_subprocess(testname, ns): + """Run the given test in a subprocess with --slaveargs. + + ns is the option Namespace parsed from command-line arguments. regrtest + is invoked in a subprocess with the --slaveargs argument; when the + subprocess exits, its return code, stdout and stderr are returned as a + 3-tuple. + """ + from subprocess import Popen, PIPE + base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + + ['-X', 'faulthandler', '-m', 'test.regrtest']) + + slaveargs = ( + (testname, ns.verbose, ns.quiet), + dict(huntrleaks=ns.huntrleaks, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + timeout=ns.timeout, failfast=ns.failfast, + match_tests=ns.match_tests)) + # Running the child from the same working directory as regrtest's original + # invocation ensures that TEMPDIR for the child is the same when + # sysconfig.is_python_build() is true. See issue 15300. + popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], + stdout=PIPE, stderr=PIPE, + universal_newlines=True, + close_fds=(os.name != 'nt'), + cwd=support.SAVEDCWD) + stdout, stderr = popen.communicate() + retcode = popen.wait() + return retcode, stdout, stderr + + +# small set of tests to determine if we have a basically functioning interpreter +# (i.e. if any of these fail, then anything else is likely to follow) +STDTESTS = [ + 'test_grammar', + 'test_opcodes', + 'test_dict', + 'test_builtin', + 'test_exceptions', + 'test_types', + 'test_unittest', + 'test_doctest', + 'test_doctest2', + 'test_support' +] + +# set of tests that we don't want to be executed when using regrtest +NOTTESTS = set() + + +def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): + """Return a list of all applicable test modules.""" + testdir = findtestdir(testdir) + names = os.listdir(testdir) + tests = [] + others = set(stdtests) | nottests + for name in names: + mod, ext = os.path.splitext(name) + if mod[:5] == "test_" and ext in (".py", "") and mod not in others: + tests.append(mod) + return stdtests + sorted(tests) + + +def runtest(test, verbose, quiet, + huntrleaks=False, use_resources=None, + output_on_failure=False, failfast=False, match_tests=None, + timeout=None): + """Run a single test. + + test -- the name of the test + verbose -- if true, print more messages + quiet -- if true, don't print 'skipped' messages (probably redundant) + huntrleaks -- run multiple times to test for leaks; requires a debug + build; a triple corresponding to -R's three arguments + use_resources -- list of extra resources to use + output_on_failure -- if true, display test output on failure + timeout -- dump the traceback and exit if a test takes more than + timeout seconds + failfast, match_tests -- See regrtest command-line flags for these. + + Returns the tuple result, test_time, where result is one of the constants: + INTERRUPTED KeyboardInterrupt when run under -j + RESOURCE_DENIED test skipped because resource denied + SKIPPED test skipped for some other reason + ENV_CHANGED test failed because it changed the execution environment + FAILED test failed + PASSED test passed + """ + + if use_resources is not None: + support.use_resources = use_resources + use_timeout = (timeout is not None) + if use_timeout: + faulthandler.dump_traceback_later(timeout, exit=True) + try: + support.match_tests = match_tests + if failfast: + support.failfast = True + if output_on_failure: + support.verbose = True + + # Reuse the same instance to all calls to runtest(). Some + # tests keep a reference to sys.stdout or sys.stderr + # (eg. test_argparse). + if runtest.stringio is None: + stream = io.StringIO() + runtest.stringio = stream + else: + stream = runtest.stringio + stream.seek(0) + stream.truncate() + + orig_stdout = sys.stdout + orig_stderr = sys.stderr + try: + sys.stdout = stream + sys.stderr = stream + result = runtest_inner(test, verbose, quiet, huntrleaks, + display_failure=False) + if result[0] == FAILED: + output = stream.getvalue() + orig_stderr.write(output) + orig_stderr.flush() + finally: + sys.stdout = orig_stdout + sys.stderr = orig_stderr + else: + support.verbose = verbose # Tell tests to be moderately quiet + result = runtest_inner(test, verbose, quiet, huntrleaks, + display_failure=not verbose) + return result + finally: + if use_timeout: + faulthandler.cancel_dump_traceback_later() + cleanup_test_droppings(test, verbose) +runtest.stringio = None + + +def runtest_inner(test, verbose, quiet, + huntrleaks=False, display_failure=True): + support.unload(test) + + test_time = 0.0 + refleak = False # True if the test leaked references. + try: + if test.startswith('test.'): + abstest = test + else: + # Always import it from the test package + abstest = 'test.' + test + with saved_test_environment(test, verbose, quiet) as environment: + start_time = time.time() + the_module = importlib.import_module(abstest) + # If the test has a test_main, that will run the appropriate + # tests. If not, use normal unittest test loading. + test_runner = getattr(the_module, "test_main", None) + if test_runner is None: + def test_runner(): + loader = unittest.TestLoader() + tests = loader.loadTestsFromModule(the_module) + for error in loader.errors: + print(error, file=sys.stderr) + if loader.errors: + raise Exception("errors while loading tests") + support.run_unittest(tests) + test_runner() + if huntrleaks: + refleak = dash_R(the_module, test, test_runner, huntrleaks) + test_time = time.time() - start_time + except support.ResourceDenied as msg: + if not quiet: + print(test, "skipped --", msg) + sys.stdout.flush() + return RESOURCE_DENIED, test_time + except unittest.SkipTest as msg: + if not quiet: + print(test, "skipped --", msg) + sys.stdout.flush() + return SKIPPED, test_time + except KeyboardInterrupt: + raise + except support.TestFailed as msg: + if display_failure: + print("test", test, "failed --", msg, file=sys.stderr) + else: + print("test", test, "failed", file=sys.stderr) + sys.stderr.flush() + return FAILED, test_time + except: + msg = traceback.format_exc() + print("test", test, "crashed --", msg, file=sys.stderr) + sys.stderr.flush() + return FAILED, test_time + else: + if refleak: + return FAILED, test_time + if environment.changed: + return ENV_CHANGED, test_time + return PASSED, test_time + + +def cleanup_test_droppings(testname, verbose): + import shutil + import stat + import gc + + # First kill any dangling references to open files etc. + # This can also issue some ResourceWarnings which would otherwise get + # triggered during the following test run, and possibly produce failures. + gc.collect() + + # Try to clean up junk commonly left behind. While tests shouldn't leave + # any files or directories behind, when a test fails that can be tedious + # for it to arrange. The consequences can be especially nasty on Windows, + # since if a test leaves a file open, it cannot be deleted by name (while + # there's nothing we can do about that here either, we can display the + # name of the offending test, which is a real help). + for name in (support.TESTFN, + "db_home", + ): + if not os.path.exists(name): + continue + + if os.path.isdir(name): + kind, nuker = "directory", shutil.rmtree + elif os.path.isfile(name): + kind, nuker = "file", os.unlink + else: + raise SystemError("os.path says %r exists but is neither " + "directory nor file" % name) + + if verbose: + print("%r left behind %s %r" % (testname, kind, name)) + try: + # if we have chmod, fix possible permissions problems + # that might prevent cleanup + if (hasattr(os, 'chmod')): + os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + nuker(name) + except Exception as msg: + print(("%r left behind %s %r and it couldn't be " + "removed: %s" % (testname, kind, name, msg)), file=sys.stderr) + + +def findtestdir(path=None): + return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py new file mode 100644 index 0000000000..4f6a1aa3b4 --- /dev/null +++ b/Lib/test/libregrtest/save_env.py @@ -0,0 +1,284 @@ +import builtins +import locale +import logging +import os +import shutil +import sys +import sysconfig +import warnings +from test import support +try: + import threading +except ImportError: + threading = None +try: + import _multiprocessing, multiprocessing.process +except ImportError: + multiprocessing = None + + +# Unit tests are supposed to leave the execution environment unchanged +# once they complete. But sometimes tests have bugs, especially when +# tests fail, and the changes to environment go on to mess up other +# tests. This can cause issues with buildbot stability, since tests +# are run in random order and so problems may appear to come and go. +# There are a few things we can save and restore to mitigate this, and +# the following context manager handles this task. + +class saved_test_environment: + """Save bits of the test environment and restore them at block exit. + + with saved_test_environment(testname, verbose, quiet): + #stuff + + Unless quiet is True, a warning is printed to stderr if any of + the saved items was changed by the test. The attribute 'changed' + is initially False, but is set to True if a change is detected. + + If verbose is more than 1, the before and after state of changed + items is also printed. + """ + + changed = False + + def __init__(self, testname, verbose=0, quiet=False): + self.testname = testname + self.verbose = verbose + self.quiet = quiet + + # To add things to save and restore, add a name XXX to the resources list + # and add corresponding get_XXX/restore_XXX functions. get_XXX should + # return the value to be saved and compared against a second call to the + # get function when test execution completes. restore_XXX should accept + # the saved value and restore the resource using it. It will be called if + # and only if a change in the value is detected. + # + # Note: XXX will have any '.' replaced with '_' characters when determining + # the corresponding method names. + + resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', + 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', + 'warnings.filters', 'asyncore.socket_map', + 'logging._handlers', 'logging._handlerList', 'sys.gettrace', + 'sys.warnoptions', + # multiprocessing.process._cleanup() may release ref + # to a thread, so check processes first. + 'multiprocessing.process._dangling', 'threading._dangling', + 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', + 'files', 'locale', 'warnings.showwarning', + ) + + def get_sys_argv(self): + return id(sys.argv), sys.argv, sys.argv[:] + def restore_sys_argv(self, saved_argv): + sys.argv = saved_argv[1] + sys.argv[:] = saved_argv[2] + + def get_cwd(self): + return os.getcwd() + def restore_cwd(self, saved_cwd): + os.chdir(saved_cwd) + + def get_sys_stdout(self): + return sys.stdout + def restore_sys_stdout(self, saved_stdout): + sys.stdout = saved_stdout + + def get_sys_stderr(self): + return sys.stderr + def restore_sys_stderr(self, saved_stderr): + sys.stderr = saved_stderr + + def get_sys_stdin(self): + return sys.stdin + def restore_sys_stdin(self, saved_stdin): + sys.stdin = saved_stdin + + def get_os_environ(self): + return id(os.environ), os.environ, dict(os.environ) + def restore_os_environ(self, saved_environ): + os.environ = saved_environ[1] + os.environ.clear() + os.environ.update(saved_environ[2]) + + def get_sys_path(self): + return id(sys.path), sys.path, sys.path[:] + def restore_sys_path(self, saved_path): + sys.path = saved_path[1] + sys.path[:] = saved_path[2] + + def get_sys_path_hooks(self): + return id(sys.path_hooks), sys.path_hooks, sys.path_hooks[:] + def restore_sys_path_hooks(self, saved_hooks): + sys.path_hooks = saved_hooks[1] + sys.path_hooks[:] = saved_hooks[2] + + def get_sys_gettrace(self): + return sys.gettrace() + def restore_sys_gettrace(self, trace_fxn): + sys.settrace(trace_fxn) + + def get___import__(self): + return builtins.__import__ + def restore___import__(self, import_): + builtins.__import__ = import_ + + def get_warnings_filters(self): + return id(warnings.filters), warnings.filters, warnings.filters[:] + def restore_warnings_filters(self, saved_filters): + warnings.filters = saved_filters[1] + warnings.filters[:] = saved_filters[2] + + def get_asyncore_socket_map(self): + asyncore = sys.modules.get('asyncore') + # XXX Making a copy keeps objects alive until __exit__ gets called. + return asyncore and asyncore.socket_map.copy() or {} + def restore_asyncore_socket_map(self, saved_map): + asyncore = sys.modules.get('asyncore') + if asyncore is not None: + asyncore.close_all(ignore_all=True) + asyncore.socket_map.update(saved_map) + + def get_shutil_archive_formats(self): + # we could call get_archives_formats() but that only returns the + # registry keys; we want to check the values too (the functions that + # are registered) + return shutil._ARCHIVE_FORMATS, shutil._ARCHIVE_FORMATS.copy() + def restore_shutil_archive_formats(self, saved): + shutil._ARCHIVE_FORMATS = saved[0] + shutil._ARCHIVE_FORMATS.clear() + shutil._ARCHIVE_FORMATS.update(saved[1]) + + def get_shutil_unpack_formats(self): + return shutil._UNPACK_FORMATS, shutil._UNPACK_FORMATS.copy() + def restore_shutil_unpack_formats(self, saved): + shutil._UNPACK_FORMATS = saved[0] + shutil._UNPACK_FORMATS.clear() + shutil._UNPACK_FORMATS.update(saved[1]) + + def get_logging__handlers(self): + # _handlers is a WeakValueDictionary + return id(logging._handlers), logging._handlers, logging._handlers.copy() + def restore_logging__handlers(self, saved_handlers): + # Can't easily revert the logging state + pass + + def get_logging__handlerList(self): + # _handlerList is a list of weakrefs to handlers + return id(logging._handlerList), logging._handlerList, logging._handlerList[:] + def restore_logging__handlerList(self, saved_handlerList): + # Can't easily revert the logging state + pass + + def get_sys_warnoptions(self): + return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] + def restore_sys_warnoptions(self, saved_options): + sys.warnoptions = saved_options[1] + sys.warnoptions[:] = saved_options[2] + + # Controlling dangling references to Thread objects can make it easier + # to track reference leaks. + def get_threading__dangling(self): + if not threading: + return None + # This copies the weakrefs without making any strong reference + return threading._dangling.copy() + def restore_threading__dangling(self, saved): + if not threading: + return + threading._dangling.clear() + threading._dangling.update(saved) + + # Same for Process objects + def get_multiprocessing_process__dangling(self): + if not multiprocessing: + return None + # Unjoined process objects can survive after process exits + multiprocessing.process._cleanup() + # This copies the weakrefs without making any strong reference + return multiprocessing.process._dangling.copy() + def restore_multiprocessing_process__dangling(self, saved): + if not multiprocessing: + return + multiprocessing.process._dangling.clear() + multiprocessing.process._dangling.update(saved) + + def get_sysconfig__CONFIG_VARS(self): + # make sure the dict is initialized + sysconfig.get_config_var('prefix') + return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, + dict(sysconfig._CONFIG_VARS)) + def restore_sysconfig__CONFIG_VARS(self, saved): + sysconfig._CONFIG_VARS = saved[1] + sysconfig._CONFIG_VARS.clear() + sysconfig._CONFIG_VARS.update(saved[2]) + + def get_sysconfig__INSTALL_SCHEMES(self): + return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, + sysconfig._INSTALL_SCHEMES.copy()) + def restore_sysconfig__INSTALL_SCHEMES(self, saved): + sysconfig._INSTALL_SCHEMES = saved[1] + sysconfig._INSTALL_SCHEMES.clear() + sysconfig._INSTALL_SCHEMES.update(saved[2]) + + def get_files(self): + return sorted(fn + ('/' if os.path.isdir(fn) else '') + for fn in os.listdir()) + def restore_files(self, saved_value): + fn = support.TESTFN + if fn not in saved_value and (fn + '/') not in saved_value: + if os.path.isfile(fn): + support.unlink(fn) + elif os.path.isdir(fn): + support.rmtree(fn) + + _lc = [getattr(locale, lc) for lc in dir(locale) + if lc.startswith('LC_')] + def get_locale(self): + pairings = [] + for lc in self._lc: + try: + pairings.append((lc, locale.setlocale(lc, None))) + except (TypeError, ValueError): + continue + return pairings + def restore_locale(self, saved): + for lc, setting in saved: + locale.setlocale(lc, setting) + + def get_warnings_showwarning(self): + return warnings.showwarning + def restore_warnings_showwarning(self, fxn): + warnings.showwarning = fxn + + def resource_info(self): + for name in self.resources: + method_suffix = name.replace('.', '_') + get_name = 'get_' + method_suffix + restore_name = 'restore_' + method_suffix + yield name, getattr(self, get_name), getattr(self, restore_name) + + def __enter__(self): + self.saved_values = dict((name, get()) for name, get, restore + in self.resource_info()) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + saved_values = self.saved_values + del self.saved_values + for name, get, restore in self.resource_info(): + current = get() + original = saved_values.pop(name) + # Check for changes to the resource's value + if current != original: + self.changed = True + restore(original) + if not self.quiet: + print("Warning -- {} was modified by {}".format( + name, self.testname), + file=sys.stderr) + if self.verbose > 1: + print(" Before: {}\n After: {} ".format( + original, current), + file=sys.stderr) + return False diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 51e9e18029..f01cad45a7 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -9,1232 +9,9 @@ Run this script with -h or --help for documentation. # We import importlib *ASAP* in order to test #15386 import importlib -import argparse -import builtins -import faulthandler -import io -import json -import locale -import logging import os -import platform -import random -import re -import shutil -import signal import sys -import sysconfig -import tempfile -import time -import traceback -import unittest -import warnings -from inspect import isabstract - -try: - import threading -except ImportError: - threading = None -try: - import _multiprocessing, multiprocessing.process -except ImportError: - multiprocessing = None - -from test.libregrtest import _parse_args - - -# Some times __path__ and __file__ are not absolute (e.g. while running from -# Lib/) and, if we change the CWD to run the tests in a temporary dir, some -# imports might fail. This affects only the modules imported before os.chdir(). -# These modules are searched first in sys.path[0] (so '' -- the CWD) and if -# they are found in the CWD their __file__ and __path__ will be relative (this -# happens before the chdir). All the modules imported after the chdir, are -# not found in the CWD, and since the other paths in sys.path[1:] are absolute -# (site.py absolutize them), the __file__ and __path__ will be absolute too. -# Therefore it is necessary to absolutize manually the __file__ and __path__ of -# the packages to prevent later imports to fail when the CWD is different. -for module in sys.modules.values(): - if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) for path in module.__path__] - if hasattr(module, '__file__'): - module.__file__ = os.path.abspath(module.__file__) - - -# MacOSX (a.k.a. Darwin) has a default stack size that is too small -# for deeply recursive regular expressions. We see this as crashes in -# the Python test suite when running test_re.py and test_sre.py. The -# fix is to set the stack limit to 2048. -# This approach may also be useful for other Unixy platforms that -# suffer from small default stack limits. -if sys.platform == 'darwin': - try: - import resource - except ImportError: - pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) - -# Test result constants. -PASSED = 1 -FAILED = 0 -ENV_CHANGED = -1 -SKIPPED = -2 -RESOURCE_DENIED = -3 -INTERRUPTED = -4 -CHILD_ERROR = -5 # error in a child process - -from test import support - -# When tests are run from the Python build directory, it is best practice -# to keep the test files in a subfolder. This eases the cleanup of leftover -# files using the "make distclean" command. -if sysconfig.is_python_build(): - TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') -else: - TEMPDIR = tempfile.gettempdir() -TEMPDIR = os.path.abspath(TEMPDIR) - -def run_test_in_subprocess(testname, ns): - """Run the given test in a subprocess with --slaveargs. - - ns is the option Namespace parsed from command-line arguments. regrtest - is invoked in a subprocess with the --slaveargs argument; when the - subprocess exits, its return code, stdout and stderr are returned as a - 3-tuple. - """ - from subprocess import Popen, PIPE - base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + - ['-X', 'faulthandler', '-m', 'test.regrtest']) - - slaveargs = ( - (testname, ns.verbose, ns.quiet), - dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests)) - # Running the child from the same working directory as regrtest's original - # invocation ensures that TEMPDIR for the child is the same when - # sysconfig.is_python_build() is true. See issue 15300. - popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], - stdout=PIPE, stderr=PIPE, - universal_newlines=True, - close_fds=(os.name != 'nt'), - cwd=support.SAVEDCWD) - stdout, stderr = popen.communicate() - retcode = popen.wait() - return retcode, stdout, stderr - - -def main(tests=None, **kwargs): - """Execute a test suite. - - This also parses command-line options and modifies its behavior - accordingly. - - tests -- a list of strings containing test names (optional) - testdir -- the directory in which to look for tests (optional) - - Users other than the Python test suite will certainly want to - specify testdir; if it's omitted, the directory containing the - Python test suite is searched for. - - If the tests argument is omitted, the tests listed on the - command-line will be used. If that's empty, too, then all *.py - files beginning with test_ will be used. - - The other default arguments (verbose, quiet, exclude, - single, randomize, findleaks, use_resources, trace, coverdir, - print_slow, and random_seed) allow programmers calling main() - directly to set the values that would normally be set by flags - on the command line. - """ - # Display the Python traceback on fatal errors (e.g. segfault) - faulthandler.enable(all_threads=True) - - # Display the Python traceback on SIGALRM or SIGUSR1 signal - signals = [] - if hasattr(signal, 'SIGALRM'): - signals.append(signal.SIGALRM) - if hasattr(signal, 'SIGUSR1'): - signals.append(signal.SIGUSR1) - for signum in signals: - faulthandler.register(signum, chain=True) - - replace_stdout() - - support.record_original_stdout(sys.stdout) - - ns = _parse_args(sys.argv[1:], **kwargs) - - if ns.huntrleaks: - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) - if ns.threshold is not None: - import gc - gc.set_threshold(ns.threshold) - if ns.nowindows: - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - if ns.wait: - input("Press any key to continue...") - - if ns.slaveargs is not None: - args, kwargs = json.loads(ns.slaveargs) - if kwargs.get('huntrleaks'): - unittest.BaseTestSuite._cleanup = False - try: - result = runtest(*args, **kwargs) - except KeyboardInterrupt: - result = INTERRUPTED, '' - except BaseException as e: - traceback.print_exc() - result = CHILD_ERROR, str(e) - sys.stdout.flush() - print() # Force a newline (just in case) - print(json.dumps(result)) - sys.exit(0) - - good = [] - bad = [] - skipped = [] - resource_denieds = [] - environment_changed = [] - interrupted = False - - if ns.findleaks: - try: - import gc - except ImportError: - print('No GC available, disabling findleaks.') - ns.findleaks = False - else: - # Uncomment the line below to report garbage that is not - # freeable by reference counting alone. By default only - # garbage that is not collectable by the GC is reported. - #gc.set_debug(gc.DEBUG_SAVEALL) - found_garbage = [] - - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False - - if ns.single: - filename = os.path.join(TEMPDIR, 'pynexttest') - try: - with open(filename, 'r') as fp: - next_test = fp.read().strip() - tests = [next_test] - except OSError: - pass - - if ns.fromfile: - tests = [] - with open(os.path.join(support.SAVEDCWD, ns.fromfile)) as fp: - count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') - for line in fp: - line = count_pat.sub('', line) - guts = line.split() # assuming no test has whitespace in its name - if guts and not guts[0].startswith('#'): - tests.extend(guts) - - # Strip .py extensions. - removepy(ns.args) - removepy(tests) - - stdtests = STDTESTS[:] - nottests = NOTTESTS.copy() - if ns.exclude: - for arg in ns.args: - if arg in stdtests: - stdtests.remove(arg) - nottests.add(arg) - ns.args = [] - - # For a partial run, we do not need to clutter the output. - if ns.verbose or ns.header or not (ns.quiet or ns.single or tests or ns.args): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("== ", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== ", "hash algorithm:", sys.hash_info.algorithm, - "64bit" if sys.maxsize > 2**32 else "32bit") - print("== ", os.getcwd()) - print("Testing with flags:", sys.flags) - - # if testdir is set, then we are not running the python tests suite, so - # don't add default tests to be executed or skipped (pass empty values) - if ns.testdir: - alltests = findtests(ns.testdir, list(), set()) - else: - alltests = findtests(ns.testdir, stdtests, nottests) - - selected = tests or ns.args or alltests - if ns.single: - selected = selected[:1] - try: - next_single_test = alltests[alltests.index(selected[0])+1] - except IndexError: - next_single_test = None - # Remove all the selected tests that precede start if it's set. - if ns.start: - try: - del selected[:selected.index(ns.start)] - except ValueError: - print("Couldn't find starting test (%s), using all tests" % ns.start) - if ns.randomize: - if ns.random_seed is None: - ns.random_seed = random.randrange(10000000) - random.seed(ns.random_seed) - print("Using random seed", ns.random_seed) - random.shuffle(selected) - if ns.trace: - import trace, tempfile - tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) - - test_times = [] - support.verbose = ns.verbose # Tell tests to be moderately quiet - support.use_resources = ns.use_resources - save_modules = sys.modules.keys() - - def accumulate_result(test, result): - ok, test_time = result - test_times.append((test_time, test)) - if ok == PASSED: - good.append(test) - elif ok == FAILED: - bad.append(test) - elif ok == ENV_CHANGED: - environment_changed.append(test) - elif ok == SKIPPED: - skipped.append(test) - elif ok == RESOURCE_DENIED: - skipped.append(test) - resource_denieds.append(test) - - if ns.forever: - def test_forever(tests=list(selected)): - while True: - for test in tests: - yield test - if bad: - return - tests = test_forever() - test_count = '' - test_count_width = 3 - else: - tests = iter(selected) - test_count = '/{}'.format(len(selected)) - test_count_width = len(test_count) - 1 - - if ns.use_mp: - try: - from threading import Thread - except ImportError: - print("Multiprocess option requires thread support") - sys.exit(2) - from queue import Queue - debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") - output = Queue() - pending = MultiprocessTests(tests) - def work(): - # A worker thread. - try: - while True: - try: - test = next(pending) - except StopIteration: - output.put((None, None, None, None)) - return - retcode, stdout, stderr = run_test_in_subprocess(test, ns) - # Strip last refcount output line if it exists, since it - # comes from the shutdown of the interpreter in the subcommand. - stderr = debug_output_pat.sub("", stderr) - stdout, _, result = stdout.strip().rpartition("\n") - if retcode != 0: - result = (CHILD_ERROR, "Exit code %s" % retcode) - output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - return - if not result: - output.put((None, None, None, None)) - return - result = json.loads(result) - output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - except BaseException: - output.put((None, None, None, None)) - raise - workers = [Thread(target=work) for i in range(ns.use_mp)] - for worker in workers: - worker.start() - finished = 0 - test_index = 1 - try: - while finished < ns.use_mp: - test, stdout, stderr, result = output.get() - if test is None: - finished += 1 - continue - accumulate_result(test, result) - if not ns.quiet: - fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - test_count_width, test_index, test_count, - len(bad), test)) - if stdout: - print(stdout) - if stderr: - print(stderr, file=sys.stderr) - sys.stdout.flush() - sys.stderr.flush() - if result[0] == INTERRUPTED: - raise KeyboardInterrupt - if result[0] == CHILD_ERROR: - raise Exception("Child error on {}: {}".format(test, result[1])) - test_index += 1 - except KeyboardInterrupt: - interrupted = True - pending.interrupted = True - for worker in workers: - worker.join() - else: - for test_index, test in enumerate(tests, 1): - if not ns.quiet: - fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - test_count_width, test_index, test_count, len(bad), test)) - sys.stdout.flush() - if ns.trace: - # If we're tracing code coverage, then we don't exit with status - # if on a false return value from main. - tracer.runctx('runtest(test, ns.verbose, ns.quiet, timeout=ns.timeout)', - globals=globals(), locals=vars()) - else: - try: - result = runtest(test, ns.verbose, ns.quiet, - ns.huntrleaks, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests) - accumulate_result(test, result) - except KeyboardInterrupt: - interrupted = True - break - if ns.findleaks: - gc.collect() - if gc.garbage: - print("Warning: test created", len(gc.garbage), end=' ') - print("uncollectable object(s).") - # move the uncollectable objects somewhere so we don't see - # them again - found_garbage.extend(gc.garbage) - del gc.garbage[:] - # Unload the newly imported modules (best effort finalization) - for module in sys.modules.keys(): - if module not in save_modules and module.startswith("test."): - support.unload(module) - - if interrupted: - # print a newline after ^C - print() - print("Test suite interrupted by signal SIGINT.") - omitted = set(selected) - set(good) - set(bad) - set(skipped) - print(count(len(omitted), "test"), "omitted:") - printlist(omitted) - if good and not ns.quiet: - if not bad and not skipped and not interrupted and len(good) > 1: - print("All", end=' ') - print(count(len(good), "test"), "OK.") - if ns.print_slow: - test_times.sort(reverse=True) - print("10 slowest tests:") - for time, test in test_times[:10]: - print("%s: %.1fs" % (test, time)) - if bad: - print(count(len(bad), "test"), "failed:") - printlist(bad) - if environment_changed: - print("{} altered the execution environment:".format( - count(len(environment_changed), "test"))) - printlist(environment_changed) - if skipped and not ns.quiet: - print(count(len(skipped), "test"), "skipped:") - printlist(skipped) - - if ns.verbose2 and bad: - print("Re-running failed tests in verbose mode") - for test in bad[:]: - print("Re-running test %r in verbose mode" % test) - sys.stdout.flush() - try: - ns.verbose = True - ok = runtest(test, True, ns.quiet, ns.huntrleaks, - timeout=ns.timeout) - except KeyboardInterrupt: - # print a newline separate from the ^C - print() - break - else: - if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: - bad.remove(test) - else: - if bad: - print(count(len(bad), 'test'), "failed again:") - printlist(bad) - - if ns.single: - if next_single_test: - with open(filename, 'w') as fp: - fp.write(next_single_test + '\n') - else: - os.unlink(filename) - - if ns.trace: - r = tracer.results() - r.write_results(show_missing=True, summary=True, coverdir=ns.coverdir) - - if ns.runleaks: - os.system("leaks %d" % os.getpid()) - - sys.exit(len(bad) > 0 or interrupted) - - -# small set of tests to determine if we have a basically functioning interpreter -# (i.e. if any of these fail, then anything else is likely to follow) -STDTESTS = [ - 'test_grammar', - 'test_opcodes', - 'test_dict', - 'test_builtin', - 'test_exceptions', - 'test_types', - 'test_unittest', - 'test_doctest', - 'test_doctest2', - 'test_support' -] - -# set of tests that we don't want to be executed when using regrtest -NOTTESTS = set() - -def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): - """Return a list of all applicable test modules.""" - testdir = findtestdir(testdir) - names = os.listdir(testdir) - tests = [] - others = set(stdtests) | nottests - for name in names: - mod, ext = os.path.splitext(name) - if mod[:5] == "test_" and ext in (".py", "") and mod not in others: - tests.append(mod) - return stdtests + sorted(tests) - -# We do not use a generator so multiple threads can call next(). -class MultiprocessTests(object): - - """A thread-safe iterator over tests for multiprocess mode.""" - - def __init__(self, tests): - self.interrupted = False - self.lock = threading.Lock() - self.tests = tests - - def __iter__(self): - return self - - def __next__(self): - with self.lock: - if self.interrupted: - raise StopIteration('tests interrupted') - return next(self.tests) - -def replace_stdout(): - """Set stdout encoder error handler to backslashreplace (as stderr error - handler) to avoid UnicodeEncodeError when printing a traceback""" - import atexit - - stdout = sys.stdout - sys.stdout = open(stdout.fileno(), 'w', - encoding=stdout.encoding, - errors="backslashreplace", - closefd=False, - newline='\n') - - def restore_stdout(): - sys.stdout.close() - sys.stdout = stdout - atexit.register(restore_stdout) - -def runtest(test, verbose, quiet, - huntrleaks=False, use_resources=None, - output_on_failure=False, failfast=False, match_tests=None, - timeout=None): - """Run a single test. - - test -- the name of the test - verbose -- if true, print more messages - quiet -- if true, don't print 'skipped' messages (probably redundant) - huntrleaks -- run multiple times to test for leaks; requires a debug - build; a triple corresponding to -R's three arguments - use_resources -- list of extra resources to use - output_on_failure -- if true, display test output on failure - timeout -- dump the traceback and exit if a test takes more than - timeout seconds - failfast, match_tests -- See regrtest command-line flags for these. - - Returns the tuple result, test_time, where result is one of the constants: - INTERRUPTED KeyboardInterrupt when run under -j - RESOURCE_DENIED test skipped because resource denied - SKIPPED test skipped for some other reason - ENV_CHANGED test failed because it changed the execution environment - FAILED test failed - PASSED test passed - """ - - if use_resources is not None: - support.use_resources = use_resources - use_timeout = (timeout is not None) - if use_timeout: - faulthandler.dump_traceback_later(timeout, exit=True) - try: - support.match_tests = match_tests - if failfast: - support.failfast = True - if output_on_failure: - support.verbose = True - - # Reuse the same instance to all calls to runtest(). Some - # tests keep a reference to sys.stdout or sys.stderr - # (eg. test_argparse). - if runtest.stringio is None: - stream = io.StringIO() - runtest.stringio = stream - else: - stream = runtest.stringio - stream.seek(0) - stream.truncate() - - orig_stdout = sys.stdout - orig_stderr = sys.stderr - try: - sys.stdout = stream - sys.stderr = stream - result = runtest_inner(test, verbose, quiet, huntrleaks, - display_failure=False) - if result[0] == FAILED: - output = stream.getvalue() - orig_stderr.write(output) - orig_stderr.flush() - finally: - sys.stdout = orig_stdout - sys.stderr = orig_stderr - else: - support.verbose = verbose # Tell tests to be moderately quiet - result = runtest_inner(test, verbose, quiet, huntrleaks, - display_failure=not verbose) - return result - finally: - if use_timeout: - faulthandler.cancel_dump_traceback_later() - cleanup_test_droppings(test, verbose) -runtest.stringio = None - -# Unit tests are supposed to leave the execution environment unchanged -# once they complete. But sometimes tests have bugs, especially when -# tests fail, and the changes to environment go on to mess up other -# tests. This can cause issues with buildbot stability, since tests -# are run in random order and so problems may appear to come and go. -# There are a few things we can save and restore to mitigate this, and -# the following context manager handles this task. - -class saved_test_environment: - """Save bits of the test environment and restore them at block exit. - - with saved_test_environment(testname, verbose, quiet): - #stuff - - Unless quiet is True, a warning is printed to stderr if any of - the saved items was changed by the test. The attribute 'changed' - is initially False, but is set to True if a change is detected. - - If verbose is more than 1, the before and after state of changed - items is also printed. - """ - - changed = False - - def __init__(self, testname, verbose=0, quiet=False): - self.testname = testname - self.verbose = verbose - self.quiet = quiet - - # To add things to save and restore, add a name XXX to the resources list - # and add corresponding get_XXX/restore_XXX functions. get_XXX should - # return the value to be saved and compared against a second call to the - # get function when test execution completes. restore_XXX should accept - # the saved value and restore the resource using it. It will be called if - # and only if a change in the value is detected. - # - # Note: XXX will have any '.' replaced with '_' characters when determining - # the corresponding method names. - - resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', - 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', - 'warnings.filters', 'asyncore.socket_map', - 'logging._handlers', 'logging._handlerList', 'sys.gettrace', - 'sys.warnoptions', - # multiprocessing.process._cleanup() may release ref - # to a thread, so check processes first. - 'multiprocessing.process._dangling', 'threading._dangling', - 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', - 'files', 'locale', 'warnings.showwarning', - ) - - def get_sys_argv(self): - return id(sys.argv), sys.argv, sys.argv[:] - def restore_sys_argv(self, saved_argv): - sys.argv = saved_argv[1] - sys.argv[:] = saved_argv[2] - - def get_cwd(self): - return os.getcwd() - def restore_cwd(self, saved_cwd): - os.chdir(saved_cwd) - - def get_sys_stdout(self): - return sys.stdout - def restore_sys_stdout(self, saved_stdout): - sys.stdout = saved_stdout - - def get_sys_stderr(self): - return sys.stderr - def restore_sys_stderr(self, saved_stderr): - sys.stderr = saved_stderr - - def get_sys_stdin(self): - return sys.stdin - def restore_sys_stdin(self, saved_stdin): - sys.stdin = saved_stdin - - def get_os_environ(self): - return id(os.environ), os.environ, dict(os.environ) - def restore_os_environ(self, saved_environ): - os.environ = saved_environ[1] - os.environ.clear() - os.environ.update(saved_environ[2]) - - def get_sys_path(self): - return id(sys.path), sys.path, sys.path[:] - def restore_sys_path(self, saved_path): - sys.path = saved_path[1] - sys.path[:] = saved_path[2] - - def get_sys_path_hooks(self): - return id(sys.path_hooks), sys.path_hooks, sys.path_hooks[:] - def restore_sys_path_hooks(self, saved_hooks): - sys.path_hooks = saved_hooks[1] - sys.path_hooks[:] = saved_hooks[2] - - def get_sys_gettrace(self): - return sys.gettrace() - def restore_sys_gettrace(self, trace_fxn): - sys.settrace(trace_fxn) - - def get___import__(self): - return builtins.__import__ - def restore___import__(self, import_): - builtins.__import__ = import_ - - def get_warnings_filters(self): - return id(warnings.filters), warnings.filters, warnings.filters[:] - def restore_warnings_filters(self, saved_filters): - warnings.filters = saved_filters[1] - warnings.filters[:] = saved_filters[2] - - def get_asyncore_socket_map(self): - asyncore = sys.modules.get('asyncore') - # XXX Making a copy keeps objects alive until __exit__ gets called. - return asyncore and asyncore.socket_map.copy() or {} - def restore_asyncore_socket_map(self, saved_map): - asyncore = sys.modules.get('asyncore') - if asyncore is not None: - asyncore.close_all(ignore_all=True) - asyncore.socket_map.update(saved_map) - - def get_shutil_archive_formats(self): - # we could call get_archives_formats() but that only returns the - # registry keys; we want to check the values too (the functions that - # are registered) - return shutil._ARCHIVE_FORMATS, shutil._ARCHIVE_FORMATS.copy() - def restore_shutil_archive_formats(self, saved): - shutil._ARCHIVE_FORMATS = saved[0] - shutil._ARCHIVE_FORMATS.clear() - shutil._ARCHIVE_FORMATS.update(saved[1]) - - def get_shutil_unpack_formats(self): - return shutil._UNPACK_FORMATS, shutil._UNPACK_FORMATS.copy() - def restore_shutil_unpack_formats(self, saved): - shutil._UNPACK_FORMATS = saved[0] - shutil._UNPACK_FORMATS.clear() - shutil._UNPACK_FORMATS.update(saved[1]) - - def get_logging__handlers(self): - # _handlers is a WeakValueDictionary - return id(logging._handlers), logging._handlers, logging._handlers.copy() - def restore_logging__handlers(self, saved_handlers): - # Can't easily revert the logging state - pass - - def get_logging__handlerList(self): - # _handlerList is a list of weakrefs to handlers - return id(logging._handlerList), logging._handlerList, logging._handlerList[:] - def restore_logging__handlerList(self, saved_handlerList): - # Can't easily revert the logging state - pass - - def get_sys_warnoptions(self): - return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] - def restore_sys_warnoptions(self, saved_options): - sys.warnoptions = saved_options[1] - sys.warnoptions[:] = saved_options[2] - - # Controlling dangling references to Thread objects can make it easier - # to track reference leaks. - def get_threading__dangling(self): - if not threading: - return None - # This copies the weakrefs without making any strong reference - return threading._dangling.copy() - def restore_threading__dangling(self, saved): - if not threading: - return - threading._dangling.clear() - threading._dangling.update(saved) - - # Same for Process objects - def get_multiprocessing_process__dangling(self): - if not multiprocessing: - return None - # Unjoined process objects can survive after process exits - multiprocessing.process._cleanup() - # This copies the weakrefs without making any strong reference - return multiprocessing.process._dangling.copy() - def restore_multiprocessing_process__dangling(self, saved): - if not multiprocessing: - return - multiprocessing.process._dangling.clear() - multiprocessing.process._dangling.update(saved) - - def get_sysconfig__CONFIG_VARS(self): - # make sure the dict is initialized - sysconfig.get_config_var('prefix') - return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, - dict(sysconfig._CONFIG_VARS)) - def restore_sysconfig__CONFIG_VARS(self, saved): - sysconfig._CONFIG_VARS = saved[1] - sysconfig._CONFIG_VARS.clear() - sysconfig._CONFIG_VARS.update(saved[2]) - - def get_sysconfig__INSTALL_SCHEMES(self): - return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, - sysconfig._INSTALL_SCHEMES.copy()) - def restore_sysconfig__INSTALL_SCHEMES(self, saved): - sysconfig._INSTALL_SCHEMES = saved[1] - sysconfig._INSTALL_SCHEMES.clear() - sysconfig._INSTALL_SCHEMES.update(saved[2]) - - def get_files(self): - return sorted(fn + ('/' if os.path.isdir(fn) else '') - for fn in os.listdir()) - def restore_files(self, saved_value): - fn = support.TESTFN - if fn not in saved_value and (fn + '/') not in saved_value: - if os.path.isfile(fn): - support.unlink(fn) - elif os.path.isdir(fn): - support.rmtree(fn) - - _lc = [getattr(locale, lc) for lc in dir(locale) - if lc.startswith('LC_')] - def get_locale(self): - pairings = [] - for lc in self._lc: - try: - pairings.append((lc, locale.setlocale(lc, None))) - except (TypeError, ValueError): - continue - return pairings - def restore_locale(self, saved): - for lc, setting in saved: - locale.setlocale(lc, setting) - - def get_warnings_showwarning(self): - return warnings.showwarning - def restore_warnings_showwarning(self, fxn): - warnings.showwarning = fxn - - def resource_info(self): - for name in self.resources: - method_suffix = name.replace('.', '_') - get_name = 'get_' + method_suffix - restore_name = 'restore_' + method_suffix - yield name, getattr(self, get_name), getattr(self, restore_name) - - def __enter__(self): - self.saved_values = dict((name, get()) for name, get, restore - in self.resource_info()) - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - saved_values = self.saved_values - del self.saved_values - for name, get, restore in self.resource_info(): - current = get() - original = saved_values.pop(name) - # Check for changes to the resource's value - if current != original: - self.changed = True - restore(original) - if not self.quiet: - print("Warning -- {} was modified by {}".format( - name, self.testname), - file=sys.stderr) - if self.verbose > 1: - print(" Before: {}\n After: {} ".format( - original, current), - file=sys.stderr) - return False - - -def runtest_inner(test, verbose, quiet, - huntrleaks=False, display_failure=True): - support.unload(test) - - test_time = 0.0 - refleak = False # True if the test leaked references. - try: - if test.startswith('test.'): - abstest = test - else: - # Always import it from the test package - abstest = 'test.' + test - with saved_test_environment(test, verbose, quiet) as environment: - start_time = time.time() - the_module = importlib.import_module(abstest) - # If the test has a test_main, that will run the appropriate - # tests. If not, use normal unittest test loading. - test_runner = getattr(the_module, "test_main", None) - if test_runner is None: - def test_runner(): - loader = unittest.TestLoader() - tests = loader.loadTestsFromModule(the_module) - for error in loader.errors: - print(error, file=sys.stderr) - if loader.errors: - raise Exception("errors while loading tests") - support.run_unittest(tests) - test_runner() - if huntrleaks: - refleak = dash_R(the_module, test, test_runner, huntrleaks) - test_time = time.time() - start_time - except support.ResourceDenied as msg: - if not quiet: - print(test, "skipped --", msg) - sys.stdout.flush() - return RESOURCE_DENIED, test_time - except unittest.SkipTest as msg: - if not quiet: - print(test, "skipped --", msg) - sys.stdout.flush() - return SKIPPED, test_time - except KeyboardInterrupt: - raise - except support.TestFailed as msg: - if display_failure: - print("test", test, "failed --", msg, file=sys.stderr) - else: - print("test", test, "failed", file=sys.stderr) - sys.stderr.flush() - return FAILED, test_time - except: - msg = traceback.format_exc() - print("test", test, "crashed --", msg, file=sys.stderr) - sys.stderr.flush() - return FAILED, test_time - else: - if refleak: - return FAILED, test_time - if environment.changed: - return ENV_CHANGED, test_time - return PASSED, test_time - -def cleanup_test_droppings(testname, verbose): - import shutil - import stat - import gc - - # First kill any dangling references to open files etc. - # This can also issue some ResourceWarnings which would otherwise get - # triggered during the following test run, and possibly produce failures. - gc.collect() - - # Try to clean up junk commonly left behind. While tests shouldn't leave - # any files or directories behind, when a test fails that can be tedious - # for it to arrange. The consequences can be especially nasty on Windows, - # since if a test leaves a file open, it cannot be deleted by name (while - # there's nothing we can do about that here either, we can display the - # name of the offending test, which is a real help). - for name in (support.TESTFN, - "db_home", - ): - if not os.path.exists(name): - continue - - if os.path.isdir(name): - kind, nuker = "directory", shutil.rmtree - elif os.path.isfile(name): - kind, nuker = "file", os.unlink - else: - raise SystemError("os.path says %r exists but is neither " - "directory nor file" % name) - - if verbose: - print("%r left behind %s %r" % (testname, kind, name)) - try: - # if we have chmod, fix possible permissions problems - # that might prevent cleanup - if (hasattr(os, 'chmod')): - os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - nuker(name) - except Exception as msg: - print(("%r left behind %s %r and it couldn't be " - "removed: %s" % (testname, kind, name, msg)), file=sys.stderr) - -def dash_R(the_module, test, indirect_test, huntrleaks): - """Run a test multiple times, looking for reference leaks. - - Returns: - False if the test didn't leak references; True if we detected refleaks. - """ - # This code is hackish and inelegant, but it seems to do the job. - import copyreg - import collections.abc - - if not hasattr(sys, 'gettotalrefcount'): - raise Exception("Tracking reference leaks requires a debug build " - "of Python") - - # Save current values for dash_R_cleanup() to restore. - fs = warnings.filters[:] - ps = copyreg.dispatch_table.copy() - pic = sys.path_importer_cache.copy() - try: - import zipimport - except ImportError: - zdc = None # Run unmodified on platforms without zipimport support - else: - zdc = zipimport._zip_directory_cache.copy() - abcs = {} - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue - for obj in abc.__subclasses__() + [abc]: - abcs[obj] = obj._abc_registry.copy() - - nwarmup, ntracked, fname = huntrleaks - fname = os.path.join(support.SAVEDCWD, fname) - repcount = nwarmup + ntracked - rc_deltas = [0] * repcount - alloc_deltas = [0] * repcount - - print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr) - sys.stderr.flush() - for i in range(repcount): - indirect_test() - alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) - sys.stderr.write('.') - sys.stderr.flush() - if i >= nwarmup: - rc_deltas[i] = rc_after - rc_before - alloc_deltas[i] = alloc_after - alloc_before - alloc_before, rc_before = alloc_after, rc_after - print(file=sys.stderr) - # These checkers return False on success, True on failure - def check_rc_deltas(deltas): - return any(deltas) - def check_alloc_deltas(deltas): - # At least 1/3rd of 0s - if 3 * deltas.count(0) < len(deltas): - return True - # Nothing else than 1s, 0s and -1s - if not set(deltas) <= {1,0,-1}: - return True - return False - failed = False - for deltas, item_name, checker in [ - (rc_deltas, 'references', check_rc_deltas), - (alloc_deltas, 'memory blocks', check_alloc_deltas)]: - if checker(deltas): - msg = '%s leaked %s %s, sum=%s' % ( - test, deltas[nwarmup:], item_name, sum(deltas)) - print(msg, file=sys.stderr) - sys.stderr.flush() - with open(fname, "a") as refrep: - print(msg, file=refrep) - refrep.flush() - failed = True - return failed - -def dash_R_cleanup(fs, ps, pic, zdc, abcs): - import gc, copyreg - import _strptime, linecache - import urllib.parse, urllib.request, mimetypes, doctest - import struct, filecmp, collections.abc - from distutils.dir_util import _path_created - from weakref import WeakSet - - # Clear the warnings registry, so they can be displayed again - for mod in sys.modules.values(): - if hasattr(mod, '__warningregistry__'): - del mod.__warningregistry__ - - # Restore some original values. - warnings.filters[:] = fs - copyreg.dispatch_table.clear() - copyreg.dispatch_table.update(ps) - sys.path_importer_cache.clear() - sys.path_importer_cache.update(pic) - try: - import zipimport - except ImportError: - pass # Run unmodified on platforms without zipimport support - else: - zipimport._zip_directory_cache.clear() - zipimport._zip_directory_cache.update(zdc) - - # clear type cache - sys._clear_type_cache() - - # Clear ABC registries, restoring previously saved ABC registries. - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue - for obj in abc.__subclasses__() + [abc]: - obj._abc_registry = abcs.get(obj, WeakSet()).copy() - obj._abc_cache.clear() - obj._abc_negative_cache.clear() - - # Flush standard output, so that buffered data is sent to the OS and - # associated Python objects are reclaimed. - for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): - if stream is not None: - stream.flush() - - # Clear assorted module caches. - _path_created.clear() - re.purge() - _strptime._regex_cache.clear() - urllib.parse.clear_cache() - urllib.request.urlcleanup() - linecache.clearcache() - mimetypes._default_mime_types() - filecmp._cache.clear() - struct._clearcache() - doctest.master = None - try: - import ctypes - except ImportError: - # Don't worry about resetting the cache if ctypes is not supported - pass - else: - ctypes._reset_cache() - - # Collect cyclic trash and read memory statistics immediately after. - func1 = sys.getallocatedblocks - func2 = sys.gettotalrefcount - gc.collect() - return func1(), func2() - -def warm_caches(): - # char cache - s = bytes(range(256)) - for i in range(256): - s[i:i+1] - # unicode cache - x = [chr(i) for i in range(256)] - # int cache - x = list(range(-5, 257)) - -def findtestdir(path=None): - return path or os.path.dirname(__file__) or os.curdir - -def removepy(names): - if not names: - return - for idx, name in enumerate(names): - basename, ext = os.path.splitext(name) - if ext == '.py': - names[idx] = basename - -def count(n, word): - if n == 1: - return "%d %s" % (n, word) - else: - return "%d %ss" % (n, word) - -def printlist(x, width=70, indent=4): - """Print the elements of iterable x to stdout. - - Optional arg width (default 70) is the maximum line length. - Optional arg indent (default 4) is the number of blanks with which to - begin each line. - """ - - from textwrap import fill - blanks = ' ' * indent - # Print the sorted list: 'x' may be a '--random' list or a set() - print(fill(' '.join(str(elt) for elt in sorted(x)), width, - initial_indent=blanks, subsequent_indent=blanks)) - - -def main_in_temp_cwd(): - """Run main() in a temporary working directory.""" - if sysconfig.is_python_build(): - try: - os.mkdir(TEMPDIR) - except FileExistsError: - pass - - # Define a writable temp dir that will be used as cwd while running - # the tests. The name of the dir includes the pid to allow parallel - # testing (see the -j option). - test_cwd = 'test_python_{}'.format(os.getpid()) - test_cwd = os.path.join(TEMPDIR, test_cwd) - - # Run the tests in a context manager that temporarily changes the CWD to a - # temporary and writable directory. If it's not possible to create or - # change the CWD, the original CWD will be used. The original CWD is - # available from support.SAVEDCWD. - with support.temp_cwd(test_cwd, quiet=True): - main() +from test.libregrtest import main_in_temp_cwd if __name__ == '__main__': diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index cf4b84fe19..f74412720d 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,7 +7,8 @@ import faulthandler import getopt import os.path import unittest -from test import regrtest, support, libregrtest +from test import libregrtest +from test import support class ParseArgsTestCase(unittest.TestCase): @@ -15,7 +16,7 @@ class ParseArgsTestCase(unittest.TestCase): def checkError(self, args, msg): with support.captured_stderr() as err, self.assertRaises(SystemExit): - regrtest._parse_args(args) + libregrtest._parse_args(args) self.assertIn(msg, err.getvalue()) def test_help(self): @@ -23,82 +24,82 @@ class ParseArgsTestCase(unittest.TestCase): with self.subTest(opt=opt): with support.captured_stdout() as out, \ self.assertRaises(SystemExit): - regrtest._parse_args([opt]) + libregrtest._parse_args([opt]) self.assertIn('Run Python regression tests.', out.getvalue()) @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'), "faulthandler.dump_traceback_later() required") def test_timeout(self): - ns = regrtest._parse_args(['--timeout', '4.2']) + ns = libregrtest._parse_args(['--timeout', '4.2']) self.assertEqual(ns.timeout, 4.2) self.checkError(['--timeout'], 'expected one argument') self.checkError(['--timeout', 'foo'], 'invalid float value') def test_wait(self): - ns = regrtest._parse_args(['--wait']) + ns = libregrtest._parse_args(['--wait']) self.assertTrue(ns.wait) def test_slaveargs(self): - ns = regrtest._parse_args(['--slaveargs', '[[], {}]']) + ns = libregrtest._parse_args(['--slaveargs', '[[], {}]']) self.assertEqual(ns.slaveargs, '[[], {}]') self.checkError(['--slaveargs'], 'expected one argument') def test_start(self): for opt in '-S', '--start': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'foo']) + ns = libregrtest._parse_args([opt, 'foo']) self.assertEqual(ns.start, 'foo') self.checkError([opt], 'expected one argument') def test_verbose(self): - ns = regrtest._parse_args(['-v']) + ns = libregrtest._parse_args(['-v']) self.assertEqual(ns.verbose, 1) - ns = regrtest._parse_args(['-vvv']) + ns = libregrtest._parse_args(['-vvv']) self.assertEqual(ns.verbose, 3) - ns = regrtest._parse_args(['--verbose']) + ns = libregrtest._parse_args(['--verbose']) self.assertEqual(ns.verbose, 1) - ns = regrtest._parse_args(['--verbose'] * 3) + ns = libregrtest._parse_args(['--verbose'] * 3) self.assertEqual(ns.verbose, 3) - ns = regrtest._parse_args([]) + ns = libregrtest._parse_args([]) self.assertEqual(ns.verbose, 0) def test_verbose2(self): for opt in '-w', '--verbose2': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.verbose2) def test_verbose3(self): for opt in '-W', '--verbose3': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.verbose3) def test_quiet(self): for opt in '-q', '--quiet': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) def test_slow(self): for opt in '-o', '--slow': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.print_slow) def test_header(self): - ns = regrtest._parse_args(['--header']) + ns = libregrtest._parse_args(['--header']) self.assertTrue(ns.header) def test_randomize(self): for opt in '-r', '--randomize': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.randomize) def test_randseed(self): - ns = regrtest._parse_args(['--randseed', '12345']) + ns = libregrtest._parse_args(['--randseed', '12345']) self.assertEqual(ns.random_seed, 12345) self.assertTrue(ns.randomize) self.checkError(['--randseed'], 'expected one argument') @@ -107,7 +108,7 @@ class ParseArgsTestCase(unittest.TestCase): def test_fromfile(self): for opt in '-f', '--fromfile': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'foo']) + ns = libregrtest._parse_args([opt, 'foo']) self.assertEqual(ns.fromfile, 'foo') self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo', '-s'], "don't go together") @@ -115,42 +116,42 @@ class ParseArgsTestCase(unittest.TestCase): def test_exclude(self): for opt in '-x', '--exclude': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.exclude) def test_single(self): for opt in '-s', '--single': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.single) self.checkError([opt, '-f', 'foo'], "don't go together") def test_match(self): for opt in '-m', '--match': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'pattern']) + ns = libregrtest._parse_args([opt, 'pattern']) self.assertEqual(ns.match_tests, 'pattern') self.checkError([opt], 'expected one argument') def test_failfast(self): for opt in '-G', '--failfast': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '-v']) + ns = libregrtest._parse_args([opt, '-v']) self.assertTrue(ns.failfast) - ns = regrtest._parse_args([opt, '-W']) + ns = libregrtest._parse_args([opt, '-W']) self.assertTrue(ns.failfast) self.checkError([opt], '-G/--failfast needs either -v or -W') def test_use(self): for opt in '-u', '--use': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'gui,network']) + ns = libregrtest._parse_args([opt, 'gui,network']) self.assertEqual(ns.use_resources, ['gui', 'network']) - ns = regrtest._parse_args([opt, 'gui,none,network']) + ns = libregrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) expected = list(libregrtest.RESOURCE_NAMES) expected.remove('gui') - ns = regrtest._parse_args([opt, 'all,-gui']) + ns = libregrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid resource') @@ -158,31 +159,31 @@ class ParseArgsTestCase(unittest.TestCase): def test_memlimit(self): for opt in '-M', '--memlimit': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '4G']) + ns = libregrtest._parse_args([opt, '4G']) self.assertEqual(ns.memlimit, '4G') self.checkError([opt], 'expected one argument') def test_testdir(self): - ns = regrtest._parse_args(['--testdir', 'foo']) + ns = libregrtest._parse_args(['--testdir', 'foo']) self.assertEqual(ns.testdir, os.path.join(support.SAVEDCWD, 'foo')) self.checkError(['--testdir'], 'expected one argument') def test_runleaks(self): for opt in '-L', '--runleaks': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.runleaks) def test_huntrleaks(self): for opt in '-R', '--huntrleaks': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, ':']) + ns = libregrtest._parse_args([opt, ':']) self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt')) - ns = regrtest._parse_args([opt, '6:']) + ns = libregrtest._parse_args([opt, '6:']) self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt')) - ns = regrtest._parse_args([opt, ':3']) + ns = libregrtest._parse_args([opt, ':3']) self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt')) - ns = regrtest._parse_args([opt, '6:3:leaks.log']) + ns = libregrtest._parse_args([opt, '6:3:leaks.log']) self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log')) self.checkError([opt], 'expected one argument') self.checkError([opt, '6'], @@ -193,7 +194,7 @@ class ParseArgsTestCase(unittest.TestCase): def test_multiprocess(self): for opt in '-j', '--multiprocess': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '2']) + ns = libregrtest._parse_args([opt, '2']) self.assertEqual(ns.use_mp, 2) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') @@ -204,13 +205,13 @@ class ParseArgsTestCase(unittest.TestCase): def test_coverage(self): for opt in '-T', '--coverage': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.trace) def test_coverdir(self): for opt in '-D', '--coverdir': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'foo']) + ns = libregrtest._parse_args([opt, 'foo']) self.assertEqual(ns.coverdir, os.path.join(support.SAVEDCWD, 'foo')) self.checkError([opt], 'expected one argument') @@ -218,13 +219,13 @@ class ParseArgsTestCase(unittest.TestCase): def test_nocoverdir(self): for opt in '-N', '--nocoverdir': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertIsNone(ns.coverdir) def test_threshold(self): for opt in '-t', '--threshold': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '1000']) + ns = libregrtest._parse_args([opt, '1000']) self.assertEqual(ns.threshold, 1000) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') @@ -232,13 +233,13 @@ class ParseArgsTestCase(unittest.TestCase): def test_nowindows(self): for opt in '-n', '--nowindows': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.nowindows) def test_forever(self): for opt in '-F', '--forever': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.forever) @@ -246,26 +247,26 @@ class ParseArgsTestCase(unittest.TestCase): self.checkError(['--xxx'], 'usage:') def test_long_option__partial(self): - ns = regrtest._parse_args(['--qui']) + ns = libregrtest._parse_args(['--qui']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) def test_two_options(self): - ns = regrtest._parse_args(['--quiet', '--exclude']) + ns = libregrtest._parse_args(['--quiet', '--exclude']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) self.assertTrue(ns.exclude) def test_option_with_empty_string_value(self): - ns = regrtest._parse_args(['--start', '']) + ns = libregrtest._parse_args(['--start', '']) self.assertEqual(ns.start, '') def test_arg(self): - ns = regrtest._parse_args(['foo']) + ns = libregrtest._parse_args(['foo']) self.assertEqual(ns.args, ['foo']) def test_option_and_arg(self): - ns = regrtest._parse_args(['--quiet', 'foo']) + ns = libregrtest._parse_args(['--quiet', 'foo']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) self.assertEqual(ns.args, ['foo']) -- cgit v1.2.1 From 4e3fe34a5bb103137f46d2c9cd0d4357c9e8c6d8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 27 Sep 2015 11:19:08 +0200 Subject: Issue #25220: Fix Lib/test/autotest.py --- Lib/test/libregrtest/__init__.py | 2 +- Lib/test/regrtest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py index a882ed2960..9f7b1c1fe2 100644 --- a/Lib/test/libregrtest/__init__.py +++ b/Lib/test/libregrtest/__init__.py @@ -1,2 +1,2 @@ from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES -from test.libregrtest.main import main_in_temp_cwd +from test.libregrtest.main import main, main_in_temp_cwd diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index f01cad45a7..fcc39375c0 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -11,7 +11,7 @@ import importlib import os import sys -from test.libregrtest import main_in_temp_cwd +from test.libregrtest import main, main_in_temp_cwd if __name__ == '__main__': -- cgit v1.2.1 From dd01cc739e98b6ca372059f716f52581231ae86f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 27 Sep 2015 13:26:03 +0300 Subject: Issue #25209: rlcomplete now can add a space or a colon after completed keyword. --- Lib/rlcompleter.py | 6 ++++++ Lib/test/test_rlcompleter.py | 13 +++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) (limited to 'Lib') diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index d517c0e2d3..568cf3680f 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -106,6 +106,12 @@ class Completer: n = len(text) for word in keyword.kwlist: if word[:n] == text: + if word in {'finally', 'try'}: + word = word + ':' + elif word not in {'False', 'None', 'True', + 'break', 'continue', 'pass', + 'else'}: + word = word + ' ' matches.append(word) for nspace in [builtins.__dict__, self.namespace]: for word, val in nspace.items(): diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index d37b620b7a..11a83059e0 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -67,10 +67,15 @@ class TestRlcompleter(unittest.TestCase): def test_complete(self): completer = rlcompleter.Completer() self.assertEqual(completer.complete('', 0), '\t') - self.assertEqual(completer.complete('a', 0), 'and') - self.assertEqual(completer.complete('a', 1), 'as') - self.assertEqual(completer.complete('as', 2), 'assert') - self.assertEqual(completer.complete('an', 0), 'and') + self.assertEqual(completer.complete('a', 0), 'and ') + self.assertEqual(completer.complete('a', 1), 'as ') + self.assertEqual(completer.complete('as', 2), 'assert ') + self.assertEqual(completer.complete('an', 0), 'and ') + self.assertEqual(completer.complete('pa', 0), 'pass') + self.assertEqual(completer.complete('Fa', 0), 'False') + self.assertEqual(completer.complete('el', 0), 'elif ') + self.assertEqual(completer.complete('el', 1), 'else') + self.assertEqual(completer.complete('tr', 0), 'try:') if __name__ == '__main__': unittest.main() -- cgit v1.2.1 From 8a919e96cdacdceab230066ddef61a3fcc8f5942 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 27 Sep 2015 13:43:50 +0300 Subject: Issue #25011: rlcomplete now omits private and special attribute names unless the prefix starts with underscores. --- Lib/rlcompleter.py | 35 +++++++++++++++++++++++++---------- Lib/test/test_rlcompleter.py | 15 +++++++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-) (limited to 'Lib') diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index 568cf3680f..613848f7ad 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -142,20 +142,35 @@ class Completer: return [] # get the content of the object, except __builtins__ - words = dir(thisobject) - if "__builtins__" in words: - words.remove("__builtins__") + words = set(dir(thisobject)) + words.discard("__builtins__") if hasattr(thisobject, '__class__'): - words.append('__class__') - words.extend(get_class_members(thisobject.__class__)) + words.add('__class__') + words.update(get_class_members(thisobject.__class__)) matches = [] n = len(attr) - for word in words: - if word[:n] == attr and hasattr(thisobject, word): - val = getattr(thisobject, word) - word = self._callable_postfix(val, "%s.%s" % (expr, word)) - matches.append(word) + if attr == '': + noprefix = '_' + elif attr == '_': + noprefix = '__' + else: + noprefix = None + while True: + for word in words: + if (word[:n] == attr and + not (noprefix and word[:n+1] == noprefix) and + hasattr(thisobject, word)): + val = getattr(thisobject, word) + word = self._callable_postfix(val, "%s.%s" % (expr, word)) + matches.append(word) + if matches or not noprefix: + break + if noprefix == '_': + noprefix = '__' + else: + noprefix = None + matches.sort() return matches def get_class_members(klass): diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 11a83059e0..2ff0788429 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -5,6 +5,7 @@ import rlcompleter class CompleteMe: """ Trivial class used in testing rlcompleter.Completer. """ spam = 1 + _ham = 2 class TestRlcompleter(unittest.TestCase): @@ -51,11 +52,25 @@ class TestRlcompleter(unittest.TestCase): ['str.{}('.format(x) for x in dir(str) if x.startswith('s')]) self.assertEqual(self.stdcompleter.attr_matches('tuple.foospamegg'), []) + expected = sorted({'None.%s%s' % (x, '(' if x != '__doc__' else '') + for x in dir(None)}) + self.assertEqual(self.stdcompleter.attr_matches('None.'), expected) + self.assertEqual(self.stdcompleter.attr_matches('None._'), expected) + self.assertEqual(self.stdcompleter.attr_matches('None.__'), expected) # test with a customized namespace self.assertEqual(self.completer.attr_matches('CompleteMe.sp'), ['CompleteMe.spam']) self.assertEqual(self.completer.attr_matches('Completeme.egg'), []) + self.assertEqual(self.completer.attr_matches('CompleteMe.'), + ['CompleteMe.mro(', 'CompleteMe.spam']) + self.assertEqual(self.completer.attr_matches('CompleteMe._'), + ['CompleteMe._ham']) + matches = self.completer.attr_matches('CompleteMe.__') + for x in matches: + self.assertTrue(x.startswith('CompleteMe.__'), x) + self.assertIn('CompleteMe.__name__', matches) + self.assertIn('CompleteMe.__new__(', matches) CompleteMe.me = CompleteMe self.assertEqual(self.completer.attr_matches('CompleteMe.me.me.sp'), -- cgit v1.2.1 From 6e0539f60ec8ad34be1c947f747b0ab91227e8ed Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 28 Sep 2015 15:04:11 +0200 Subject: Issue #25122: Remove verbose mode of test_eintr "./python -m test -W test_eintr" wrote Lib/test/eintrdata/eintr_tester.py output to stdout which was not expected. Since test_eintr doesn't hang anymore, remove the verbose mode instead. --- Lib/test/test_eintr.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index aabad835a0..d3cdda04e6 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -16,14 +16,7 @@ class EINTRTests(unittest.TestCase): # Run the tester in a sub-process, to make sure there is only one # thread (for reliable signal delivery). tester = support.findfile("eintr_tester.py", subdir="eintrdata") - - if support.verbose: - args = [sys.executable, tester] - with subprocess.Popen(args) as proc: - exitcode = proc.wait() - self.assertEqual(exitcode, 0) - else: - script_helper.assert_python_ok(tester) + script_helper.assert_python_ok(tester) if __name__ == "__main__": -- cgit v1.2.1 From 49bfa78a12bf20d365dc966219617e02605b50ea Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 28 Sep 2015 23:16:17 +0200 Subject: Issue #25220: Add functional tests to test_regrtest * test all available ways to run the Python test suite * test many regrtest options: --slow, --coverage, -r, -u, etc. Note: python -m test --coverage doesn't work on Windows. --- Lib/test/test_regrtest.py | 293 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 291 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index f74412720d..c0f1e7b2af 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1,18 +1,33 @@ """ Tests of regrtest.py. + +Note: test_regrtest cannot be run twice in parallel. """ import argparse import faulthandler import getopt import os.path +import platform +import re +import subprocess +import sys +import textwrap import unittest from test import libregrtest from test import support +from test.support import script_helper -class ParseArgsTestCase(unittest.TestCase): - """Test regrtest's argument parsing.""" +Py_DEBUG = hasattr(sys, 'getobjects') +ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') +ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) + + +class ParseArgsTestCase(unittest.TestCase): + """ + Test regrtest's argument parsing, function _parse_args(). + """ def checkError(self, args, msg): with support.captured_stderr() as err, self.assertRaises(SystemExit): @@ -272,5 +287,279 @@ class ParseArgsTestCase(unittest.TestCase): self.assertEqual(ns.args, ['foo']) +class BaseTestCase(unittest.TestCase): + TEST_UNIQUE_ID = 1 + TESTNAME_PREFIX = 'test_regrtest_' + TESTNAME_REGEX = r'test_[a-z0-9_]+' + + def setUp(self): + self.testdir = os.path.join(ROOT_DIR, 'Lib', 'test') + + # When test_regrtest is interrupted by CTRL+c, it can leave + # temporary test files + remove = [entry.path + for entry in os.scandir(self.testdir) + if (entry.name.startswith(self.TESTNAME_PREFIX) + and entry.name.endswith(".py"))] + for path in remove: + print("WARNING: test_regrtest: remove %s" % path) + support.unlink(path) + + def create_test(self, name=None, code=''): + if not name: + name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID + BaseTestCase.TEST_UNIQUE_ID += 1 + + # test_regrtest cannot be run twice in parallel because + # of setUp() and create_test() + name = self.TESTNAME_PREFIX + "%s_%s" % (os.getpid(), name) + path = os.path.join(self.testdir, name + '.py') + + self.addCleanup(support.unlink, path) + # Use 'x' mode to ensure that we do not override existing tests + with open(path, 'x', encoding='utf-8') as fp: + fp.write(code) + return name + + def regex_search(self, regex, output): + match = re.search(regex, output, re.MULTILINE) + if not match: + self.fail("%r not found in %r" % (regex, output)) + return match + + def check_line(self, output, regex): + regex = re.compile(r'^' + regex, re.MULTILINE) + self.assertRegex(output, regex) + + def parse_executed_tests(self, output): + parser = re.finditer(r'^\[[0-9]+/[0-9]+\] (%s)$' % self.TESTNAME_REGEX, + output, + re.MULTILINE) + return set(match.group(1) for match in parser) + + def check_executed_tests(self, output, tests, skipped=None): + if isinstance(tests, str): + tests = [tests] + executed = self.parse_executed_tests(output) + self.assertEqual(executed, set(tests), output) + ntest = len(tests) + if skipped: + if isinstance(skipped, str): + skipped = [skipped] + nskipped = len(skipped) + + plural = 's' if nskipped != 1 else '' + names = ' '.join(sorted(skipped)) + expected = (r'%s test%s skipped:\n %s$' + % (nskipped, plural, names)) + self.check_line(output, expected) + + ok = ntest - nskipped + if ok: + self.check_line(output, r'%s test OK\.$' % ok) + else: + self.check_line(output, r'All %s tests OK\.$' % ntest) + + def parse_random_seed(self, output): + match = self.regex_search(r'Using random seed ([0-9]+)', output) + randseed = int(match.group(1)) + self.assertTrue(0 <= randseed <= 10000000, randseed) + return randseed + + +class ProgramsTestCase(BaseTestCase): + """ + Test various ways to run the Python test suite. Use options close + to options used on the buildbot. + """ + + NTEST = 4 + + def setUp(self): + super().setUp() + + # Create NTEST tests doing nothing + self.tests = [self.create_test() for index in range(self.NTEST)] + + self.python_args = ['-Wd', '-E', '-bb'] + self.regrtest_args = ['-uall', '-rwW', '--timeout', '3600', '-j4'] + if sys.platform == 'win32': + self.regrtest_args.append('-n') + + def check_output(self, output): + self.parse_random_seed(output) + self.check_executed_tests(output, self.tests) + + def run_tests(self, args): + res = script_helper.assert_python_ok(*args) + output = os.fsdecode(res.out) + self.check_output(output) + + def test_script_regrtest(self): + # Lib/test/regrtest.py + script = os.path.join(ROOT_DIR, 'Lib', 'test', 'regrtest.py') + + args = [*self.python_args, script, *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_test(self): + # -m test + args = [*self.python_args, '-m', 'test', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_regrtest(self): + # -m test.regrtest + args = [*self.python_args, '-m', 'test.regrtest', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_autotest(self): + # -m test.autotest + args = [*self.python_args, '-m', 'test.autotest', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_from_test_autotest(self): + # from test import autotest + code = 'from test import autotest' + args = [*self.python_args, '-c', code, + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_script_autotest(self): + # Lib/test/autotest.py + script = os.path.join(ROOT_DIR, 'Lib', 'test', 'autotest.py') + args = [*self.python_args, script, *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_tools_script_run_tests(self): + # Tools/scripts/run_tests.py + script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') + self.run_tests([script, *self.tests]) + + def run_rt_bat(self, script, *args): + rt_args = [] + if platform.architecture()[0] == '64bit': + rt_args.append('-x64') # 64-bit build + if Py_DEBUG: + rt_args.append('-d') # Debug build + + args = [script, *rt_args, *args] + proc = subprocess.run(args, + check=True, universal_newlines=True, + input='', + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.check_output(proc.stdout) + + @unittest.skipUnless(sys.platform == 'win32', 'Windows only') + def test_tools_buildbot_test(self): + # Tools\buildbot\test.bat + script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat') + self.run_rt_bat(script, *self.tests) + + @unittest.skipUnless(sys.platform == 'win32', 'Windows only') + def test_pcbuild_rt(self): + # PCbuild\rt.bat + script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat') + # -q: quick, don't run tests twice + rt_args = ["-q"] + self.run_rt_bat(script, *rt_args, *self.regrtest_args, *self.tests) + + +class ArgsTestCase(BaseTestCase): + """ + Test arguments of the Python test suite. + """ + + def run_tests(self, *args): + args = ['-m', 'test', *args] + res = script_helper.assert_python_ok(*args) + return os.fsdecode(res.out) + + def test_resources(self): + # test -u command line option + tests = {} + for resource in ('audio', 'network'): + code = 'from test import support\nsupport.requires(%r)' % resource + tests[resource] = self.create_test(resource, code) + test_names = sorted(tests.values()) + + # -u all: 2 resources enabled + output = self.run_tests('-u', 'all', *test_names) + self.check_executed_tests(output, test_names) + + # -u audio: 1 resource enabled + output = self.run_tests('-uaudio', *test_names) + self.check_executed_tests(output, test_names, + skipped=tests['network']) + + # no option: 0 resources enabled + output = self.run_tests(*test_names) + self.check_executed_tests(output, test_names, + skipped=test_names) + + def test_random(self): + # test -r and --randseed command line option + code = textwrap.dedent(""" + import random + print("TESTRANDOM: %s" % random.randint(1, 1000)) + """) + test = self.create_test('random', code) + + # first run to get the output with the random seed + output = self.run_tests('-r', test) + randseed = self.parse_random_seed(output) + match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) + test_random = int(match.group(1)) + + # try to reproduce with the random seed + output = self.run_tests('-r', '--randseed=%s' % randseed, test) + randseed2 = self.parse_random_seed(output) + self.assertEqual(randseed2, randseed) + + match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) + test_random2 = int(match.group(1)) + self.assertEqual(test_random2, test_random) + + def test_fromfile(self): + # test --fromfile + tests = [self.create_test() for index in range(5)] + + # Write the list of files using a format similar to regrtest output: + # [1/2] test_1 + # [2/2] test_2 + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + with open(filename, "w") as fp: + for index, name in enumerate(tests, 1): + print("[%s/%s] %s" % (index, len(tests), name), file=fp) + + output = self.run_tests('--fromfile', filename) + self.check_executed_tests(output, tests) + + def test_slow(self): + # test --slow + tests = [self.create_test() for index in range(3)] + output = self.run_tests("--slow", *tests) + self.check_executed_tests(output, tests) + regex = ('10 slowest tests:\n' + '(?:%s: [0-9]+\.[0-9]+s\n){%s}' + % (self.TESTNAME_REGEX, len(tests))) + self.check_line(output, regex) + + @unittest.skipIf(sys.platform == 'win32', + "FIXME: coverage doesn't work on Windows") + def test_coverage(self): + # test --coverage + test = self.create_test() + output = self.run_tests("--coverage", test) + executed = self.parse_executed_tests(output) + self.assertEqual(executed, {test}, output) + regex = ('lines +cov% +module +\(path\)\n' + '(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') + self.check_line(output, regex) + + if __name__ == '__main__': unittest.main() -- cgit v1.2.1 From 8d7ff406bcc8bf2c170203426b2b3a5bf253ad7f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 01:02:37 +0200 Subject: Fix test_regrtest.test_tools_buildbot_test() Issue #25220: Fix test_regrtest.test_tools_buildbot_test() on release build (on Windows), pass "+d" option to test.bat. --- Lib/test/test_regrtest.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c0f1e7b2af..ca4b356fc4 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -438,14 +438,7 @@ class ProgramsTestCase(BaseTestCase): script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') self.run_tests([script, *self.tests]) - def run_rt_bat(self, script, *args): - rt_args = [] - if platform.architecture()[0] == '64bit': - rt_args.append('-x64') # 64-bit build - if Py_DEBUG: - rt_args.append('-d') # Debug build - - args = [script, *rt_args, *args] + def run_batch(self, *args): proc = subprocess.run(args, check=True, universal_newlines=True, input='', @@ -456,15 +449,23 @@ class ProgramsTestCase(BaseTestCase): def test_tools_buildbot_test(self): # Tools\buildbot\test.bat script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat') - self.run_rt_bat(script, *self.tests) + test_args = [] + if platform.architecture()[0] == '64bit': + test_args.append('-x64') # 64-bit build + if not Py_DEBUG: + test_args.append('+d') # Release build, use python.exe + self.run_batch(script, *test_args, *self.tests) @unittest.skipUnless(sys.platform == 'win32', 'Windows only') def test_pcbuild_rt(self): # PCbuild\rt.bat script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat') - # -q: quick, don't run tests twice - rt_args = ["-q"] - self.run_rt_bat(script, *rt_args, *self.regrtest_args, *self.tests) + rt_args = ["-q"] # Quick, don't run tests twice + if platform.architecture()[0] == '64bit': + rt_args.append('-x64') # 64-bit build + if Py_DEBUG: + rt_args.append('-d') # Debug build, use python_d.exe + self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) class ArgsTestCase(BaseTestCase): -- cgit v1.2.1 From 3bbd4667ba3fa0ef9475bab0d711b38d1708b88d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 12:32:13 +0200 Subject: Optimize ascii/latin1+surrogateescape encoders Issue #25227: Optimize ASCII and latin1 encoders with the ``surrogateescape`` error handler: the encoders are now up to 3 times as fast. Initial patch written by Serhiy Storchaka. --- Lib/test/test_codecs.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index e0e31199cc..254c0c1d64 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3060,7 +3060,31 @@ class CodePageTest(unittest.TestCase): class ASCIITest(unittest.TestCase): + def test_encode(self): + self.assertEqual('abc123'.encode('ascii'), b'abc123') + + def test_encode_error(self): + for data, error_handler, expected in ( + ('[\x80\xff\u20ac]', 'ignore', b'[]'), + ('[\x80\xff\u20ac]', 'replace', b'[???]'), + ('[\x80\xff\u20ac]', 'xmlcharrefreplace', b'[€ÿ€]'), + ('[\x80\xff\u20ac]', 'backslashreplace', b'[\\x80\\xff\\u20ac]'), + ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.encode('ascii', error_handler), + expected) + + def test_encode_surrogateescape_error(self): + with self.assertRaises(UnicodeEncodeError): + # the first character can be decoded, but not the second + '\udc80\xff'.encode('ascii', 'surrogateescape') + def test_decode(self): + self.assertEqual(b'abc'.decode('ascii'), 'abc') + + def test_decode_error(self): for data, error_handler, expected in ( (b'[\x80\xff]', 'ignore', '[]'), (b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'), @@ -3073,5 +3097,41 @@ class ASCIITest(unittest.TestCase): expected) +class Latin1Test(unittest.TestCase): + def test_encode(self): + for data, expected in ( + ('abc', b'abc'), + ('\x80\xe9\xff', b'\x80\xe9\xff'), + ): + with self.subTest(data=data, expected=expected): + self.assertEqual(data.encode('latin1'), expected) + + def test_encode_errors(self): + for data, error_handler, expected in ( + ('[\u20ac\udc80]', 'ignore', b'[]'), + ('[\u20ac\udc80]', 'replace', b'[??]'), + ('[\u20ac\udc80]', 'backslashreplace', b'[\\u20ac\\udc80]'), + ('[\u20ac\udc80]', 'xmlcharrefreplace', b'[€�]'), + ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.encode('latin1', error_handler), + expected) + + def test_encode_surrogateescape_error(self): + with self.assertRaises(UnicodeEncodeError): + # the first character can be decoded, but not the second + '\udc80\u20ac'.encode('latin1', 'surrogateescape') + + def test_decode(self): + for data, expected in ( + (b'abc', 'abc'), + (b'[\x80\xff]', '[\x80\xff]'), + ): + with self.subTest(data=data, expected=expected): + self.assertEqual(data.decode('latin1'), expected) + + if __name__ == "__main__": unittest.main() -- cgit v1.2.1 From e91e4e95302cb13fed6083ff025c27f2104239b3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 13:47:15 +0200 Subject: test --- Lib/test/libregrtest/main.py | 4 +--- Lib/test/test_regrtest.py | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 5d34cffd49..3724f27b23 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -246,9 +246,7 @@ def main(tests=None, **kwargs): random.shuffle(selected) if ns.trace: import trace, tempfile - tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) + tracer = trace.Trace(trace=False, count=True) test_times = [] support.verbose = ns.verbose # Tell tests to be moderately quiet diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ca4b356fc4..5906c17365 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -549,8 +549,6 @@ class ArgsTestCase(BaseTestCase): % (self.TESTNAME_REGEX, len(tests))) self.check_line(output, regex) - @unittest.skipIf(sys.platform == 'win32', - "FIXME: coverage doesn't work on Windows") def test_coverage(self): # test --coverage test = self.create_test() -- cgit v1.2.1 From 5a84690ade4861e24e02a5a816ee187d581683a2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 14:02:35 +0200 Subject: Oops, revert unwanted change, sorry --- Lib/test/libregrtest/main.py | 4 +++- Lib/test/test_regrtest.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 3724f27b23..5d34cffd49 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -246,7 +246,9 @@ def main(tests=None, **kwargs): random.shuffle(selected) if ns.trace: import trace, tempfile - tracer = trace.Trace(trace=False, count=True) + tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, + tempfile.gettempdir()], + trace=False, count=True) test_times = [] support.verbose = ns.verbose # Tell tests to be moderately quiet diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 5906c17365..ca4b356fc4 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -549,6 +549,8 @@ class ArgsTestCase(BaseTestCase): % (self.TESTNAME_REGEX, len(tests))) self.check_line(output, regex) + @unittest.skipIf(sys.platform == 'win32', + "FIXME: coverage doesn't work on Windows") def test_coverage(self): # test --coverage test = self.create_test() -- cgit v1.2.1 From 1ff09acb85b2410e9a03a356e8f5bc332bdf1177 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 14:17:09 +0200 Subject: Issue #25220: Add test for --wait in test_regrtest Replace script_helper.assert_python_ok() with subprocess.run(). --- Lib/test/test_regrtest.py | 48 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 12 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ca4b356fc4..54bbfe4cd7 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -16,7 +16,6 @@ import textwrap import unittest from test import libregrtest from test import support -from test.support import script_helper Py_DEBUG = hasattr(sys, 'getobjects') @@ -366,6 +365,31 @@ class BaseTestCase(unittest.TestCase): self.assertTrue(0 <= randseed <= 10000000, randseed) return randseed + def run_command(self, args, input=None): + if not input: + input = '' + try: + return subprocess.run(args, + check=True, universal_newlines=True, + input=input, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except subprocess.CalledProcessError as exc: + self.fail("%s\n" + "\n" + "stdout:\n" + "%s\n" + "\n" + "stderr:\n" + "%s" + % (str(exc), exc.stdout, exc.stderr)) + + + def run_python(self, args, **kw): + args = [sys.executable, '-X', 'faulthandler', '-I', *args] + proc = self.run_command(args, **kw) + return proc.stdout + class ProgramsTestCase(BaseTestCase): """ @@ -391,9 +415,8 @@ class ProgramsTestCase(BaseTestCase): self.check_executed_tests(output, self.tests) def run_tests(self, args): - res = script_helper.assert_python_ok(*args) - output = os.fsdecode(res.out) - self.check_output(output) + stdout = self.run_python(args) + self.check_output(stdout) def test_script_regrtest(self): # Lib/test/regrtest.py @@ -439,10 +462,7 @@ class ProgramsTestCase(BaseTestCase): self.run_tests([script, *self.tests]) def run_batch(self, *args): - proc = subprocess.run(args, - check=True, universal_newlines=True, - input='', - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = self.run_command(args) self.check_output(proc.stdout) @unittest.skipUnless(sys.platform == 'win32', 'Windows only') @@ -473,10 +493,8 @@ class ArgsTestCase(BaseTestCase): Test arguments of the Python test suite. """ - def run_tests(self, *args): - args = ['-m', 'test', *args] - res = script_helper.assert_python_ok(*args) - return os.fsdecode(res.out) + def run_tests(self, *args, input=None): + return self.run_python(['-m', 'test', *args], input=input) def test_resources(self): # test -u command line option @@ -561,6 +579,12 @@ class ArgsTestCase(BaseTestCase): '(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') self.check_line(output, regex) + def test_wait(self): + # test --wait + test = self.create_test() + output = self.run_tests("--wait", test, input='key') + self.check_line(output, 'Press any key to continue') + if __name__ == '__main__': unittest.main() -- cgit v1.2.1 From 34c435531477a049e79d93c757a1db68f5a30719 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 22:48:52 +0200 Subject: Issue #25220: Split the huge main() function of libregrtest.main into a class with attributes and methods. The --threshold command line option is now ignored if the gc module is missing. * Convert main() variables to Regrtest attributes, document some attributes * Convert accumulate_result() function to a method * Create setup_python() function and setup_regrtest() method. * Import gc at top level * Move resource.setrlimit() and the code to make the module paths absolute into the new setup_python() function. So this code is no more executed when the module is imported, only when main() is executed. We have a better control on when the setup is done. * Move textwrap import from printlist() to the top level. * Some other minor cleanup. --- Lib/test/libregrtest/main.py | 701 ++++++++++++++++++++++++------------------- 1 file changed, 387 insertions(+), 314 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 5d34cffd49..63388c957c 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,13 +1,14 @@ import faulthandler import json import os +import platform +import random import re +import signal import sys -import tempfile import sysconfig -import signal -import random -import platform +import tempfile +import textwrap import traceback import unittest from test.libregrtest.runtest import ( @@ -18,46 +19,16 @@ from test.libregrtest.runtest import ( from test.libregrtest.refleak import warm_caches from test.libregrtest.cmdline import _parse_args from test import support +try: + import gc +except ImportError: + gc = None try: import threading except ImportError: threading = None -# Some times __path__ and __file__ are not absolute (e.g. while running from -# Lib/) and, if we change the CWD to run the tests in a temporary dir, some -# imports might fail. This affects only the modules imported before os.chdir(). -# These modules are searched first in sys.path[0] (so '' -- the CWD) and if -# they are found in the CWD their __file__ and __path__ will be relative (this -# happens before the chdir). All the modules imported after the chdir, are -# not found in the CWD, and since the other paths in sys.path[1:] are absolute -# (site.py absolutize them), the __file__ and __path__ will be absolute too. -# Therefore it is necessary to absolutize manually the __file__ and __path__ of -# the packages to prevent later imports to fail when the CWD is different. -for module in sys.modules.values(): - if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) for path in module.__path__] - if hasattr(module, '__file__'): - module.__file__ = os.path.abspath(module.__file__) - - -# MacOSX (a.k.a. Darwin) has a default stack size that is too small -# for deeply recursive regular expressions. We see this as crashes in -# the Python test suite when running test_re.py and test_sre.py. The -# fix is to set the stack limit to 2048. -# This approach may also be useful for other Unixy platforms that -# suffer from small default stack limits. -if sys.platform == 'darwin': - try: - import resource - except ImportError: - pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) - - # When tests are run from the Python build directory, it is best practice # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. @@ -68,7 +39,73 @@ else: TEMPDIR = os.path.abspath(TEMPDIR) -def main(tests=None, **kwargs): +def slave_runner(slaveargs): + args, kwargs = json.loads(slaveargs) + if kwargs.get('huntrleaks'): + unittest.BaseTestSuite._cleanup = False + try: + result = runtest(*args, **kwargs) + except KeyboardInterrupt: + result = INTERRUPTED, '' + except BaseException as e: + traceback.print_exc() + result = CHILD_ERROR, str(e) + sys.stdout.flush() + print() # Force a newline (just in case) + print(json.dumps(result)) + sys.exit(0) + + +def setup_python(): + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + + replace_stdout() + support.record_original_stdout(sys.stdout) + + # Some times __path__ and __file__ are not absolute (e.g. while running from + # Lib/) and, if we change the CWD to run the tests in a temporary dir, some + # imports might fail. This affects only the modules imported before os.chdir(). + # These modules are searched first in sys.path[0] (so '' -- the CWD) and if + # they are found in the CWD their __file__ and __path__ will be relative (this + # happens before the chdir). All the modules imported after the chdir, are + # not found in the CWD, and since the other paths in sys.path[1:] are absolute + # (site.py absolutize them), the __file__ and __path__ will be absolute too. + # Therefore it is necessary to absolutize manually the __file__ and __path__ of + # the packages to prevent later imports to fail when the CWD is different. + for module in sys.modules.values(): + if hasattr(module, '__path__'): + module.__path__ = [os.path.abspath(path) for path in module.__path__] + if hasattr(module, '__file__'): + module.__file__ = os.path.abspath(module.__file__) + + # MacOSX (a.k.a. Darwin) has a default stack size that is too small + # for deeply recursive regular expressions. We see this as crashes in + # the Python test suite when running test_re.py and test_sre.py. The + # fix is to set the stack limit to 2048. + # This approach may also be useful for other Unixy platforms that + # suffer from small default stack limits. + if sys.platform == 'darwin': + try: + import resource + except ImportError: + pass + else: + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + newsoft = min(hard, max(soft, 1024*2048)) + resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + + +class Regrtest: """Execute a test suite. This also parses command-line options and modifies its behavior @@ -91,210 +128,257 @@ def main(tests=None, **kwargs): directly to set the values that would normally be set by flags on the command line. """ - # Display the Python traceback on fatal errors (e.g. segfault) - faulthandler.enable(all_threads=True) + def __init__(self): + # Namespace of command line options + self.ns = None + + # tests + self.tests = [] + self.selected = [] + + # test results + self.good = [] + self.bad = [] + self.skipped = [] + self.resource_denieds = [] + self.environment_changed = [] + self.interrupted = False - # Display the Python traceback on SIGALRM or SIGUSR1 signal - signals = [] - if hasattr(signal, 'SIGALRM'): - signals.append(signal.SIGALRM) - if hasattr(signal, 'SIGUSR1'): - signals.append(signal.SIGUSR1) - for signum in signals: - faulthandler.register(signum, chain=True) + # used by --slow + self.test_times = [] - replace_stdout() + # used by --coverage, trace.Trace instance + self.tracer = None - support.record_original_stdout(sys.stdout) + # used by --findleaks, store for gc.garbage + self.found_garbage = [] - ns = _parse_args(sys.argv[1:], **kwargs) - - if ns.huntrleaks: - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) - if ns.threshold is not None: - import gc - gc.set_threshold(ns.threshold) - if ns.nowindows: - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - if ns.wait: - input("Press any key to continue...") - - if ns.slaveargs is not None: - args, kwargs = json.loads(ns.slaveargs) - if kwargs.get('huntrleaks'): - unittest.BaseTestSuite._cleanup = False - try: - result = runtest(*args, **kwargs) - except KeyboardInterrupt: - result = INTERRUPTED, '' - except BaseException as e: - traceback.print_exc() - result = CHILD_ERROR, str(e) - sys.stdout.flush() - print() # Force a newline (just in case) - print(json.dumps(result)) - sys.exit(0) - - good = [] - bad = [] - skipped = [] - resource_denieds = [] - environment_changed = [] - interrupted = False - - if ns.findleaks: - try: - import gc - except ImportError: - print('No GC available, disabling findleaks.') - ns.findleaks = False - else: - # Uncomment the line below to report garbage that is not - # freeable by reference counting alone. By default only - # garbage that is not collectable by the GC is reported. - #gc.set_debug(gc.DEBUG_SAVEALL) - found_garbage = [] - - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False + # used to display the progress bar "[ 3/100]" + self.test_count = '' + self.test_count_width = 1 - if ns.single: - filename = os.path.join(TEMPDIR, 'pynexttest') - try: - with open(filename, 'r') as fp: - next_test = fp.read().strip() - tests = [next_test] - except OSError: - pass + # used by --single + self.next_single_test = None + self.next_single_filename = None - if ns.fromfile: - tests = [] - with open(os.path.join(support.SAVEDCWD, ns.fromfile)) as fp: - count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') - for line in fp: - line = count_pat.sub('', line) - guts = line.split() # assuming no test has whitespace in its name - if guts and not guts[0].startswith('#'): - tests.extend(guts) - - # Strip .py extensions. - removepy(ns.args) - removepy(tests) - - stdtests = STDTESTS[:] - nottests = NOTTESTS.copy() - if ns.exclude: - for arg in ns.args: - if arg in stdtests: - stdtests.remove(arg) - nottests.add(arg) - ns.args = [] - - # For a partial run, we do not need to clutter the output. - if ns.verbose or ns.header or not (ns.quiet or ns.single or tests or ns.args): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("== ", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== ", "hash algorithm:", sys.hash_info.algorithm, - "64bit" if sys.maxsize > 2**32 else "32bit") - print("== ", os.getcwd()) - print("Testing with flags:", sys.flags) - - # if testdir is set, then we are not running the python tests suite, so - # don't add default tests to be executed or skipped (pass empty values) - if ns.testdir: - alltests = findtests(ns.testdir, list(), set()) - else: - alltests = findtests(ns.testdir, stdtests, nottests) - - selected = tests or ns.args or alltests - if ns.single: - selected = selected[:1] - try: - next_single_test = alltests[alltests.index(selected[0])+1] - except IndexError: - next_single_test = None - # Remove all the selected tests that precede start if it's set. - if ns.start: - try: - del selected[:selected.index(ns.start)] - except ValueError: - print("Couldn't find starting test (%s), using all tests" % ns.start) - if ns.randomize: - if ns.random_seed is None: - ns.random_seed = random.randrange(10000000) - random.seed(ns.random_seed) - print("Using random seed", ns.random_seed) - random.shuffle(selected) - if ns.trace: - import trace, tempfile - tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) - - test_times = [] - support.verbose = ns.verbose # Tell tests to be moderately quiet - support.use_resources = ns.use_resources - save_modules = sys.modules.keys() - - def accumulate_result(test, result): + def accumulate_result(self, test, result): ok, test_time = result - test_times.append((test_time, test)) + self.test_times.append((test_time, test)) if ok == PASSED: - good.append(test) + self.good.append(test) elif ok == FAILED: - bad.append(test) + self.bad.append(test) elif ok == ENV_CHANGED: - environment_changed.append(test) + self.environment_changed.append(test) elif ok == SKIPPED: - skipped.append(test) + self.skipped.append(test) elif ok == RESOURCE_DENIED: - skipped.append(test) - resource_denieds.append(test) - - if ns.forever: - def test_forever(tests=list(selected)): - while True: - for test in tests: - yield test - if bad: - return - tests = test_forever() - test_count = '' - test_count_width = 3 - else: - tests = iter(selected) - test_count = '/{}'.format(len(selected)) - test_count_width = len(test_count) - 1 + self.skipped.append(test) + self.resource_denieds.append(test) + + def display_progress(self, test_index, test): + if self.ns.quiet: + return + fmt = "[{1:{0}}{2}/{3}] {4}" if self.bad else "[{1:{0}}{2}] {4}" + print(fmt.format( + self.test_count_width, test_index, self.test_count, len(self.bad), test)) + sys.stdout.flush() + + def setup_regrtest(self): + if self.ns.huntrleaks: + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() - if ns.use_mp: + if self.ns.memlimit is not None: + support.set_memlimit(self.ns.memlimit) + + if self.ns.threshold is not None: + if gc is not None: + gc.set_threshold(self.ns.threshold) + else: + print('No GC available, ignore --threshold.') + + if self.ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + + if self.ns.findleaks: + if gc is not None: + # Uncomment the line below to report garbage that is not + # freeable by reference counting alone. By default only + # garbage that is not collectable by the GC is reported. + pass + #gc.set_debug(gc.DEBUG_SAVEALL) + else: + print('No GC available, disabling --findleaks') + self.ns.findleaks = False + + if self.ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + # Strip .py extensions. + removepy(self.ns.args) + + if self.ns.trace: + import trace + self.tracer = trace.Trace(ignoredirs=[sys.base_prefix, + sys.base_exec_prefix, + tempfile.gettempdir()], + trace=False, count=True) + + def find_tests(self, tests): + self.tests = tests + + if self.ns.single: + self.next_single_filename = os.path.join(TEMPDIR, 'pynexttest') + try: + with open(self.next_single_filename, 'r') as fp: + next_test = fp.read().strip() + self.tests = [next_test] + except OSError: + pass + + if self.ns.fromfile: + self.tests = [] + with open(os.path.join(support.SAVEDCWD, self.ns.fromfile)) as fp: + count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') + for line in fp: + line = count_pat.sub('', line) + guts = line.split() # assuming no test has whitespace in its name + if guts and not guts[0].startswith('#'): + self.tests.extend(guts) + + removepy(self.tests) + + stdtests = STDTESTS[:] + nottests = NOTTESTS.copy() + if self.ns.exclude: + for arg in self.ns.args: + if arg in stdtests: + stdtests.remove(arg) + nottests.add(arg) + self.ns.args = [] + + # For a partial run, we do not need to clutter the output. + if self.ns.verbose or self.ns.header or not (self.ns.quiet or self.ns.single or self.tests or self.ns.args): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("== ", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== ", "hash algorithm:", sys.hash_info.algorithm, + "64bit" if sys.maxsize > 2**32 else "32bit") + print("== ", os.getcwd()) + print("Testing with flags:", sys.flags) + + # if testdir is set, then we are not running the python tests suite, so + # don't add default tests to be executed or skipped (pass empty values) + if self.ns.testdir: + alltests = findtests(self.ns.testdir, list(), set()) + else: + alltests = findtests(self.ns.testdir, stdtests, nottests) + + self.selected = self.tests or self.ns.args or alltests + if self.ns.single: + self.selected = self.selected[:1] + try: + pos = alltests.index(self.selected[0]) + self.next_single_test = alltests[pos + 1] + except IndexError: + pass + + # Remove all the self.selected tests that precede start if it's set. + if self.ns.start: + try: + del self.selected[:self.selected.index(self.ns.start)] + except ValueError: + print("Couldn't find starting test (%s), using all tests" % self.ns.start) + + if self.ns.randomize: + if self.ns.random_seed is None: + self.ns.random_seed = random.randrange(10000000) + random.seed(self.ns.random_seed) + print("Using random seed", self.ns.random_seed) + random.shuffle(self.selected) + + def display_result(self): + if self.interrupted: + # print a newline after ^C + print() + print("Test suite interrupted by signal SIGINT.") + omitted = set(self.selected) - set(self.good) - set(self.bad) - set(self.skipped) + print(count(len(omitted), "test"), "omitted:") + printlist(omitted) + + if self.good and not self.ns.quiet: + if not self.bad and not self.skipped and not self.interrupted and len(self.good) > 1: + print("All", end=' ') + print(count(len(self.good), "test"), "OK.") + + if self.ns.print_slow: + self.test_times.sort(reverse=True) + print("10 slowest tests:") + for time, test in self.test_times[:10]: + print("%s: %.1fs" % (test, time)) + + if self.bad: + print(count(len(self.bad), "test"), "failed:") + printlist(self.bad) + + if self.environment_changed: + print("{} altered the execution environment:".format( + count(len(self.environment_changed), "test"))) + printlist(self.environment_changed) + + if self.skipped and not self.ns.quiet: + print(count(len(self.skipped), "test"), "skipped:") + printlist(self.skipped) + + if self.ns.verbose2 and self.bad: + print("Re-running failed tests in verbose mode") + for test in self.bad[:]: + print("Re-running test %r in verbose mode" % test) + sys.stdout.flush() + try: + self.ns.verbose = True + ok = runtest(test, True, self.ns.quiet, self.ns.huntrleaks, + timeout=self.ns.timeout) + except KeyboardInterrupt: + # print a newline separate from the ^C + print() + break + else: + if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: + self.bad.remove(test) + else: + if self.bad: + print(count(len(self.bad), 'test'), "failed again:") + printlist(self.bad) + + def _run_tests_mp(self): try: from threading import Thread except ImportError: print("Multiprocess option requires thread support") sys.exit(2) from queue import Queue + debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") output = Queue() - pending = MultiprocessTests(tests) + pending = MultiprocessTests(self.tests) + def work(): # A worker thread. try: @@ -304,7 +388,7 @@ def main(tests=None, **kwargs): except StopIteration: output.put((None, None, None, None)) return - retcode, stdout, stderr = run_test_in_subprocess(test, ns) + retcode, stdout, stderr = run_test_in_subprocess(test, self.ns) # Strip last refcount output line if it exists, since it # comes from the shutdown of the interpreter in the subcommand. stderr = debug_output_pat.sub("", stderr) @@ -321,23 +405,20 @@ def main(tests=None, **kwargs): except BaseException: output.put((None, None, None, None)) raise - workers = [Thread(target=work) for i in range(ns.use_mp)] + + workers = [Thread(target=work) for i in range(self.ns.use_mp)] for worker in workers: worker.start() finished = 0 test_index = 1 try: - while finished < ns.use_mp: + while finished < self.ns.use_mp: test, stdout, stderr, result = output.get() if test is None: finished += 1 continue - accumulate_result(test, result) - if not ns.quiet: - fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - test_count_width, test_index, test_count, - len(bad), test)) + self.accumulate_result(test, result) + self.display_progress(test_index, test) if stdout: print(stdout) if stderr: @@ -350,110 +431,99 @@ def main(tests=None, **kwargs): raise Exception("Child error on {}: {}".format(test, result[1])) test_index += 1 except KeyboardInterrupt: - interrupted = True + self.interrupted = True pending.interrupted = True for worker in workers: worker.join() - else: - for test_index, test in enumerate(tests, 1): - if not ns.quiet: - fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - test_count_width, test_index, test_count, len(bad), test)) - sys.stdout.flush() - if ns.trace: + + def _run_tests_sequential(self): + save_modules = sys.modules.keys() + + for test_index, test in enumerate(self.tests, 1): + self.display_progress(test_index, test) + if self.ns.trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - tracer.runctx('runtest(test, ns.verbose, ns.quiet, timeout=ns.timeout)', - globals=globals(), locals=vars()) + cmd = 'runtest(test, self.ns.verbose, self.ns.quiet, timeout=self.ns.timeout)' + self.tracer.runctx(cmd, globals=globals(), locals=vars()) else: try: - result = runtest(test, ns.verbose, ns.quiet, - ns.huntrleaks, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests) - accumulate_result(test, result) + result = runtest(test, self.ns.verbose, self.ns.quiet, + self.ns.huntrleaks, + output_on_failure=self.ns.verbose3, + timeout=self.ns.timeout, failfast=self.ns.failfast, + match_tests=self.ns.match_tests) + self.accumulate_result(test, result) except KeyboardInterrupt: - interrupted = True + self.interrupted = True break - if ns.findleaks: + if self.ns.findleaks: gc.collect() if gc.garbage: print("Warning: test created", len(gc.garbage), end=' ') print("uncollectable object(s).") # move the uncollectable objects somewhere so we don't see # them again - found_garbage.extend(gc.garbage) + self.found_garbage.extend(gc.garbage) del gc.garbage[:] # Unload the newly imported modules (best effort finalization) for module in sys.modules.keys(): if module not in save_modules and module.startswith("test."): support.unload(module) - if interrupted: - # print a newline after ^C - print() - print("Test suite interrupted by signal SIGINT.") - omitted = set(selected) - set(good) - set(bad) - set(skipped) - print(count(len(omitted), "test"), "omitted:") - printlist(omitted) - if good and not ns.quiet: - if not bad and not skipped and not interrupted and len(good) > 1: - print("All", end=' ') - print(count(len(good), "test"), "OK.") - if ns.print_slow: - test_times.sort(reverse=True) - print("10 slowest tests:") - for time, test in test_times[:10]: - print("%s: %.1fs" % (test, time)) - if bad: - print(count(len(bad), "test"), "failed:") - printlist(bad) - if environment_changed: - print("{} altered the execution environment:".format( - count(len(environment_changed), "test"))) - printlist(environment_changed) - if skipped and not ns.quiet: - print(count(len(skipped), "test"), "skipped:") - printlist(skipped) - - if ns.verbose2 and bad: - print("Re-running failed tests in verbose mode") - for test in bad[:]: - print("Re-running test %r in verbose mode" % test) - sys.stdout.flush() - try: - ns.verbose = True - ok = runtest(test, True, ns.quiet, ns.huntrleaks, - timeout=ns.timeout) - except KeyboardInterrupt: - # print a newline separate from the ^C - print() - break - else: - if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: - bad.remove(test) - else: - if bad: - print(count(len(bad), 'test'), "failed again:") - printlist(bad) - - if ns.single: - if next_single_test: - with open(filename, 'w') as fp: - fp.write(next_single_test + '\n') - else: - os.unlink(filename) + def run_tests(self): + support.verbose = self.ns.verbose # Tell tests to be moderately quiet + support.use_resources = self.ns.use_resources - if ns.trace: - r = tracer.results() - r.write_results(show_missing=True, summary=True, coverdir=ns.coverdir) + if self.ns.forever: + def test_forever(tests): + while True: + for test in tests: + yield test + if self.bad: + return + self.tests = test_forever(list(self.selected)) + self.test_count = '' + self.test_count_width = 3 + else: + self.tests = iter(self.selected) + self.test_count = '/{}'.format(len(self.selected)) + self.test_count_width = len(self.test_count) - 1 - if ns.runleaks: - os.system("leaks %d" % os.getpid()) + if self.ns.use_mp: + self._run_tests_mp() + else: + self._run_tests_sequential() - sys.exit(len(bad) > 0 or interrupted) + def finalize(self): + if self.next_single_filename: + if self.next_single_test: + with open(self.next_single_filename, 'w') as fp: + fp.write(self.next_single_test + '\n') + else: + os.unlink(self.next_single_filename) + + if self.ns.trace: + r = self.tracer.results() + r.write_results(show_missing=True, summary=True, + coverdir=self.ns.coverdir) + + if self.ns.runleaks: + os.system("leaks %d" % os.getpid()) + + def main(self, tests=None, **kwargs): + setup_python() + self.ns = _parse_args(sys.argv[1:], **kwargs) + self.setup_regrtest() + if self.ns.wait: + input("Press any key to continue...") + if self.ns.slaveargs is not None: + slave_runner(self.ns.slaveargs) + self.find_tests(tests) + self.run_tests() + self.display_result() + self.finalize() + sys.exit(len(self.bad) > 0 or self.interrupted) # We do not use a generator so multiple threads can call next(). @@ -518,11 +588,14 @@ def printlist(x, width=70, indent=4): begin each line. """ - from textwrap import fill blanks = ' ' * indent # Print the sorted list: 'x' may be a '--random' list or a set() - print(fill(' '.join(str(elt) for elt in sorted(x)), width, - initial_indent=blanks, subsequent_indent=blanks)) + print(textwrap.fill(' '.join(str(elt) for elt in sorted(x)), width, + initial_indent=blanks, subsequent_indent=blanks)) + + +def main(tests=None, **kwargs): + Regrtest().main(tests=tests, **kwargs) def main_in_temp_cwd(): -- cgit v1.2.1 From 63ddb4e643736ffb8e899173626b6656f28c899d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:15:38 +0200 Subject: Issue #25220: Create libregrtest/runtest_mp.py Move the code to run tests in multiple processes using threading and subprocess to a new submodule. Move also slave_runner() (renamed to run_tests_slave()) and run_test_in_subprocess() (renamed to run_tests_in_subprocess()) there. --- Lib/test/libregrtest/main.py | 120 ++-------------------------- Lib/test/libregrtest/runtest.py | 33 -------- Lib/test/libregrtest/runtest_mp.py | 158 +++++++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 147 deletions(-) create mode 100644 Lib/test/libregrtest/runtest_mp.py (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 63388c957c..b66f045045 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,5 +1,4 @@ import faulthandler -import json import os import platform import random @@ -9,13 +8,10 @@ import sys import sysconfig import tempfile import textwrap -import traceback import unittest from test.libregrtest.runtest import ( - findtests, runtest, run_test_in_subprocess, - STDTESTS, NOTTESTS, - PASSED, FAILED, ENV_CHANGED, SKIPPED, - RESOURCE_DENIED, INTERRUPTED, CHILD_ERROR) + findtests, runtest, + STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) from test.libregrtest.refleak import warm_caches from test.libregrtest.cmdline import _parse_args from test import support @@ -39,23 +35,6 @@ else: TEMPDIR = os.path.abspath(TEMPDIR) -def slave_runner(slaveargs): - args, kwargs = json.loads(slaveargs) - if kwargs.get('huntrleaks'): - unittest.BaseTestSuite._cleanup = False - try: - result = runtest(*args, **kwargs) - except KeyboardInterrupt: - result = INTERRUPTED, '' - except BaseException as e: - traceback.print_exc() - result = CHILD_ERROR, str(e) - sys.stdout.flush() - print() # Force a newline (just in case) - print(json.dumps(result)) - sys.exit(0) - - def setup_python(): # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) @@ -367,75 +346,6 @@ class Regrtest: print(count(len(self.bad), 'test'), "failed again:") printlist(self.bad) - def _run_tests_mp(self): - try: - from threading import Thread - except ImportError: - print("Multiprocess option requires thread support") - sys.exit(2) - from queue import Queue - - debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") - output = Queue() - pending = MultiprocessTests(self.tests) - - def work(): - # A worker thread. - try: - while True: - try: - test = next(pending) - except StopIteration: - output.put((None, None, None, None)) - return - retcode, stdout, stderr = run_test_in_subprocess(test, self.ns) - # Strip last refcount output line if it exists, since it - # comes from the shutdown of the interpreter in the subcommand. - stderr = debug_output_pat.sub("", stderr) - stdout, _, result = stdout.strip().rpartition("\n") - if retcode != 0: - result = (CHILD_ERROR, "Exit code %s" % retcode) - output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - return - if not result: - output.put((None, None, None, None)) - return - result = json.loads(result) - output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - except BaseException: - output.put((None, None, None, None)) - raise - - workers = [Thread(target=work) for i in range(self.ns.use_mp)] - for worker in workers: - worker.start() - finished = 0 - test_index = 1 - try: - while finished < self.ns.use_mp: - test, stdout, stderr, result = output.get() - if test is None: - finished += 1 - continue - self.accumulate_result(test, result) - self.display_progress(test_index, test) - if stdout: - print(stdout) - if stderr: - print(stderr, file=sys.stderr) - sys.stdout.flush() - sys.stderr.flush() - if result[0] == INTERRUPTED: - raise KeyboardInterrupt - if result[0] == CHILD_ERROR: - raise Exception("Child error on {}: {}".format(test, result[1])) - test_index += 1 - except KeyboardInterrupt: - self.interrupted = True - pending.interrupted = True - for worker in workers: - worker.join() - def _run_tests_sequential(self): save_modules = sys.modules.keys() @@ -491,7 +401,8 @@ class Regrtest: self.test_count_width = len(self.test_count) - 1 if self.ns.use_mp: - self._run_tests_mp() + from test.libregrtest.runtest_mp import run_tests_multiprocess + run_tests_multiprocess(self) else: self._run_tests_sequential() @@ -518,7 +429,8 @@ class Regrtest: if self.ns.wait: input("Press any key to continue...") if self.ns.slaveargs is not None: - slave_runner(self.ns.slaveargs) + from test.libregrtest.runtest_mp import run_tests_slave + run_tests_slave(self.ns.slaveargs) self.find_tests(tests) self.run_tests() self.display_result() @@ -526,26 +438,6 @@ class Regrtest: sys.exit(len(self.bad) > 0 or self.interrupted) -# We do not use a generator so multiple threads can call next(). -class MultiprocessTests(object): - - """A thread-safe iterator over tests for multiprocess mode.""" - - def __init__(self, tests): - self.interrupted = False - self.lock = threading.Lock() - self.tests = tests - - def __iter__(self): - return self - - def __next__(self): - with self.lock: - if self.interrupted: - raise StopIteration('tests interrupted') - return next(self.tests) - - def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error handler) to avoid UnicodeEncodeError when printing a traceback""" diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index d8c0eb2a86..60b8f5f538 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -1,7 +1,6 @@ import faulthandler import importlib import io -import json import os import sys import time @@ -22,38 +21,6 @@ INTERRUPTED = -4 CHILD_ERROR = -5 # error in a child process -def run_test_in_subprocess(testname, ns): - """Run the given test in a subprocess with --slaveargs. - - ns is the option Namespace parsed from command-line arguments. regrtest - is invoked in a subprocess with the --slaveargs argument; when the - subprocess exits, its return code, stdout and stderr are returned as a - 3-tuple. - """ - from subprocess import Popen, PIPE - base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + - ['-X', 'faulthandler', '-m', 'test.regrtest']) - - slaveargs = ( - (testname, ns.verbose, ns.quiet), - dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests)) - # Running the child from the same working directory as regrtest's original - # invocation ensures that TEMPDIR for the child is the same when - # sysconfig.is_python_build() is true. See issue 15300. - popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], - stdout=PIPE, stderr=PIPE, - universal_newlines=True, - close_fds=(os.name != 'nt'), - cwd=support.SAVEDCWD) - stdout, stderr = popen.communicate() - retcode = popen.wait() - return retcode, stdout, stderr - - # small set of tests to determine if we have a basically functioning interpreter # (i.e. if any of these fail, then anything else is likely to follow) STDTESTS = [ diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py new file mode 100644 index 0000000000..b55c11afb5 --- /dev/null +++ b/Lib/test/libregrtest/runtest_mp.py @@ -0,0 +1,158 @@ +import json +import os +import re +import sys +import traceback +import unittest +from queue import Queue +from test import support +try: + import threading +except ImportError: + print("Multiprocess option requires thread support") + sys.exit(2) + +from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR + + +debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") + + +def run_tests_in_subprocess(testname, ns): + """Run the given test in a subprocess with --slaveargs. + + ns is the option Namespace parsed from command-line arguments. regrtest + is invoked in a subprocess with the --slaveargs argument; when the + subprocess exits, its return code, stdout and stderr are returned as a + 3-tuple. + """ + from subprocess import Popen, PIPE + base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + + ['-X', 'faulthandler', '-m', 'test.regrtest']) + + slaveargs = ( + (testname, ns.verbose, ns.quiet), + dict(huntrleaks=ns.huntrleaks, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + timeout=ns.timeout, failfast=ns.failfast, + match_tests=ns.match_tests)) + # Running the child from the same working directory as regrtest's original + # invocation ensures that TEMPDIR for the child is the same when + # sysconfig.is_python_build() is true. See issue 15300. + popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], + stdout=PIPE, stderr=PIPE, + universal_newlines=True, + close_fds=(os.name != 'nt'), + cwd=support.SAVEDCWD) + stdout, stderr = popen.communicate() + retcode = popen.wait() + return retcode, stdout, stderr + + +def run_tests_slave(slaveargs): + args, kwargs = json.loads(slaveargs) + if kwargs.get('huntrleaks'): + unittest.BaseTestSuite._cleanup = False + try: + result = runtest(*args, **kwargs) + except KeyboardInterrupt: + result = INTERRUPTED, '' + except BaseException as e: + traceback.print_exc() + result = CHILD_ERROR, str(e) + sys.stdout.flush() + print() # Force a newline (just in case) + print(json.dumps(result)) + sys.exit(0) + + +# We do not use a generator so multiple threads can call next(). +class MultiprocessIterator: + + """A thread-safe iterator over tests for multiprocess mode.""" + + def __init__(self, tests): + self.interrupted = False + self.lock = threading.Lock() + self.tests = tests + + def __iter__(self): + return self + + def __next__(self): + with self.lock: + if self.interrupted: + raise StopIteration('tests interrupted') + return next(self.tests) + + +class MultiprocessThread(threading.Thread): + def __init__(self, pending, output, ns): + super().__init__() + self.pending = pending + self.output = output + self.ns = ns + + def run(self): + # A worker thread. + try: + while True: + try: + test = next(self.pending) + except StopIteration: + self.output.put((None, None, None, None)) + return + retcode, stdout, stderr = run_tests_in_subprocess(test, self.ns) + # Strip last refcount output line if it exists, since it + # comes from the shutdown of the interpreter in the subcommand. + stderr = debug_output_pat.sub("", stderr) + stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + return + if not result: + self.output.put((None, None, None, None)) + return + result = json.loads(result) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + except BaseException: + self.output.put((None, None, None, None)) + raise + + +def run_tests_multiprocess(regrtest): + output = Queue() + pending = MultiprocessIterator(regrtest.tests) + + workers = [MultiprocessThread(pending, output, regrtest.ns) + for i in range(regrtest.ns.use_mp)] + for worker in workers: + worker.start() + finished = 0 + test_index = 1 + try: + while finished < regrtest.ns.use_mp: + test, stdout, stderr, result = output.get() + if test is None: + finished += 1 + continue + regrtest.accumulate_result(test, result) + regrtest.display_progress(test_index, test) + if stdout: + print(stdout) + if stderr: + print(stderr, file=sys.stderr) + sys.stdout.flush() + sys.stderr.flush() + if result[0] == INTERRUPTED: + raise KeyboardInterrupt + if result[0] == CHILD_ERROR: + raise Exception("Child error on {}: {}".format(test, result[1])) + test_index += 1 + except KeyboardInterrupt: + regrtest.interrupted = True + pending.interrupted = True + for worker in workers: + worker.join() -- cgit v1.2.1 From 74e84f1dec668e528701c56bc6c5644c8639d84a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:36:27 +0200 Subject: Issue #25220: Enhance regrtest --coverage Add a new Regrtest.run_test() method to ensure that --coverage pass the same options to the runtest() function. --- Lib/test/libregrtest/main.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index b66f045045..41c32d63c4 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -346,7 +346,18 @@ class Regrtest: print(count(len(self.bad), 'test'), "failed again:") printlist(self.bad) - def _run_tests_sequential(self): + def run_test(self, test): + result = runtest(test, + self.ns.verbose, + self.ns.quiet, + self.ns.huntrleaks, + output_on_failure=self.ns.verbose3, + timeout=self.ns.timeout, + failfast=self.ns.failfast, + match_tests=self.ns.match_tests) + self.accumulate_result(test, result) + + def run_tests_sequential(self): save_modules = sys.modules.keys() for test_index, test in enumerate(self.tests, 1): @@ -354,19 +365,15 @@ class Regrtest: if self.ns.trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - cmd = 'runtest(test, self.ns.verbose, self.ns.quiet, timeout=self.ns.timeout)' + cmd = 'self.run_test(test)' self.tracer.runctx(cmd, globals=globals(), locals=vars()) else: try: - result = runtest(test, self.ns.verbose, self.ns.quiet, - self.ns.huntrleaks, - output_on_failure=self.ns.verbose3, - timeout=self.ns.timeout, failfast=self.ns.failfast, - match_tests=self.ns.match_tests) - self.accumulate_result(test, result) + self.run_test(test) except KeyboardInterrupt: self.interrupted = True break + if self.ns.findleaks: gc.collect() if gc.garbage: @@ -376,13 +383,14 @@ class Regrtest: # them again self.found_garbage.extend(gc.garbage) del gc.garbage[:] + # Unload the newly imported modules (best effort finalization) for module in sys.modules.keys(): if module not in save_modules and module.startswith("test."): support.unload(module) def run_tests(self): - support.verbose = self.ns.verbose # Tell tests to be moderately quiet + support.verbose = self.ns.verbose # Tell tests to be moderately quiet support.use_resources = self.ns.use_resources if self.ns.forever: @@ -404,7 +412,7 @@ class Regrtest: from test.libregrtest.runtest_mp import run_tests_multiprocess run_tests_multiprocess(self) else: - self._run_tests_sequential() + self.run_tests_sequential() def finalize(self): if self.next_single_filename: -- cgit v1.2.1 From 03df0557d3e8656977e8bf28743e8381d14f8ab4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:37:14 +0200 Subject: Issue #25220: regrtest setups Python after parsing command line options --- Lib/test/libregrtest/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 41c32d63c4..2716536c4f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -431,8 +431,8 @@ class Regrtest: os.system("leaks %d" % os.getpid()) def main(self, tests=None, **kwargs): - setup_python() self.ns = _parse_args(sys.argv[1:], **kwargs) + setup_python() self.setup_regrtest() if self.ns.wait: input("Press any key to continue...") -- cgit v1.2.1 From 2d174ede71b499d0530d9eeafc63341a05d7488c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:43:33 +0200 Subject: Issue #25220: truncate some long lines in libregrtest/*.py --- Lib/test/libregrtest/main.py | 23 ++++++++++++++++------- Lib/test/libregrtest/runtest_mp.py | 12 ++++++++---- 2 files changed, 24 insertions(+), 11 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2716536c4f..ee1591df5f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -63,7 +63,8 @@ def setup_python(): # the packages to prevent later imports to fail when the CWD is different. for module in sys.modules.values(): if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) for path in module.__path__] + module.__path__ = [os.path.abspath(path) + for path in module.__path__] if hasattr(module, '__file__'): module.__file__ = os.path.abspath(module.__file__) @@ -159,8 +160,8 @@ class Regrtest: if self.ns.quiet: return fmt = "[{1:{0}}{2}/{3}] {4}" if self.bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - self.test_count_width, test_index, self.test_count, len(self.bad), test)) + print(fmt.format(self.test_count_width, test_index, + self.test_count, len(self.bad), test)) sys.stdout.flush() def setup_regrtest(self): @@ -252,7 +253,10 @@ class Regrtest: self.ns.args = [] # For a partial run, we do not need to clutter the output. - if self.ns.verbose or self.ns.header or not (self.ns.quiet or self.ns.single or self.tests or self.ns.args): + if (self.ns.verbose + or self.ns.header + or not (self.ns.quiet or self.ns.single + or self.tests or self.ns.args)): # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) print("== ", platform.platform(aliased=True), @@ -283,7 +287,8 @@ class Regrtest: try: del self.selected[:self.selected.index(self.ns.start)] except ValueError: - print("Couldn't find starting test (%s), using all tests" % self.ns.start) + print("Couldn't find starting test (%s), using all tests" + % self.ns.start) if self.ns.randomize: if self.ns.random_seed is None: @@ -297,12 +302,16 @@ class Regrtest: # print a newline after ^C print() print("Test suite interrupted by signal SIGINT.") - omitted = set(self.selected) - set(self.good) - set(self.bad) - set(self.skipped) + executed = set(self.good) | set(self.bad) | set(self.skipped) + omitted = set(self.selected) - executed print(count(len(omitted), "test"), "omitted:") printlist(omitted) if self.good and not self.ns.quiet: - if not self.bad and not self.skipped and not self.interrupted and len(self.good) > 1: + if (not self.bad + and not self.skipped + and not self.interrupted + and len(self.good) > 1): print("All", end=' ') print(count(len(self.good), "test"), "OK.") diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index b55c11afb5..c5a6b3dab5 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -103,20 +103,23 @@ class MultiprocessThread(threading.Thread): except StopIteration: self.output.put((None, None, None, None)) return - retcode, stdout, stderr = run_tests_in_subprocess(test, self.ns) + retcode, stdout, stderr = run_tests_in_subprocess(test, + self.ns) # Strip last refcount output line if it exists, since it # comes from the shutdown of the interpreter in the subcommand. stderr = debug_output_pat.sub("", stderr) stdout, _, result = stdout.strip().rpartition("\n") if retcode != 0: result = (CHILD_ERROR, "Exit code %s" % retcode) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) return if not result: self.output.put((None, None, None, None)) return result = json.loads(result) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) except BaseException: self.output.put((None, None, None, None)) raise @@ -149,7 +152,8 @@ def run_tests_multiprocess(regrtest): if result[0] == INTERRUPTED: raise KeyboardInterrupt if result[0] == CHILD_ERROR: - raise Exception("Child error on {}: {}".format(test, result[1])) + msg = "Child error on {}: {}".format(test, result[1]) + raise Exception(msg) test_index += 1 except KeyboardInterrupt: regrtest.interrupted = True -- cgit v1.2.1 From a87930f079ee932d16d97ca78f1a7fad9c03d0c6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:50:19 +0200 Subject: Issue #25220, libregrtest: Remove unused import --- Lib/test/libregrtest/main.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index ee1591df5f..306beb81fc 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -19,10 +19,6 @@ try: import gc except ImportError: gc = None -try: - import threading -except ImportError: - threading = None # When tests are run from the Python build directory, it is best practice -- cgit v1.2.1 From 3263ce558fcb5fdad4b2e6e990c9ded2a3c5c456 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:52:33 +0200 Subject: Don't strip refcount in libregrtest/runtest_mp.py Python doesn't display the refcount anymore by default. It only displays it when -X showrefcount command line option is used, which is not the case here. regrtest can be run with -X showrefcount, the option is not inherited by child processes. --- Lib/test/libregrtest/runtest_mp.py | 6 ------ 1 file changed, 6 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index c5a6b3dab5..8311ecf40c 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -15,9 +15,6 @@ except ImportError: from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR -debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") - - def run_tests_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. @@ -105,9 +102,6 @@ class MultiprocessThread(threading.Thread): return retcode, stdout, stderr = run_tests_in_subprocess(test, self.ns) - # Strip last refcount output line if it exists, since it - # comes from the shutdown of the interpreter in the subcommand. - stderr = debug_output_pat.sub("", stderr) stdout, _, result = stdout.strip().rpartition("\n") if retcode != 0: result = (CHILD_ERROR, "Exit code %s" % retcode) -- cgit v1.2.1 From 1e3161646f1a24926675dd51b5121ad5ccae9242 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 00:33:29 +0200 Subject: Issue #25220: Enhance regrtest -jN Running the Python test suite with -jN now: - Display the duration of tests which took longer than 30 seconds - Display the tests currently running since at least 30 seconds - Display the tests we are waiting for when the test suite is interrupted Clenaup also run_test_in_subprocess() code. --- Lib/test/libregrtest/runtest_mp.py | 127 ++++++++++++++++++++++++++----------- 1 file changed, 90 insertions(+), 37 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 8311ecf40c..71096d3590 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -1,7 +1,7 @@ import json import os -import re import sys +import time import traceback import unittest from queue import Queue @@ -15,7 +15,12 @@ except ImportError: from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR -def run_tests_in_subprocess(testname, ns): +# Minimum duration of a test to display its duration or to mention that +# the test is running in background +PROGRESS_MIN_TIME = 30.0 # seconds + + +def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. ns is the option Namespace parsed from command-line arguments. regrtest @@ -24,26 +29,33 @@ def run_tests_in_subprocess(testname, ns): 3-tuple. """ from subprocess import Popen, PIPE - base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + - ['-X', 'faulthandler', '-m', 'test.regrtest']) - - slaveargs = ( - (testname, ns.verbose, ns.quiet), - dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests)) + + args = (testname, ns.verbose, ns.quiet) + kwargs = dict(huntrleaks=ns.huntrleaks, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + timeout=ns.timeout, + failfast=ns.failfast, + match_tests=ns.match_tests) + slaveargs = (args, kwargs) + slaveargs = json.dumps(slaveargs) + + cmd = [sys.executable, *support.args_from_interpreter_flags(), + '-X', 'faulthandler', + '-m', 'test.regrtest', + '--slaveargs', slaveargs] + # Running the child from the same working directory as regrtest's original # invocation ensures that TEMPDIR for the child is the same when # sysconfig.is_python_build() is true. See issue 15300. - popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], + popen = Popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True, close_fds=(os.name != 'nt'), cwd=support.SAVEDCWD) - stdout, stderr = popen.communicate() - retcode = popen.wait() + with popen: + stdout, stderr = popen.communicate() + retcode = popen.wait() return retcode, stdout, stderr @@ -90,30 +102,45 @@ class MultiprocessThread(threading.Thread): self.pending = pending self.output = output self.ns = ns + self.current_test = None + self.start_time = None + + def _runtest(self): + try: + test = next(self.pending) + except StopIteration: + self.output.put((None, None, None, None)) + return True + + try: + self.start_time = time.monotonic() + self.current_test = test + + retcode, stdout, stderr = run_test_in_subprocess(test, self.ns) + finally: + self.current_test = None + + stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) + return True + + if not result: + self.output.put((None, None, None, None)) + return True + + result = json.loads(result) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) + return False def run(self): - # A worker thread. try: - while True: - try: - test = next(self.pending) - except StopIteration: - self.output.put((None, None, None, None)) - return - retcode, stdout, stderr = run_tests_in_subprocess(test, - self.ns) - stdout, _, result = stdout.strip().rpartition("\n") - if retcode != 0: - result = (CHILD_ERROR, "Exit code %s" % retcode) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), - result)) - return - if not result: - self.output.put((None, None, None, None)) - return - result = json.loads(result) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), - result)) + stop = False + while not stop: + stop = self._runtest() except BaseException: self.output.put((None, None, None, None)) raise @@ -136,13 +163,33 @@ def run_tests_multiprocess(regrtest): finished += 1 continue regrtest.accumulate_result(test, result) - regrtest.display_progress(test_index, test) + + # Display progress + text = test + ok, test_time = result + if (ok not in (CHILD_ERROR, INTERRUPTED) + and test_time >= PROGRESS_MIN_TIME): + text += ' (%.0f sec)' % test_time + running = [] + for worker in workers: + current_test = worker.current_test + if not current_test: + continue + dt = time.monotonic() - worker.start_time + if dt >= PROGRESS_MIN_TIME: + running.append('%s (%.0f sec)' % (current_test, dt)) + if running: + text += ' -- running: %s' % ', '.join(running) + regrtest.display_progress(test_index, text) + + # Copy stdout and stderr from the child process if stdout: print(stdout) if stderr: print(stderr, file=sys.stderr) sys.stdout.flush() sys.stderr.flush() + if result[0] == INTERRUPTED: raise KeyboardInterrupt if result[0] == CHILD_ERROR: @@ -152,5 +199,11 @@ def run_tests_multiprocess(regrtest): except KeyboardInterrupt: regrtest.interrupted = True pending.interrupted = True + print() + + running = [worker.current_test for worker in workers] + running = list(filter(bool, running)) + if running: + print("Waiting for %s" % ', '.join(running)) for worker in workers: worker.join() -- cgit v1.2.1 From db6cf4b087bf76a83fed76dbc5e1916602328fae Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 00:48:27 +0200 Subject: Issue #25220: Use print(flush=True) in libregrtest --- Lib/test/libregrtest/main.py | 7 +++---- Lib/test/libregrtest/refleak.py | 10 ++++------ Lib/test/libregrtest/runtest.py | 14 +++++--------- Lib/test/libregrtest/runtest_mp.py | 9 +++------ 4 files changed, 15 insertions(+), 25 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 306beb81fc..7440e9d60c 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -157,8 +157,8 @@ class Regrtest: return fmt = "[{1:{0}}{2}/{3}] {4}" if self.bad else "[{1:{0}}{2}] {4}" print(fmt.format(self.test_count_width, test_index, - self.test_count, len(self.bad), test)) - sys.stdout.flush() + self.test_count, len(self.bad), test), + flush=True) def setup_regrtest(self): if self.ns.huntrleaks: @@ -333,8 +333,7 @@ class Regrtest: if self.ns.verbose2 and self.bad: print("Re-running failed tests in verbose mode") for test in self.bad[:]: - print("Re-running test %r in verbose mode" % test) - sys.stdout.flush() + print("Re-running test %r in verbose mode" % test, flush=True) try: self.ns.verbose = True ok = runtest(test, True, self.ns.quiet, self.ns.huntrleaks, diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index dd0d05d3ec..db8a4455ca 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -44,13 +44,12 @@ def dash_R(the_module, test, indirect_test, huntrleaks): alloc_deltas = [0] * repcount print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr) - sys.stderr.flush() + print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, + flush=True) for i in range(repcount): indirect_test() alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) - sys.stderr.write('.') - sys.stderr.flush() + print('.', end='', flush=True) if i >= nwarmup: rc_deltas[i] = rc_after - rc_before alloc_deltas[i] = alloc_after - alloc_before @@ -74,8 +73,7 @@ def dash_R(the_module, test, indirect_test, huntrleaks): if checker(deltas): msg = '%s leaked %s %s, sum=%s' % ( test, deltas[nwarmup:], item_name, sum(deltas)) - print(msg, file=sys.stderr) - sys.stderr.flush() + print(msg, file=sys.stderr, flush=True) with open(fname, "a") as refrep: print(msg, file=refrep) refrep.flush() diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 60b8f5f538..f57784de65 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -161,27 +161,23 @@ def runtest_inner(test, verbose, quiet, test_time = time.time() - start_time except support.ResourceDenied as msg: if not quiet: - print(test, "skipped --", msg) - sys.stdout.flush() + print(test, "skipped --", msg, flush=True) return RESOURCE_DENIED, test_time except unittest.SkipTest as msg: if not quiet: - print(test, "skipped --", msg) - sys.stdout.flush() + print(test, "skipped --", msg, flush=True) return SKIPPED, test_time except KeyboardInterrupt: raise except support.TestFailed as msg: if display_failure: - print("test", test, "failed --", msg, file=sys.stderr) + print("test", test, "failed --", msg, file=sys.stderr, flush=True) else: - print("test", test, "failed", file=sys.stderr) - sys.stderr.flush() + print("test", test, "failed", file=sys.stderr, flush=True) return FAILED, test_time except: msg = traceback.format_exc() - print("test", test, "crashed --", msg, file=sys.stderr) - sys.stderr.flush() + print("test", test, "crashed --", msg, file=sys.stderr, flush=True) return FAILED, test_time else: if refleak: diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 71096d3590..1a82b3d257 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -70,9 +70,8 @@ def run_tests_slave(slaveargs): except BaseException as e: traceback.print_exc() result = CHILD_ERROR, str(e) - sys.stdout.flush() print() # Force a newline (just in case) - print(json.dumps(result)) + print(json.dumps(result), flush=True) sys.exit(0) @@ -184,11 +183,9 @@ def run_tests_multiprocess(regrtest): # Copy stdout and stderr from the child process if stdout: - print(stdout) + print(stdout, flush=True) if stderr: - print(stderr, file=sys.stderr) - sys.stdout.flush() - sys.stderr.flush() + print(stderr, file=sys.stderr, flush=True) if result[0] == INTERRUPTED: raise KeyboardInterrupt -- cgit v1.2.1 From 527ae93126214d46c02bcbc676827e73eb5bae88 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 00:59:35 +0200 Subject: Issue #25220, libregrtest: Cleanup setup code --- Lib/test/libregrtest/main.py | 96 ++++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 47 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 7440e9d60c..f410bfd418 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -31,7 +31,7 @@ else: TEMPDIR = os.path.abspath(TEMPDIR) -def setup_python(): +def setup_python(ns): # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) @@ -80,6 +80,38 @@ def setup_python(): newsoft = min(hard, max(soft, 1024*2048)) resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + if ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() + + if ns.memlimit is not None: + support.set_memlimit(ns.memlimit) + + if ns.threshold is not None: + if gc is not None: + gc.set_threshold(ns.threshold) + else: + print('No GC available, ignore --threshold.') + + if ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + class Regrtest: """Execute a test suite. @@ -161,36 +193,6 @@ class Regrtest: flush=True) def setup_regrtest(self): - if self.ns.huntrleaks: - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - - if self.ns.memlimit is not None: - support.set_memlimit(self.ns.memlimit) - - if self.ns.threshold is not None: - if gc is not None: - gc.set_threshold(self.ns.threshold) - else: - print('No GC available, ignore --threshold.') - - if self.ns.nowindows: - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - if self.ns.findleaks: if gc is not None: # Uncomment the line below to report garbage that is not @@ -202,19 +204,9 @@ class Regrtest: print('No GC available, disabling --findleaks') self.ns.findleaks = False - if self.ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False - # Strip .py extensions. removepy(self.ns.args) - if self.ns.trace: - import trace - self.tracer = trace.Trace(ignoredirs=[sys.base_prefix, - sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) - def find_tests(self, tests): self.tests = tests @@ -278,7 +270,7 @@ class Regrtest: except IndexError: pass - # Remove all the self.selected tests that precede start if it's set. + # Remove all the selected tests that precede start if it's set. if self.ns.start: try: del self.selected[:self.selected.index(self.ns.start)] @@ -362,11 +354,18 @@ class Regrtest: self.accumulate_result(test, result) def run_tests_sequential(self): + if self.ns.trace: + import trace + self.tracer = trace.Trace(ignoredirs=[sys.base_prefix, + sys.base_exec_prefix, + tempfile.gettempdir()], + trace=False, count=True) + save_modules = sys.modules.keys() for test_index, test in enumerate(self.tests, 1): self.display_progress(test_index, test) - if self.ns.trace: + if self.tracer: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. cmd = 'self.run_test(test)' @@ -426,7 +425,7 @@ class Regrtest: else: os.unlink(self.next_single_filename) - if self.ns.trace: + if self.tracer: r = self.tracer.results() r.write_results(show_missing=True, summary=True, coverdir=self.ns.coverdir) @@ -436,15 +435,18 @@ class Regrtest: def main(self, tests=None, **kwargs): self.ns = _parse_args(sys.argv[1:], **kwargs) - setup_python() + setup_python(self.ns) self.setup_regrtest() - if self.ns.wait: - input("Press any key to continue...") + if self.ns.slaveargs is not None: from test.libregrtest.runtest_mp import run_tests_slave run_tests_slave(self.ns.slaveargs) + if self.ns.wait: + input("Press any key to continue...") + self.find_tests(tests) self.run_tests() + self.display_result() self.finalize() sys.exit(len(self.bad) > 0 or self.interrupted) -- cgit v1.2.1 From b7830abbb5ec53f41002c80edd546232ebbbc3df Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 01:13:53 +0200 Subject: Issue #25220, libregrtest: Move setup_python() to a new submodule --- Lib/test/libregrtest/main.py | 125 +++++------------------------------------- Lib/test/libregrtest/setup.py | 108 ++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 110 deletions(-) create mode 100644 Lib/test/libregrtest/setup.py (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f410bfd418..fbbfa732af 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,19 +1,16 @@ -import faulthandler import os import platform import random import re -import signal import sys import sysconfig import tempfile import textwrap -import unittest from test.libregrtest.runtest import ( findtests, runtest, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) -from test.libregrtest.refleak import warm_caches from test.libregrtest.cmdline import _parse_args +from test.libregrtest.setup import setup_python from test import support try: import gc @@ -31,88 +28,6 @@ else: TEMPDIR = os.path.abspath(TEMPDIR) -def setup_python(ns): - # Display the Python traceback on fatal errors (e.g. segfault) - faulthandler.enable(all_threads=True) - - # Display the Python traceback on SIGALRM or SIGUSR1 signal - signals = [] - if hasattr(signal, 'SIGALRM'): - signals.append(signal.SIGALRM) - if hasattr(signal, 'SIGUSR1'): - signals.append(signal.SIGUSR1) - for signum in signals: - faulthandler.register(signum, chain=True) - - replace_stdout() - support.record_original_stdout(sys.stdout) - - # Some times __path__ and __file__ are not absolute (e.g. while running from - # Lib/) and, if we change the CWD to run the tests in a temporary dir, some - # imports might fail. This affects only the modules imported before os.chdir(). - # These modules are searched first in sys.path[0] (so '' -- the CWD) and if - # they are found in the CWD their __file__ and __path__ will be relative (this - # happens before the chdir). All the modules imported after the chdir, are - # not found in the CWD, and since the other paths in sys.path[1:] are absolute - # (site.py absolutize them), the __file__ and __path__ will be absolute too. - # Therefore it is necessary to absolutize manually the __file__ and __path__ of - # the packages to prevent later imports to fail when the CWD is different. - for module in sys.modules.values(): - if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) - for path in module.__path__] - if hasattr(module, '__file__'): - module.__file__ = os.path.abspath(module.__file__) - - # MacOSX (a.k.a. Darwin) has a default stack size that is too small - # for deeply recursive regular expressions. We see this as crashes in - # the Python test suite when running test_re.py and test_sre.py. The - # fix is to set the stack limit to 2048. - # This approach may also be useful for other Unixy platforms that - # suffer from small default stack limits. - if sys.platform == 'darwin': - try: - import resource - except ImportError: - pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) - - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False - - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) - - if ns.threshold is not None: - if gc is not None: - gc.set_threshold(ns.threshold) - else: - print('No GC available, ignore --threshold.') - - if ns.nowindows: - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - - class Regrtest: """Execute a test suite. @@ -192,8 +107,14 @@ class Regrtest: self.test_count, len(self.bad), test), flush=True) - def setup_regrtest(self): - if self.ns.findleaks: + def parse_args(self, kwargs): + ns = _parse_args(sys.argv[1:], **kwargs) + + if ns.threshold is not None and gc is None: + print('No GC available, ignore --threshold.') + ns.threshold = None + + if ns.findleaks: if gc is not None: # Uncomment the line below to report garbage that is not # freeable by reference counting alone. By default only @@ -202,10 +123,12 @@ class Regrtest: #gc.set_debug(gc.DEBUG_SAVEALL) else: print('No GC available, disabling --findleaks') - self.ns.findleaks = False + ns.findleaks = False # Strip .py extensions. - removepy(self.ns.args) + removepy(ns.args) + + return ns def find_tests(self, tests): self.tests = tests @@ -434,9 +357,9 @@ class Regrtest: os.system("leaks %d" % os.getpid()) def main(self, tests=None, **kwargs): - self.ns = _parse_args(sys.argv[1:], **kwargs) + self.ns = self.parse_args(kwargs) + setup_python(self.ns) - self.setup_regrtest() if self.ns.slaveargs is not None: from test.libregrtest.runtest_mp import run_tests_slave @@ -452,24 +375,6 @@ class Regrtest: sys.exit(len(self.bad) > 0 or self.interrupted) -def replace_stdout(): - """Set stdout encoder error handler to backslashreplace (as stderr error - handler) to avoid UnicodeEncodeError when printing a traceback""" - import atexit - - stdout = sys.stdout - sys.stdout = open(stdout.fileno(), 'w', - encoding=stdout.encoding, - errors="backslashreplace", - closefd=False, - newline='\n') - - def restore_stdout(): - sys.stdout.close() - sys.stdout = stdout - atexit.register(restore_stdout) - - def removepy(names): if not names: return diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py new file mode 100644 index 0000000000..a7dfa7947b --- /dev/null +++ b/Lib/test/libregrtest/setup.py @@ -0,0 +1,108 @@ +import atexit +import faulthandler +import os +import signal +import sys +import unittest +from test import support +try: + import gc +except ImportError: + gc = None + +from test.libregrtest.refleak import warm_caches + + +def setup_python(ns): + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + + replace_stdout() + support.record_original_stdout(sys.stdout) + + # Some times __path__ and __file__ are not absolute (e.g. while running from + # Lib/) and, if we change the CWD to run the tests in a temporary dir, some + # imports might fail. This affects only the modules imported before os.chdir(). + # These modules are searched first in sys.path[0] (so '' -- the CWD) and if + # they are found in the CWD their __file__ and __path__ will be relative (this + # happens before the chdir). All the modules imported after the chdir, are + # not found in the CWD, and since the other paths in sys.path[1:] are absolute + # (site.py absolutize them), the __file__ and __path__ will be absolute too. + # Therefore it is necessary to absolutize manually the __file__ and __path__ of + # the packages to prevent later imports to fail when the CWD is different. + for module in sys.modules.values(): + if hasattr(module, '__path__'): + module.__path__ = [os.path.abspath(path) + for path in module.__path__] + if hasattr(module, '__file__'): + module.__file__ = os.path.abspath(module.__file__) + + # MacOSX (a.k.a. Darwin) has a default stack size that is too small + # for deeply recursive regular expressions. We see this as crashes in + # the Python test suite when running test_re.py and test_sre.py. The + # fix is to set the stack limit to 2048. + # This approach may also be useful for other Unixy platforms that + # suffer from small default stack limits. + if sys.platform == 'darwin': + try: + import resource + except ImportError: + pass + else: + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + newsoft = min(hard, max(soft, 1024*2048)) + resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + + if ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() + + if ns.memlimit is not None: + support.set_memlimit(ns.memlimit) + + if ns.threshold is not None: + gc.set_threshold(ns.threshold) + + if ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + + +def replace_stdout(): + """Set stdout encoder error handler to backslashreplace (as stderr error + handler) to avoid UnicodeEncodeError when printing a traceback""" + stdout = sys.stdout + sys.stdout = open(stdout.fileno(), 'w', + encoding=stdout.encoding, + errors="backslashreplace", + closefd=False, + newline='\n') + + def restore_stdout(): + sys.stdout.close() + sys.stdout = stdout + atexit.register(restore_stdout) -- cgit v1.2.1 From 65a200830f2a737105f0b853a3bcebabcdd8c83a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 01:32:39 +0200 Subject: Issue #25220, libregrtest: Add runtest_ns() function * Factorize code to run tests. * run_test_in_subprocess() now pass the whole "ns" namespace to the child process. --- Lib/test/libregrtest/main.py | 17 ++++++----------- Lib/test/libregrtest/runtest.py | 7 +++++++ Lib/test/libregrtest/runtest_mp.py | 24 +++++++++++++----------- 3 files changed, 26 insertions(+), 22 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index fbbfa732af..df2329f2d3 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -7,7 +7,7 @@ import sysconfig import tempfile import textwrap from test.libregrtest.runtest import ( - findtests, runtest, + findtests, runtest_ns, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) from test.libregrtest.cmdline import _parse_args from test.libregrtest.setup import setup_python @@ -251,8 +251,7 @@ class Regrtest: print("Re-running test %r in verbose mode" % test, flush=True) try: self.ns.verbose = True - ok = runtest(test, True, self.ns.quiet, self.ns.huntrleaks, - timeout=self.ns.timeout) + ok = runtest_ns(test, True, self.ns) except KeyboardInterrupt: # print a newline separate from the ^C print() @@ -266,14 +265,10 @@ class Regrtest: printlist(self.bad) def run_test(self, test): - result = runtest(test, - self.ns.verbose, - self.ns.quiet, - self.ns.huntrleaks, - output_on_failure=self.ns.verbose3, - timeout=self.ns.timeout, - failfast=self.ns.failfast, - match_tests=self.ns.match_tests) + result = runtest_ns(test, self.ns.verbose, self.ns, + output_on_failure=self.ns.verbose3, + failfast=self.ns.failfast, + match_tests=self.ns.match_tests) self.accumulate_result(test, result) def run_tests_sequential(self): diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index f57784de65..fb7f82152a 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -53,6 +53,13 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): return stdtests + sorted(tests) +def runtest_ns(test, verbose, ns, **kw): + return runtest(test, verbose, ns.quiet, + huntrleaks=ns.huntrleaks, + timeout=ns.timeout, + **kw) + + def runtest(test, verbose, quiet, huntrleaks=False, use_resources=None, output_on_failure=False, failfast=False, match_tests=None, diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 1a82b3d257..74424c14ba 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -3,6 +3,7 @@ import os import sys import time import traceback +import types import unittest from queue import Queue from test import support @@ -30,14 +31,8 @@ def run_test_in_subprocess(testname, ns): """ from subprocess import Popen, PIPE - args = (testname, ns.verbose, ns.quiet) - kwargs = dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, - failfast=ns.failfast, - match_tests=ns.match_tests) - slaveargs = (args, kwargs) + ns_dict = vars(ns) + slaveargs = (ns_dict, testname) slaveargs = json.dumps(slaveargs) cmd = [sys.executable, *support.args_from_interpreter_flags(), @@ -60,11 +55,18 @@ def run_test_in_subprocess(testname, ns): def run_tests_slave(slaveargs): - args, kwargs = json.loads(slaveargs) - if kwargs.get('huntrleaks'): + ns_dict, testname = json.loads(slaveargs) + ns = types.SimpleNamespace(**ns_dict) + + if ns.huntrleaks: unittest.BaseTestSuite._cleanup = False + try: - result = runtest(*args, **kwargs) + result = runtest_ns(testname, ns.verbose, ns.quiet, ns, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + failfast=ns.failfast, + match_tests=ns.match_tests) except KeyboardInterrupt: result = INTERRUPTED, '' except BaseException as e: -- cgit v1.2.1 From d3c53bbb5b13c91407d7234f0cda6667a9cf5aa2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 01:39:28 +0200 Subject: Issue #25220, libregrtest: Call setup_python(ns) in the slaves Slaves (child processes running tests for regrtest -jN) now inherit --memlimit/-M, --threshold/-t and --nowindows/-n options. * -M, -t and -n are now supported with -jN * Factorize code to run tests. * run_test_in_subprocess() now pass the whole "ns" namespace to the child process. --- Lib/test/libregrtest/cmdline.py | 2 -- Lib/test/libregrtest/main.py | 5 +++-- Lib/test/libregrtest/runtest_mp.py | 8 ++++---- Lib/test/test_regrtest.py | 1 - 4 files changed, 7 insertions(+), 9 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 19c52a2fe6..e55e53f45e 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -295,8 +295,6 @@ def _parse_args(args, **kwargs): parser.error("-T and -j don't go together!") if ns.use_mp and ns.findleaks: parser.error("-l and -j don't go together!") - if ns.use_mp and ns.memlimit: - parser.error("-M and -j don't go together!") if ns.failfast and not (ns.verbose or ns.verbose3): parser.error("-G/--failfast needs either -v or -W") diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index df2329f2d3..e68597038e 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -354,14 +354,15 @@ class Regrtest: def main(self, tests=None, **kwargs): self.ns = self.parse_args(kwargs) - setup_python(self.ns) - if self.ns.slaveargs is not None: from test.libregrtest.runtest_mp import run_tests_slave run_tests_slave(self.ns.slaveargs) + if self.ns.wait: input("Press any key to continue...") + setup_python(self.ns) + self.find_tests(tests) self.run_tests() diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 74424c14ba..47393aad8d 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -13,7 +13,8 @@ except ImportError: print("Multiprocess option requires thread support") sys.exit(2) -from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR +from test.libregrtest.runtest import runtest_ns, INTERRUPTED, CHILD_ERROR +from test.libregrtest.setup import setup_python # Minimum duration of a test to display its duration or to mention that @@ -58,11 +59,10 @@ def run_tests_slave(slaveargs): ns_dict, testname = json.loads(slaveargs) ns = types.SimpleNamespace(**ns_dict) - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False + setup_python(ns) try: - result = runtest_ns(testname, ns.verbose, ns.quiet, ns, + result = runtest_ns(testname, ns.verbose, ns, use_resources=ns.use_resources, output_on_failure=ns.verbose3, failfast=ns.failfast, diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 54bbfe4cd7..c277e10e7e 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -214,7 +214,6 @@ class ParseArgsTestCase(unittest.TestCase): self.checkError([opt, 'foo'], 'invalid int value') self.checkError([opt, '2', '-T'], "don't go together") self.checkError([opt, '2', '-l'], "don't go together") - self.checkError([opt, '2', '-M', '4G'], "don't go together") def test_coverage(self): for opt in '-T', '--coverage': -- cgit v1.2.1 From 3753a949dde92e735769a4c9f3162cea174a9caf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 02:02:49 +0200 Subject: Issue #25274: Workaround test_sys crash just to keep buildbots usable --- Lib/test/test_sys.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 83549bc537..ccbf262af8 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -208,7 +208,10 @@ class SysModuleTest(unittest.TestCase): def f(): f() try: - for i in (50, 1000): + # FIXME: workaround crash for the issue #25274 + # FIXME: until the crash is fixed + #for i in (50, 1000): + for i in (150, 1000): # Issue #5392: stack overflow after hitting recursion limit twice sys.setrecursionlimit(i) self.assertRaises(RecursionError, f) -- cgit v1.2.1 From 40c23705e68b60325205a98ba87625dd9b62f756 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 02:17:28 +0200 Subject: Issue #25220, libregrtest: Set support.use_resources in setup_tests() * Rename setup_python() to setup_tests() * Remove use_resources parameter of runtest() --- Lib/test/libregrtest/main.py | 5 ++--- Lib/test/libregrtest/runtest.py | 5 +---- Lib/test/libregrtest/runtest_mp.py | 5 ++--- Lib/test/libregrtest/setup.py | 4 +++- 4 files changed, 8 insertions(+), 11 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e68597038e..9ca407a0c4 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -10,7 +10,7 @@ from test.libregrtest.runtest import ( findtests, runtest_ns, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) from test.libregrtest.cmdline import _parse_args -from test.libregrtest.setup import setup_python +from test.libregrtest.setup import setup_tests from test import support try: import gc @@ -312,7 +312,6 @@ class Regrtest: def run_tests(self): support.verbose = self.ns.verbose # Tell tests to be moderately quiet - support.use_resources = self.ns.use_resources if self.ns.forever: def test_forever(tests): @@ -361,7 +360,7 @@ class Regrtest: if self.ns.wait: input("Press any key to continue...") - setup_python(self.ns) + setup_tests(self.ns) self.find_tests(tests) self.run_tests() diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index fb7f82152a..a3d4e795f6 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -61,7 +61,7 @@ def runtest_ns(test, verbose, ns, **kw): def runtest(test, verbose, quiet, - huntrleaks=False, use_resources=None, + huntrleaks=False, output_on_failure=False, failfast=False, match_tests=None, timeout=None): """Run a single test. @@ -71,7 +71,6 @@ def runtest(test, verbose, quiet, quiet -- if true, don't print 'skipped' messages (probably redundant) huntrleaks -- run multiple times to test for leaks; requires a debug build; a triple corresponding to -R's three arguments - use_resources -- list of extra resources to use output_on_failure -- if true, display test output on failure timeout -- dump the traceback and exit if a test takes more than timeout seconds @@ -86,8 +85,6 @@ def runtest(test, verbose, quiet, PASSED test passed """ - if use_resources is not None: - support.use_resources = use_resources use_timeout = (timeout is not None) if use_timeout: faulthandler.dump_traceback_later(timeout, exit=True) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 47393aad8d..8332a0b46f 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -14,7 +14,7 @@ except ImportError: sys.exit(2) from test.libregrtest.runtest import runtest_ns, INTERRUPTED, CHILD_ERROR -from test.libregrtest.setup import setup_python +from test.libregrtest.setup import setup_tests # Minimum duration of a test to display its duration or to mention that @@ -59,11 +59,10 @@ def run_tests_slave(slaveargs): ns_dict, testname = json.loads(slaveargs) ns = types.SimpleNamespace(**ns_dict) - setup_python(ns) + setup_tests(ns) try: result = runtest_ns(testname, ns.verbose, ns, - use_resources=ns.use_resources, output_on_failure=ns.verbose3, failfast=ns.failfast, match_tests=ns.match_tests) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index a7dfa7947b..6a1c308ecf 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -13,7 +13,7 @@ except ImportError: from test.libregrtest.refleak import warm_caches -def setup_python(ns): +def setup_tests(ns): # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) @@ -91,6 +91,8 @@ def setup_python(ns): msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + support.use_resources = ns.use_resources + def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error -- cgit v1.2.1 From 358652cdd4ce9d5b9dc34a80562f3bc44f1bf8ee Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 02:32:11 +0200 Subject: Issue #25220, libregrtest: Pass directly ns to runtest() * Remove runtest_ns(): pass directly ns to runtest(). * Create also Regrtest.rerun_failed_tests() method. * Inline again Regrtest.run_test(): it's no more justified to have a method --- Lib/test/libregrtest/main.py | 62 ++++++++++++++++++++------------------ Lib/test/libregrtest/runtest.py | 20 ++++++------ Lib/test/libregrtest/runtest_mp.py | 7 ++--- 3 files changed, 44 insertions(+), 45 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9ca407a0c4..c102ee0e65 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -7,7 +7,7 @@ import sysconfig import tempfile import textwrap from test.libregrtest.runtest import ( - findtests, runtest_ns, + findtests, runtest, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) from test.libregrtest.cmdline import _parse_args from test.libregrtest.setup import setup_tests @@ -208,6 +208,30 @@ class Regrtest: print("Using random seed", self.ns.random_seed) random.shuffle(self.selected) + def rerun_failed_tests(self): + self.ns.verbose = True + self.ns.failfast = False + self.ns.verbose3 = False + self.ns.match_tests = None + + print("Re-running failed tests in verbose mode") + for test in self.bad[:]: + print("Re-running test %r in verbose mode" % test, flush=True) + try: + self.ns.verbose = True + ok = runtest(self.ns, test) + except KeyboardInterrupt: + # print a newline separate from the ^C + print() + break + else: + if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: + self.bad.remove(test) + else: + if self.bad: + print(count(len(self.bad), 'test'), "failed again:") + printlist(self.bad) + def display_result(self): if self.interrupted: # print a newline after ^C @@ -245,32 +269,6 @@ class Regrtest: print(count(len(self.skipped), "test"), "skipped:") printlist(self.skipped) - if self.ns.verbose2 and self.bad: - print("Re-running failed tests in verbose mode") - for test in self.bad[:]: - print("Re-running test %r in verbose mode" % test, flush=True) - try: - self.ns.verbose = True - ok = runtest_ns(test, True, self.ns) - except KeyboardInterrupt: - # print a newline separate from the ^C - print() - break - else: - if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: - self.bad.remove(test) - else: - if self.bad: - print(count(len(self.bad), 'test'), "failed again:") - printlist(self.bad) - - def run_test(self, test): - result = runtest_ns(test, self.ns.verbose, self.ns, - output_on_failure=self.ns.verbose3, - failfast=self.ns.failfast, - match_tests=self.ns.match_tests) - self.accumulate_result(test, result) - def run_tests_sequential(self): if self.ns.trace: import trace @@ -286,11 +284,13 @@ class Regrtest: if self.tracer: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - cmd = 'self.run_test(test)' + cmd = ('result = runtest(self.ns, test); ' + 'self.accumulate_result(test, result)') self.tracer.runctx(cmd, globals=globals(), locals=vars()) else: try: - self.run_test(test) + result = runtest(self.ns, test) + self.accumulate_result(test, result) except KeyboardInterrupt: self.interrupted = True break @@ -366,6 +366,10 @@ class Regrtest: self.run_tests() self.display_result() + + if self.ns.verbose2 and self.bad: + self.rerun_failed_tests() + self.finalize() sys.exit(len(self.bad) > 0 or self.interrupted) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index a3d4e795f6..4cc2588a6a 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -53,17 +53,7 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): return stdtests + sorted(tests) -def runtest_ns(test, verbose, ns, **kw): - return runtest(test, verbose, ns.quiet, - huntrleaks=ns.huntrleaks, - timeout=ns.timeout, - **kw) - - -def runtest(test, verbose, quiet, - huntrleaks=False, - output_on_failure=False, failfast=False, match_tests=None, - timeout=None): +def runtest(ns, test): """Run a single test. test -- the name of the test @@ -85,6 +75,14 @@ def runtest(test, verbose, quiet, PASSED test passed """ + verbose = ns.verbose + quiet = ns.quiet + huntrleaks = ns.huntrleaks + output_on_failure = ns.verbose3 + failfast = ns.failfast + match_tests = ns.match_tests + timeout = ns.timeout + use_timeout = (timeout is not None) if use_timeout: faulthandler.dump_traceback_later(timeout, exit=True) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 8332a0b46f..df075c101f 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -13,7 +13,7 @@ except ImportError: print("Multiprocess option requires thread support") sys.exit(2) -from test.libregrtest.runtest import runtest_ns, INTERRUPTED, CHILD_ERROR +from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR from test.libregrtest.setup import setup_tests @@ -62,10 +62,7 @@ def run_tests_slave(slaveargs): setup_tests(ns) try: - result = runtest_ns(testname, ns.verbose, ns, - output_on_failure=ns.verbose3, - failfast=ns.failfast, - match_tests=ns.match_tests) + result = runtest(ns, testname) except KeyboardInterrupt: result = INTERRUPTED, '' except BaseException as e: -- cgit v1.2.1 From 06aa9599e0fba87fe063dad5f0e754f66ee1423a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 02:39:22 +0200 Subject: Issue #25220, libregrtest: Cleanup No need to support.verbose in Regrtest.run_tests(), it's always set in runtest(). --- Lib/test/libregrtest/main.py | 17 ++++++++--------- Lib/test/libregrtest/runtest_mp.py | 1 + 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index c102ee0e65..fdb925d59d 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -310,17 +310,16 @@ class Regrtest: if module not in save_modules and module.startswith("test."): support.unload(module) - def run_tests(self): - support.verbose = self.ns.verbose # Tell tests to be moderately quiet + def _test_forever(self, tests): + while True: + for test in tests: + yield test + if self.bad: + return + def run_tests(self): if self.ns.forever: - def test_forever(tests): - while True: - for test in tests: - yield test - if self.bad: - return - self.tests = test_forever(list(self.selected)) + self.tests = _test_forever(list(self.selected)) self.test_count = '' self.test_count_width = 3 else: diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index df075c101f..6ed8dc2538 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -68,6 +68,7 @@ def run_tests_slave(slaveargs): except BaseException as e: traceback.print_exc() result = CHILD_ERROR, str(e) + print() # Force a newline (just in case) print(json.dumps(result), flush=True) sys.exit(0) -- cgit v1.2.1 From 309b0bd8454aff1f30e14283a7078dafaf20d8f0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 03:05:43 +0200 Subject: Issue #25220, libregrtest: more verbose output for -jN When the -jN command line option is used, display tests running since at least 30 seconds every minute. --- Lib/test/libregrtest/runtest_mp.py | 39 +++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 6ed8dc2538..b31b51e142 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -1,11 +1,11 @@ import json import os +import queue import sys import time import traceback import types import unittest -from queue import Queue from test import support try: import threading @@ -21,6 +21,9 @@ from test.libregrtest.setup import setup_tests # the test is running in background PROGRESS_MIN_TIME = 30.0 # seconds +# Display the running tests if nothing happened last N seconds +PROGRESS_UPDATE = 60.0 # seconds + def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. @@ -145,18 +148,39 @@ class MultiprocessThread(threading.Thread): def run_tests_multiprocess(regrtest): - output = Queue() + output = queue.Queue() pending = MultiprocessIterator(regrtest.tests) workers = [MultiprocessThread(pending, output, regrtest.ns) for i in range(regrtest.ns.use_mp)] for worker in workers: worker.start() + + def get_running(workers): + running = [] + for worker in workers: + current_test = worker.current_test + if not current_test: + continue + dt = time.monotonic() - worker.start_time + if dt >= PROGRESS_MIN_TIME: + running.append('%s (%.0f sec)' % (current_test, dt)) + return running + finished = 0 test_index = 1 + timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME) try: while finished < regrtest.ns.use_mp: - test, stdout, stderr, result = output.get() + try: + item = output.get(timeout=PROGRESS_UPDATE) + except queue.Empty: + running = get_running(workers) + if running: + print('running: %s' % ', '.join(running)) + continue + + test, stdout, stderr, result = item if test is None: finished += 1 continue @@ -168,14 +192,7 @@ def run_tests_multiprocess(regrtest): if (ok not in (CHILD_ERROR, INTERRUPTED) and test_time >= PROGRESS_MIN_TIME): text += ' (%.0f sec)' % test_time - running = [] - for worker in workers: - current_test = worker.current_test - if not current_test: - continue - dt = time.monotonic() - worker.start_time - if dt >= PROGRESS_MIN_TIME: - running.append('%s (%.0f sec)' % (current_test, dt)) + running = get_running(workers) if running: text += ' -- running: %s' % ', '.join(running) regrtest.display_progress(test_index, text) -- cgit v1.2.1 From 365f0fc04d6fe1b65c038a7bf71c6e5f3aae218a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 13:51:17 +0200 Subject: Issue #25220: Fix "-m test --forever" * Fix "-m test --forever": replace _test_forever() with self._test_forever() * Add unit test for --forever * Add unit test for a failing test * Fix also some pyflakes warnings in libregrtest --- Lib/test/libregrtest/main.py | 2 +- Lib/test/libregrtest/refleak.py | 6 +- Lib/test/libregrtest/runtest_mp.py | 3 +- Lib/test/test_regrtest.py | 133 ++++++++++++++++++++++++++----------- 4 files changed, 100 insertions(+), 44 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index fdb925d59d..e1a99fb50b 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -319,7 +319,7 @@ class Regrtest: def run_tests(self): if self.ns.forever: - self.tests = _test_forever(list(self.selected)) + self.tests = self._test_forever(list(self.selected)) self.test_count = '' self.test_count_width = 3 else: diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index db8a4455ca..9be0dec4d1 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -46,6 +46,8 @@ def dash_R(the_module, test, indirect_test, huntrleaks): print("beginning", repcount, "repetitions", file=sys.stderr) print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, flush=True) + # initialize variables to make pyflakes quiet + rc_before = alloc_before = 0 for i in range(repcount): indirect_test() alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) @@ -158,6 +160,6 @@ def warm_caches(): for i in range(256): s[i:i+1] # unicode cache - x = [chr(i) for i in range(256)] + [chr(i) for i in range(256)] # int cache - x = list(range(-5, 257)) + list(range(-5, 257)) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index b31b51e142..a732d705b1 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -5,7 +5,6 @@ import sys import time import traceback import types -import unittest from test import support try: import threading @@ -173,7 +172,7 @@ def run_tests_multiprocess(regrtest): try: while finished < regrtest.ns.use_mp: try: - item = output.get(timeout=PROGRESS_UPDATE) + item = output.get(timeout=timeout) except queue.Empty: running = get_running(workers) if running: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c277e10e7e..897598dddc 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -330,33 +330,52 @@ class BaseTestCase(unittest.TestCase): self.assertRegex(output, regex) def parse_executed_tests(self, output): - parser = re.finditer(r'^\[[0-9]+/[0-9]+\] (%s)$' % self.TESTNAME_REGEX, - output, - re.MULTILINE) - return set(match.group(1) for match in parser) + regex = r'^\[ *[0-9]+(?:/ *[0-9]+)?\] (%s)$' % self.TESTNAME_REGEX + parser = re.finditer(regex, output, re.MULTILINE) + return list(match.group(1) for match in parser) - def check_executed_tests(self, output, tests, skipped=None): + def check_executed_tests(self, output, tests, skipped=(), failed=(), + randomize=False): if isinstance(tests, str): tests = [tests] - executed = self.parse_executed_tests(output) - self.assertEqual(executed, set(tests), output) + if isinstance(skipped, str): + skipped = [skipped] + if isinstance(failed, str): + failed = [failed] ntest = len(tests) - if skipped: - if isinstance(skipped, str): - skipped = [skipped] - nskipped = len(skipped) - - plural = 's' if nskipped != 1 else '' - names = ' '.join(sorted(skipped)) - expected = (r'%s test%s skipped:\n %s$' - % (nskipped, plural, names)) - self.check_line(output, expected) - - ok = ntest - nskipped - if ok: - self.check_line(output, r'%s test OK\.$' % ok) + nskipped = len(skipped) + nfailed = len(failed) + + executed = self.parse_executed_tests(output) + if randomize: + self.assertEqual(set(executed), set(tests), output) else: - self.check_line(output, r'All %s tests OK\.$' % ntest) + self.assertEqual(executed, tests, output) + + def plural(count): + return 's' if count != 1 else '' + + def list_regex(line_format, tests): + count = len(tests) + names = ' '.join(sorted(tests)) + regex = line_format % (count, plural(count)) + regex = r'%s:\n %s$' % (regex, names) + return regex + + if skipped: + regex = list_regex('%s test%s skipped', skipped) + self.check_line(output, regex) + + if failed: + regex = list_regex('%s test%s failed', failed) + self.check_line(output, regex) + + good = ntest - nskipped - nfailed + if good: + regex = r'%s test%s OK\.$' % (good, plural(good)) + if not skipped and not failed and good > 1: + regex = 'All %s' % regex + self.check_line(output, regex) def parse_random_seed(self, output): match = self.regex_search(r'Using random seed ([0-9]+)', output) @@ -364,24 +383,28 @@ class BaseTestCase(unittest.TestCase): self.assertTrue(0 <= randseed <= 10000000, randseed) return randseed - def run_command(self, args, input=None): + def run_command(self, args, input=None, exitcode=0): if not input: input = '' - try: - return subprocess.run(args, - check=True, universal_newlines=True, - input=input, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except subprocess.CalledProcessError as exc: - self.fail("%s\n" + proc = subprocess.run(args, + universal_newlines=True, + input=input, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if proc.returncode != exitcode: + self.fail("Command %s failed with exit code %s\n" "\n" "stdout:\n" + "---\n" "%s\n" + "---\n" "\n" "stderr:\n" + "---\n" "%s" - % (str(exc), exc.stdout, exc.stderr)) + "---\n" + % (str(args), proc.returncode, proc.stdout, proc.stderr)) + return proc def run_python(self, args, **kw): @@ -411,11 +434,11 @@ class ProgramsTestCase(BaseTestCase): def check_output(self, output): self.parse_random_seed(output) - self.check_executed_tests(output, self.tests) + self.check_executed_tests(output, self.tests, randomize=True) def run_tests(self, args): - stdout = self.run_python(args) - self.check_output(stdout) + output = self.run_python(args) + self.check_output(output) def test_script_regrtest(self): # Lib/test/regrtest.py @@ -492,8 +515,24 @@ class ArgsTestCase(BaseTestCase): Test arguments of the Python test suite. """ - def run_tests(self, *args, input=None): - return self.run_python(['-m', 'test', *args], input=input) + def run_tests(self, *args, **kw): + return self.run_python(['-m', 'test', *args], **kw) + + def test_failing_test(self): + # test a failing test + code = textwrap.dedent(""" + import unittest + + class FailingTest(unittest.TestCase): + def test_failing(self): + self.fail("bug") + """) + test_ok = self.create_test() + test_failing = self.create_test(code=code) + tests = [test_ok, test_failing] + + output = self.run_tests(*tests, exitcode=1) + self.check_executed_tests(output, tests, failed=test_failing) def test_resources(self): # test -u command line option @@ -572,8 +611,7 @@ class ArgsTestCase(BaseTestCase): # test --coverage test = self.create_test() output = self.run_tests("--coverage", test) - executed = self.parse_executed_tests(output) - self.assertEqual(executed, {test}, output) + self.check_executed_tests(output, [test]) regex = ('lines +cov% +module +\(path\)\n' '(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') self.check_line(output, regex) @@ -584,6 +622,23 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests("--wait", test, input='key') self.check_line(output, 'Press any key to continue') + def test_forever(self): + # test --forever + code = textwrap.dedent(""" + import unittest + + class ForeverTester(unittest.TestCase): + RUN = 1 + + def test_run(self): + ForeverTester.RUN += 1 + if ForeverTester.RUN > 3: + self.fail("fail at the 3rd runs") + """) + test = self.create_test(code=code) + output = self.run_tests('--forever', test, exitcode=1) + self.check_executed_tests(output, [test]*3, failed=test) + if __name__ == '__main__': unittest.main() -- cgit v1.2.1 From 858695274ecb51f824d0046e7ab824d61aa9cc2e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 00:53:09 +0200 Subject: Fix regrtest --coverage on Windows Issue #25260: Fix ``python -m test --coverage`` on Windows. Remove the list of ignored directories. --- Lib/test/libregrtest/main.py | 5 +---- Lib/test/test_regrtest.py | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e1a99fb50b..c1ce3b179f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -272,10 +272,7 @@ class Regrtest: def run_tests_sequential(self): if self.ns.trace: import trace - self.tracer = trace.Trace(ignoredirs=[sys.base_prefix, - sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) + self.tracer = trace.Trace(trace=False, count=True) save_modules = sys.modules.keys() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 897598dddc..0f5f22ca4e 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -605,8 +605,6 @@ class ArgsTestCase(BaseTestCase): % (self.TESTNAME_REGEX, len(tests))) self.check_line(output, regex) - @unittest.skipIf(sys.platform == 'win32', - "FIXME: coverage doesn't work on Windows") def test_coverage(self): # test --coverage test = self.create_test() -- cgit v1.2.1 From af2ed36adce9af14cac8a111146b39bcfa0dab1b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 13:16:43 +0200 Subject: Issue #25277: Set a timeout of 10 minutes in test_eintr using faulthandler to try to debug a hang on the FreeBSD 9 buildbot. Run also eintr_tester.py with python "-u" command line option to try to get the full output on hang/crash. --- Lib/test/eintrdata/eintr_tester.py | 5 +++++ Lib/test/test_eintr.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index e1e0d91006..443ccd5433 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -9,6 +9,7 @@ sub-second periodicity (contrarily to signal()). """ import contextlib +import faulthandler import io import os import select @@ -50,6 +51,9 @@ class EINTRBaseTest(unittest.TestCase): signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) + # Issue #25277: Use faulthandler to try to debug a hang on FreeBSD + faulthandler.dump_traceback_later(10 * 60, exit=True) + @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) @@ -58,6 +62,7 @@ class EINTRBaseTest(unittest.TestCase): def tearDownClass(cls): cls.stop_alarm() signal.signal(signal.SIGALRM, cls.orig_handler) + faulthandler.cancel_dump_traceback_later() @classmethod def _sleep(cls): diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index d3cdda04e6..75452f2d41 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -16,7 +16,8 @@ class EINTRTests(unittest.TestCase): # Run the tester in a sub-process, to make sure there is only one # thread (for reliable signal delivery). tester = support.findfile("eintr_tester.py", subdir="eintrdata") - script_helper.assert_python_ok(tester) + # use -u to try to get the full output if the test hangs or crash + script_helper.assert_python_ok("-u", tester) if __name__ == "__main__": -- cgit v1.2.1 From 9c8cbea8d118fe73b5233376b1acb6c9ff981e95 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Thu, 1 Oct 2015 20:54:41 +0100 Subject: Closes #24884: refactored WatchedFileHandler file reopening into a separate method, based on a suggestion and patch by Marian Horban. --- Lib/logging/handlers.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 02a5fc1283..54bee893e1 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -440,11 +440,11 @@ class WatchedFileHandler(logging.FileHandler): sres = os.fstat(self.stream.fileno()) self.dev, self.ino = sres[ST_DEV], sres[ST_INO] - def emit(self, record): + def reopenIfNeeded(self): """ - Emit a record. + Reopen log file if needed. - First check if the underlying file has changed, and if it + Checks if the underlying file has changed, and if it has, close the old stream and reopen the file to get the current stream. """ @@ -467,6 +467,15 @@ class WatchedFileHandler(logging.FileHandler): # open a new file handle and get new stat info from that fd self.stream = self._open() self._statstream() + + def emit(self, record): + """ + Emit a record. + + If underlying file has changed, reopen the file before emitting the + record to it. + """ + self.reopenIfNeeded() logging.FileHandler.emit(self, record) -- cgit v1.2.1 From 82fe9794ce85e83409b0a9be7500b47c6bd0fc9a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 21:54:51 +0200 Subject: Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``. Patch co-written with Serhiy Storchaka. --- Lib/test/test_codecs.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 254c0c1d64..4658497aef 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -361,6 +361,12 @@ class ReadTest(MixInCheckStateHandling): self.assertEqual("[\uDC80]".encode(self.encoding, "replace"), "[?]".encode(self.encoding)) + # sequential surrogate characters + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "ignore"), + "[]".encode(self.encoding)) + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "replace"), + "[??]".encode(self.encoding)) + bom = "".encode(self.encoding) for before, after in [("\U00010fff", "A"), ("[", "]"), ("A", "\U00010fff")]: @@ -753,6 +759,7 @@ class UTF8Test(ReadTest, unittest.TestCase): encoding = "utf-8" ill_formed_sequence = b"\xed\xb2\x80" ill_formed_sequence_replace = "\ufffd" * 3 + BOM = b'' def test_partial(self): self.check_partial( @@ -785,23 +792,32 @@ class UTF8Test(ReadTest, unittest.TestCase): super().test_lone_surrogates() # not sure if this is making sense for # UTF-16 and UTF-32 - self.assertEqual("[\uDC80]".encode('utf-8', "surrogateescape"), - b'[\x80]') + self.assertEqual("[\uDC80]".encode(self.encoding, "surrogateescape"), + self.BOM + b'[\x80]') + + with self.assertRaises(UnicodeEncodeError) as cm: + "[\uDC80\uD800\uDFFF]".encode(self.encoding, "surrogateescape") + exc = cm.exception + self.assertEqual(exc.object[exc.start:exc.end], '\uD800\uDFFF') def test_surrogatepass_handler(self): - self.assertEqual("abc\ud800def".encode("utf-8", "surrogatepass"), - b"abc\xed\xa0\x80def") - self.assertEqual(b"abc\xed\xa0\x80def".decode("utf-8", "surrogatepass"), + self.assertEqual("abc\ud800def".encode(self.encoding, "surrogatepass"), + self.BOM + b"abc\xed\xa0\x80def") + self.assertEqual("\U00010fff\uD800".encode(self.encoding, "surrogatepass"), + self.BOM + b"\xf0\x90\xbf\xbf\xed\xa0\x80") + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "surrogatepass"), + self.BOM + b'[\xed\xa0\x80\xed\xb2\x80]') + + self.assertEqual(b"abc\xed\xa0\x80def".decode(self.encoding, "surrogatepass"), "abc\ud800def") - self.assertEqual("\U00010fff\uD800".encode("utf-8", "surrogatepass"), - b"\xf0\x90\xbf\xbf\xed\xa0\x80") - self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("utf-8", "surrogatepass"), + self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode(self.encoding, "surrogatepass"), "\U00010fff\uD800") + self.assertTrue(codecs.lookup_error("surrogatepass")) with self.assertRaises(UnicodeDecodeError): - b"abc\xed\xa0".decode("utf-8", "surrogatepass") + b"abc\xed\xa0".decode(self.encoding, "surrogatepass") with self.assertRaises(UnicodeDecodeError): - b"abc\xed\xa0z".decode("utf-8", "surrogatepass") + b"abc\xed\xa0z".decode(self.encoding, "surrogatepass") @unittest.skipUnless(sys.platform == 'win32', @@ -1008,6 +1024,7 @@ class ReadBufferTest(unittest.TestCase): class UTF8SigTest(UTF8Test, unittest.TestCase): encoding = "utf-8-sig" + BOM = codecs.BOM_UTF8 def test_partial(self): self.check_partial( -- cgit v1.2.1 From 49a1338ed1e8e4b0932b4b39f7bc9457f548703b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 2 Oct 2015 23:00:39 +0200 Subject: Issue #25287: Don't add crypt.METHOD_CRYPT to crypt.methods if it's not supported. Check if it is supported, it may not be supported on OpenBSD for example. --- Lib/crypt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/crypt.py b/Lib/crypt.py index 49ab96e140..fbc5f4cc35 100644 --- a/Lib/crypt.py +++ b/Lib/crypt.py @@ -54,9 +54,8 @@ METHOD_SHA256 = _Method('SHA256', '5', 16, 63) METHOD_SHA512 = _Method('SHA512', '6', 16, 106) methods = [] -for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5): +for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5, METHOD_CRYPT): _result = crypt('', _method) if _result and len(_result) == _method.total_size: methods.append(_method) -methods.append(METHOD_CRYPT) del _result, _method -- cgit v1.2.1 From 9fea91f1a1006e9667041eb02baf6c9bb99f7648 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 00:20:56 +0200 Subject: Issue #18174: "python -m test --huntrleaks ..." now also checks for leak of file descriptors. Patch written by Richard Oudkerk. --- Lib/test/libregrtest/refleak.py | 47 ++++++++++++++++++++++++++---- Lib/test/test_regrtest.py | 63 ++++++++++++++++++++++++++++++++--------- 2 files changed, 91 insertions(+), 19 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 9be0dec4d1..59dc49fe77 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -1,3 +1,4 @@ +import errno import os import re import sys @@ -6,6 +7,36 @@ from inspect import isabstract from test import support +try: + MAXFD = os.sysconf("SC_OPEN_MAX") +except Exception: + MAXFD = 256 + + +def fd_count(): + """Count the number of open file descriptors""" + if sys.platform.startswith(('linux', 'freebsd')): + try: + names = os.listdir("/proc/self/fd") + return len(names) + except FileNotFoundError: + pass + + count = 0 + for fd in range(MAXFD): + try: + # Prefer dup() over fstat(). fstat() can require input/output + # whereas dup() doesn't. + fd2 = os.dup(fd) + except OSError as e: + if e.errno != errno.EBADF: + raise + else: + os.close(fd2) + count += 1 + return count + + def dash_R(the_module, test, indirect_test, huntrleaks): """Run a test multiple times, looking for reference leaks. @@ -42,20 +73,25 @@ def dash_R(the_module, test, indirect_test, huntrleaks): repcount = nwarmup + ntracked rc_deltas = [0] * repcount alloc_deltas = [0] * repcount + fd_deltas = [0] * repcount print("beginning", repcount, "repetitions", file=sys.stderr) print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, flush=True) # initialize variables to make pyflakes quiet - rc_before = alloc_before = 0 + rc_before = alloc_before = fd_before = 0 for i in range(repcount): indirect_test() - alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) + alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc, + abcs) print('.', end='', flush=True) if i >= nwarmup: rc_deltas[i] = rc_after - rc_before alloc_deltas[i] = alloc_after - alloc_before - alloc_before, rc_before = alloc_after, rc_after + fd_deltas[i] = fd_after - fd_before + alloc_before = alloc_after + rc_before = rc_after + fd_before = fd_after print(file=sys.stderr) # These checkers return False on success, True on failure def check_rc_deltas(deltas): @@ -71,7 +107,8 @@ def dash_R(the_module, test, indirect_test, huntrleaks): failed = False for deltas, item_name, checker in [ (rc_deltas, 'references', check_rc_deltas), - (alloc_deltas, 'memory blocks', check_alloc_deltas)]: + (alloc_deltas, 'memory blocks', check_alloc_deltas), + (fd_deltas, 'file descriptors', check_rc_deltas)]: if checker(deltas): msg = '%s leaked %s %s, sum=%s' % ( test, deltas[nwarmup:], item_name, sum(deltas)) @@ -151,7 +188,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): func1 = sys.getallocatedblocks func2 = sys.gettotalrefcount gc.collect() - return func1(), func2() + return func1(), func2(), fd_count() def warm_caches(): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0f5f22ca4e..8b2449a0f5 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -383,27 +383,32 @@ class BaseTestCase(unittest.TestCase): self.assertTrue(0 <= randseed <= 10000000, randseed) return randseed - def run_command(self, args, input=None, exitcode=0): + def run_command(self, args, input=None, exitcode=0, **kw): if not input: input = '' + if 'stderr' not in kw: + kw['stderr'] = subprocess.PIPE proc = subprocess.run(args, universal_newlines=True, input=input, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + **kw) if proc.returncode != exitcode: - self.fail("Command %s failed with exit code %s\n" - "\n" - "stdout:\n" - "---\n" - "%s\n" - "---\n" - "\n" - "stderr:\n" - "---\n" - "%s" - "---\n" - % (str(args), proc.returncode, proc.stdout, proc.stderr)) + msg = ("Command %s failed with exit code %s\n" + "\n" + "stdout:\n" + "---\n" + "%s\n" + "---\n" + % (str(args), proc.returncode, proc.stdout)) + if proc.stderr: + msg += ("\n" + "stderr:\n" + "---\n" + "%s" + "---\n" + % proc.stderr) + self.fail(msg) return proc @@ -637,6 +642,36 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests('--forever', test, exitcode=1) self.check_executed_tests(output, [test]*3, failed=test) + def test_huntrleaks_fd_leak(self): + # test --huntrleaks for file descriptor leak + code = textwrap.dedent(""" + import os + import unittest + + class FDLeakTest(unittest.TestCase): + def test_leak(self): + fd = os.open(__file__, os.O_RDONLY) + # bug: never cloes the file descriptor + """) + test = self.create_test(code=code) + + filename = 'reflog.txt' + self.addCleanup(support.unlink, filename) + output = self.run_tests('--huntrleaks', '3:3:', test, + exitcode=1, + stderr=subprocess.STDOUT) + self.check_executed_tests(output, [test], failed=test) + + line = 'beginning 6 repetitions\n123456\n......\n' + self.check_line(output, re.escape(line)) + + line2 = '%s leaked [1, 1, 1] file descriptors, sum=3\n' % test + self.check_line(output, re.escape(line2)) + + with open(filename) as fp: + reflog = fp.read() + self.assertEqual(reflog, line2) + if __name__ == '__main__': unittest.main() -- cgit v1.2.1 From 42938305ac05e6cc2fc0cd85b18a93dff35a70ff Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 00:21:12 +0200 Subject: Issue #22806: Add ``python -m test --list-tests`` command to list tests. --- Lib/test/libregrtest/cmdline.py | 11 +++------ Lib/test/libregrtest/main.py | 55 +++++++++++++++++++++++++++-------------- Lib/test/test_regrtest.py | 7 ++++++ 3 files changed, 47 insertions(+), 26 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index e55e53f45e..ae1aeb9333 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -1,5 +1,4 @@ import argparse -import faulthandler import os from test import support @@ -234,6 +233,9 @@ def _create_parser(): group.add_argument('-F', '--forever', action='store_true', help='run the specified tests in a loop, until an ' 'error happens') + group.add_argument('--list-tests', action='store_true', + help="only write the name of tests that will be run, " + "don't execute them") parser.add_argument('args', nargs=argparse.REMAINDER, help=argparse.SUPPRESS) @@ -301,12 +303,7 @@ def _parse_args(args, **kwargs): if ns.quiet: ns.verbose = 0 if ns.timeout is not None: - if hasattr(faulthandler, 'dump_traceback_later'): - if ns.timeout <= 0: - ns.timeout = None - else: - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later") + if ns.timeout <= 0: ns.timeout = None if ns.use_mp is not None: if ns.use_mp <= 0: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index c1ce3b179f..30d8a59478 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,3 +1,4 @@ +import faulthandler import os import platform import random @@ -110,8 +111,13 @@ class Regrtest: def parse_args(self, kwargs): ns = _parse_args(sys.argv[1:], **kwargs) + if ns.timeout and not hasattr(faulthandler, 'dump_traceback_later'): + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later", file=sys.stderr) + ns.timeout = None + if ns.threshold is not None and gc is None: - print('No GC available, ignore --threshold.') + print('No GC available, ignore --threshold.', file=sys.stderr) ns.threshold = None if ns.findleaks: @@ -122,7 +128,8 @@ class Regrtest: pass #gc.set_debug(gc.DEBUG_SAVEALL) else: - print('No GC available, disabling --findleaks') + print('No GC available, disabling --findleaks', + file=sys.stderr) ns.findleaks = False # Strip .py extensions. @@ -163,20 +170,6 @@ class Regrtest: nottests.add(arg) self.ns.args = [] - # For a partial run, we do not need to clutter the output. - if (self.ns.verbose - or self.ns.header - or not (self.ns.quiet or self.ns.single - or self.tests or self.ns.args)): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("== ", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== ", "hash algorithm:", sys.hash_info.algorithm, - "64bit" if sys.maxsize > 2**32 else "32bit") - print("== ", os.getcwd()) - print("Testing with flags:", sys.flags) - # if testdir is set, then we are not running the python tests suite, so # don't add default tests to be executed or skipped (pass empty values) if self.ns.testdir: @@ -199,15 +192,18 @@ class Regrtest: del self.selected[:self.selected.index(self.ns.start)] except ValueError: print("Couldn't find starting test (%s), using all tests" - % self.ns.start) + % self.ns.start, file=sys.stderr) if self.ns.randomize: if self.ns.random_seed is None: self.ns.random_seed = random.randrange(10000000) random.seed(self.ns.random_seed) - print("Using random seed", self.ns.random_seed) random.shuffle(self.selected) + def list_tests(self): + for name in self.selected: + print(name) + def rerun_failed_tests(self): self.ns.verbose = True self.ns.failfast = False @@ -315,6 +311,23 @@ class Regrtest: return def run_tests(self): + # For a partial run, we do not need to clutter the output. + if (self.ns.verbose + or self.ns.header + or not (self.ns.quiet or self.ns.single + or self.tests or self.ns.args)): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("== ", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== ", "hash algorithm:", sys.hash_info.algorithm, + "64bit" if sys.maxsize > 2**32 else "32bit") + print("== ", os.getcwd()) + print("Testing with flags:", sys.flags) + + if self.ns.randomize: + print("Using random seed", self.ns.random_seed) + if self.ns.forever: self.tests = self._test_forever(list(self.selected)) self.test_count = '' @@ -359,8 +372,12 @@ class Regrtest: setup_tests(self.ns) self.find_tests(tests) - self.run_tests() + if self.ns.list_tests: + self.list_tests() + sys.exit(0) + + self.run_tests() self.display_result() if self.ns.verbose2 and self.bad: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 8b2449a0f5..e15e724403 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -672,6 +672,13 @@ class ArgsTestCase(BaseTestCase): reflog = fp.read() self.assertEqual(reflog, line2) + def test_list_tests(self): + # test --list-tests + tests = [self.create_test() for i in range(5)] + output = self.run_tests('--list-tests', *tests) + self.assertEqual(output.rstrip().splitlines(), + tests) + if __name__ == '__main__': unittest.main() -- cgit v1.2.1 From fce27aeabbc2870535d76be8a4463236f99d95be Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 02:21:35 +0200 Subject: Issue #18174: Fix test_regrtest when Python is compiled in release mode --- Lib/test/test_regrtest.py | 1 + 1 file changed, 1 insertion(+) (limited to 'Lib') diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e15e724403..e0307a85b9 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -642,6 +642,7 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests('--forever', test, exitcode=1) self.check_executed_tests(output, [test]*3, failed=test) + @unittest.skipUnless(Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak code = textwrap.dedent(""" -- cgit v1.2.1 From 6cf31b051f2a0bd56c4955a47dcfa44bab8b45bc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 21:20:41 +0200 Subject: Issue #25306: Skip test_huntrleaks_fd_leak() of test_regrtest until the bug is fixed. --- Lib/test/test_regrtest.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e0307a85b9..edbb4b0c78 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -643,6 +643,8 @@ class ArgsTestCase(BaseTestCase): self.check_executed_tests(output, [test]*3, failed=test) @unittest.skipUnless(Py_DEBUG, 'need a debug build') + # Issue #25306: the test hangs sometimes on Windows + @unittest.skipIf(sys.platform == 'win32', 'test broken on Windows') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak code = textwrap.dedent(""" -- cgit v1.2.1 From e6b3b7bbf9ae7b0ad89cd19721adf215572b612c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 21:40:21 +0200 Subject: Issue #25306: Try to fix test_huntrleaks_fd_leak() on Windows Issue #25306: Disable popup and logs to stderr on assertion failures in MSCRT. --- Lib/test/test_regrtest.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index edbb4b0c78..2e33555afe 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -643,14 +643,24 @@ class ArgsTestCase(BaseTestCase): self.check_executed_tests(output, [test]*3, failed=test) @unittest.skipUnless(Py_DEBUG, 'need a debug build') - # Issue #25306: the test hangs sometimes on Windows - @unittest.skipIf(sys.platform == 'win32', 'test broken on Windows') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak code = textwrap.dedent(""" import os import unittest + # Issue #25306: Disable popups and logs to stderr on assertion + # failures in MSCRT + try: + import msvcrt + msvcrt.CrtSetReportMode + except (ImportError, AttributeError): + # no Windows, o release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, 0) + class FDLeakTest(unittest.TestCase): def test_leak(self): fd = os.open(__file__, os.O_RDONLY) -- cgit v1.2.1 From 25c8d5cd07dc864de3d44dcfd1444ccb23ffa5fc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 5 Oct 2015 13:43:50 +0200 Subject: Issue #25301: The UTF-8 decoder is now up to 15 times as fast for error handlers: ``ignore``, ``replace`` and ``surrogateescape``. --- Lib/test/test_codecs.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index bdc331e491..7b6883fcc5 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -788,6 +788,18 @@ class UTF8Test(ReadTest, unittest.TestCase): self.check_state_handling_decode(self.encoding, u, u.encode(self.encoding)) + def test_decode_error(self): + for data, error_handler, expected in ( + (b'[\x80\xff]', 'ignore', '[]'), + (b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'), + (b'[\x80\xff]', 'surrogateescape', '[\udc80\udcff]'), + (b'[\x80\xff]', 'backslashreplace', '[\\x80\\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.decode(self.encoding, error_handler), + expected) + def test_lone_surrogates(self): super().test_lone_surrogates() # not sure if this is making sense for -- cgit v1.2.1 From 2fc5c68fef8dbe2861f4165996bfba432b498dcc Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Tue, 6 Oct 2015 13:29:56 -0400 Subject: Closes issue #12006: Add ISO 8601 year, week, and day directives to strptime. This commit adds %G, %V, and %u directives to strptime. Thanks Ashley Anderson for the implementation. --- Lib/_strptime.py | 81 +++++++++++++++++++++++++++++++++++++++-------- Lib/test/test_strptime.py | 57 +++++++++++++++++++++++++-------- 2 files changed, 112 insertions(+), 26 deletions(-) (limited to 'Lib') diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 374923dd13..fe5b046a3c 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -195,12 +195,15 @@ class TimeRE(dict): 'f': r"(?P[0-9]{1,6})", 'H': r"(?P2[0-3]|[0-1]\d|\d)", 'I': r"(?P1[0-2]|0[1-9]|[1-9])", + 'G': r"(?P\d\d\d\d)", 'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", 'm': r"(?P1[0-2]|0[1-9]|[1-9])", 'M': r"(?P[0-5]\d|\d)", 'S': r"(?P6[0-1]|[0-5]\d|\d)", 'U': r"(?P5[0-3]|[0-4]\d|\d)", 'w': r"(?P[0-6])", + 'u': r"(?P[1-7])", + 'V': r"(?P5[0-3]|0[1-9]|[1-4]\d|\d)", # W is set below by using 'U' 'y': r"(?P\d\d)", #XXX: Does 'Y' need to worry about having less or more than @@ -295,6 +298,22 @@ def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon): return 1 + days_to_week + day_of_week +def _calc_julian_from_V(iso_year, iso_week, iso_weekday): + """Calculate the Julian day based on the ISO 8601 year, week, and weekday. + ISO weeks start on Mondays, with week 01 being the week containing 4 Jan. + ISO week days range from 1 (Monday) to 7 (Sunday). + """ + correction = datetime_date(iso_year, 1, 4).isoweekday() + 3 + ordinal = (iso_week * 7) + iso_weekday - correction + # ordinal may be negative or 0 now, which means the date is in the previous + # calendar year + if ordinal < 1: + ordinal += datetime_date(iso_year, 1, 1).toordinal() + iso_year -= 1 + ordinal -= datetime_date(iso_year, 1, 1).toordinal() + return iso_year, ordinal + + def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): """Return a 2-tuple consisting of a time struct and an int containing the number of microseconds based on the input string and the @@ -339,15 +358,15 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): raise ValueError("unconverted data remains: %s" % data_string[found.end():]) - year = None + iso_year = year = None month = day = 1 hour = minute = second = fraction = 0 tz = -1 tzoffset = None # Default to -1 to signify that values not known; not critical to have, # though - week_of_year = -1 - week_of_year_start = -1 + iso_week = week_of_year = None + week_of_year_start = None # weekday and julian defaulted to None so as to signal need to calculate # values weekday = julian = None @@ -369,6 +388,8 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): year += 1900 elif group_key == 'Y': year = int(found_dict['Y']) + elif group_key == 'G': + iso_year = int(found_dict['G']) elif group_key == 'm': month = int(found_dict['m']) elif group_key == 'B': @@ -414,6 +435,9 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): weekday = 6 else: weekday -= 1 + elif group_key == 'u': + weekday = int(found_dict['u']) + weekday -= 1 elif group_key == 'j': julian = int(found_dict['j']) elif group_key in ('U', 'W'): @@ -424,6 +448,8 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): else: # W starts week on Monday. week_of_year_start = 0 + elif group_key == 'V': + iso_week = int(found_dict['V']) elif group_key == 'z': z = found_dict['z'] tzoffset = int(z[1:3]) * 60 + int(z[3:5]) @@ -444,28 +470,57 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): else: tz = value break + # Deal with the cases where ambiguities arize + # don't assume default values for ISO week/year + if year is None and iso_year is not None: + if iso_week is None or weekday is None: + raise ValueError("ISO year directive '%G' must be used with " + "the ISO week directive '%V' and a weekday " + "directive ('%A', '%a', '%w', or '%u').") + if julian is not None: + raise ValueError("Day of the year directive '%j' is not " + "compatible with ISO year directive '%G'. " + "Use '%Y' instead.") + elif week_of_year is None and iso_week is not None: + if weekday is None: + raise ValueError("ISO week directive '%V' must be used with " + "the ISO year directive '%G' and a weekday " + "directive ('%A', '%a', '%w', or '%u').") + else: + raise ValueError("ISO week directive '%V' is incompatible with " + "the year directive '%Y'. Use the ISO year '%G' " + "instead.") + leap_year_fix = False if year is None and month == 2 and day == 29: year = 1904 # 1904 is first leap year of 20th century leap_year_fix = True elif year is None: year = 1900 + + # If we know the week of the year and what day of that week, we can figure # out the Julian day of the year. - if julian is None and week_of_year != -1 and weekday is not None: - week_starts_Mon = True if week_of_year_start == 0 else False - julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, - week_starts_Mon) - # Cannot pre-calculate datetime_date() since can change in Julian - # calculation and thus could have different value for the day of the week - # calculation. + if julian is None and weekday is not None: + if week_of_year is not None: + week_starts_Mon = True if week_of_year_start == 0 else False + julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, + week_starts_Mon) + elif iso_year is not None and iso_week is not None: + year, julian = _calc_julian_from_V(iso_year, iso_week, weekday + 1) + if julian is None: + # Cannot pre-calculate datetime_date() since can change in Julian + # calculation and thus could have different value for the day of + # the week calculation. # Need to add 1 to result since first day of the year is 1, not 0. julian = datetime_date(year, month, day).toordinal() - \ datetime_date(year, 1, 1).toordinal() + 1 - else: # Assume that if they bothered to include Julian day it will - # be accurate. - datetime_result = datetime_date.fromordinal((julian - 1) + datetime_date(year, 1, 1).toordinal()) + else: # Assume that if they bothered to include Julian day (or if it was + # calculated above with year/week/weekday) it will be accurate. + datetime_result = datetime_date.fromordinal( + (julian - 1) + + datetime_date(year, 1, 1).toordinal()) year = datetime_result.year month = datetime_result.month day = datetime_result.day diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 346e2c63f8..6b26c8a55b 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -152,8 +152,8 @@ class TimeRETests(unittest.TestCase): "'%s' using '%s'; group 'a' = '%s', group 'b' = %s'" % (found.string, found.re.pattern, found.group('a'), found.group('b'))) - for directive in ('a','A','b','B','c','d','H','I','j','m','M','p','S', - 'U','w','W','x','X','y','Y','Z','%'): + for directive in ('a','A','b','B','c','d','G','H','I','j','m','M','p', + 'S','u','U','V','w','W','x','X','y','Y','Z','%'): compiled = self.time_re.compile("%" + directive) found = compiled.match(time.strftime("%" + directive)) self.assertTrue(found, "Matching failed on '%s' using '%s' regex" % @@ -218,6 +218,26 @@ class StrptimeTests(unittest.TestCase): else: self.fail("'%s' did not raise ValueError" % bad_format) + # Ambiguous or incomplete cases using ISO year/week/weekday directives + # 1. ISO week (%V) is specified, but the year is specified with %Y + # instead of %G + with self.assertRaises(ValueError): + _strptime._strptime("1999 50", "%Y %V") + # 2. ISO year (%G) and ISO week (%V) are specified, but weekday is not + with self.assertRaises(ValueError): + _strptime._strptime("1999 51", "%G %V") + # 3. ISO year (%G) and weekday are specified, but ISO week (%V) is not + for w in ('A', 'a', 'w', 'u'): + with self.assertRaises(ValueError): + _strptime._strptime("1999 51","%G %{}".format(w)) + # 4. ISO year is specified alone (e.g. time.strptime('2015', '%G')) + with self.assertRaises(ValueError): + _strptime._strptime("2015", "%G") + # 5. Julian/ordinal day (%j) is specified with %G, but not %Y + with self.assertRaises(ValueError): + _strptime._strptime("1999 256", "%G %j") + + def test_strptime_exception_context(self): # check that this doesn't chain exceptions needlessly (see #17572) with self.assertRaises(ValueError) as e: @@ -289,7 +309,7 @@ class StrptimeTests(unittest.TestCase): def test_weekday(self): # Test weekday directives - for directive in ('A', 'a', 'w'): + for directive in ('A', 'a', 'w', 'u'): self.helper(directive,6) def test_julian(self): @@ -458,16 +478,20 @@ class CalculationTests(unittest.TestCase): # Should be able to infer date if given year, week of year (%U or %W) # and day of the week def test_helper(ymd_tuple, test_reason): - for directive in ('W', 'U'): - format_string = "%%Y %%%s %%w" % directive - dt_date = datetime_date(*ymd_tuple) - strp_input = dt_date.strftime(format_string) - strp_output = _strptime._strptime_time(strp_input, format_string) - self.assertTrue(strp_output[:3] == ymd_tuple, - "%s(%s) test failed w/ '%s': %s != %s (%s != %s)" % - (test_reason, directive, strp_input, - strp_output[:3], ymd_tuple, - strp_output[7], dt_date.timetuple()[7])) + for year_week_format in ('%Y %W', '%Y %U', '%G %V'): + for weekday_format in ('%w', '%u', '%a', '%A'): + format_string = year_week_format + ' ' + weekday_format + with self.subTest(test_reason, + date=ymd_tuple, + format=format_string): + dt_date = datetime_date(*ymd_tuple) + strp_input = dt_date.strftime(format_string) + strp_output = _strptime._strptime_time(strp_input, + format_string) + msg = "%r: %s != %s" % (strp_input, + strp_output[7], + dt_date.timetuple()[7]) + self.assertEqual(strp_output[:3], ymd_tuple, msg) test_helper((1901, 1, 3), "week 0") test_helper((1901, 1, 8), "common case") test_helper((1901, 1, 13), "day on Sunday") @@ -499,18 +523,25 @@ class CalculationTests(unittest.TestCase): self.assertEqual(_strptime._strptime_time(value, format)[:-1], expected) check('2015 0 0', '%Y %U %w', 2014, 12, 28, 0, 0, 0, 6, -3) check('2015 0 0', '%Y %W %w', 2015, 1, 4, 0, 0, 0, 6, 4) + check('2015 1 1', '%G %V %u', 2014, 12, 29, 0, 0, 0, 0, 363) check('2015 0 1', '%Y %U %w', 2014, 12, 29, 0, 0, 0, 0, -2) check('2015 0 1', '%Y %W %w', 2014, 12, 29, 0, 0, 0, 0, -2) + check('2015 1 2', '%G %V %u', 2014, 12, 30, 0, 0, 0, 1, 364) check('2015 0 2', '%Y %U %w', 2014, 12, 30, 0, 0, 0, 1, -1) check('2015 0 2', '%Y %W %w', 2014, 12, 30, 0, 0, 0, 1, -1) + check('2015 1 3', '%G %V %u', 2014, 12, 31, 0, 0, 0, 2, 365) check('2015 0 3', '%Y %U %w', 2014, 12, 31, 0, 0, 0, 2, 0) check('2015 0 3', '%Y %W %w', 2014, 12, 31, 0, 0, 0, 2, 0) + check('2015 1 4', '%G %V %u', 2015, 1, 1, 0, 0, 0, 3, 1) check('2015 0 4', '%Y %U %w', 2015, 1, 1, 0, 0, 0, 3, 1) check('2015 0 4', '%Y %W %w', 2015, 1, 1, 0, 0, 0, 3, 1) + check('2015 1 5', '%G %V %u', 2015, 1, 2, 0, 0, 0, 4, 2) check('2015 0 5', '%Y %U %w', 2015, 1, 2, 0, 0, 0, 4, 2) check('2015 0 5', '%Y %W %w', 2015, 1, 2, 0, 0, 0, 4, 2) + check('2015 1 6', '%G %V %u', 2015, 1, 3, 0, 0, 0, 5, 3) check('2015 0 6', '%Y %U %w', 2015, 1, 3, 0, 0, 0, 5, 3) check('2015 0 6', '%Y %W %w', 2015, 1, 3, 0, 0, 0, 5, 3) + check('2015 1 7', '%G %V %u', 2015, 1, 4, 0, 0, 0, 6, 4) class CacheTests(unittest.TestCase): -- cgit v1.2.1 From 666cdd23b70260c0443177cc444ca3b586e3dd9a Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 8 Oct 2015 12:27:06 +0300 Subject: Issue #16099: RobotFileParser now supports Crawl-delay and Request-rate extensions. Patch by Nikolay Bogoychev. --- Lib/test/test_robotparser.py | 90 +++++++++++++++++++++++++++++++++----------- Lib/urllib/robotparser.py | 39 ++++++++++++++++++- 2 files changed, 106 insertions(+), 23 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index d01266f330..90b30722da 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -1,6 +1,7 @@ import io import unittest import urllib.robotparser +from collections import namedtuple from urllib.error import URLError, HTTPError from urllib.request import urlopen from test import support @@ -12,7 +13,8 @@ except ImportError: class RobotTestCase(unittest.TestCase): - def __init__(self, index=None, parser=None, url=None, good=None, agent=None): + def __init__(self, index=None, parser=None, url=None, good=None, + agent=None, request_rate=None, crawl_delay=None): # workaround to make unittest discovery work (see #17066) if not isinstance(index, int): return @@ -25,6 +27,8 @@ class RobotTestCase(unittest.TestCase): self.url = url self.good = good self.agent = agent + self.request_rate = request_rate + self.crawl_delay = crawl_delay def runTest(self): if isinstance(self.url, tuple): @@ -34,6 +38,18 @@ class RobotTestCase(unittest.TestCase): agent = self.agent if self.good: self.assertTrue(self.parser.can_fetch(agent, url)) + self.assertEqual(self.parser.crawl_delay(agent), self.crawl_delay) + # if we have actual values for request rate + if self.request_rate and self.parser.request_rate(agent): + self.assertEqual( + self.parser.request_rate(agent).requests, + self.request_rate.requests + ) + self.assertEqual( + self.parser.request_rate(agent).seconds, + self.request_rate.seconds + ) + self.assertEqual(self.parser.request_rate(agent), self.request_rate) else: self.assertFalse(self.parser.can_fetch(agent, url)) @@ -43,15 +59,17 @@ class RobotTestCase(unittest.TestCase): tests = unittest.TestSuite() def RobotTest(index, robots_txt, good_urls, bad_urls, - agent="test_robotparser"): + request_rate, crawl_delay, agent="test_robotparser"): lines = io.StringIO(robots_txt).readlines() parser = urllib.robotparser.RobotFileParser() parser.parse(lines) for url in good_urls: - tests.addTest(RobotTestCase(index, parser, url, 1, agent)) + tests.addTest(RobotTestCase(index, parser, url, 1, agent, + request_rate, crawl_delay)) for url in bad_urls: - tests.addTest(RobotTestCase(index, parser, url, 0, agent)) + tests.addTest(RobotTestCase(index, parser, url, 0, agent, + request_rate, crawl_delay)) # Examples from http://www.robotstxt.org/wc/norobots.html (fetched 2002) @@ -65,14 +83,18 @@ Disallow: /foo.html good = ['/','/test.html'] bad = ['/cyberworld/map/index.html','/tmp/xxx','/foo.html'] +request_rate = None +crawl_delay = None -RobotTest(1, doc, good, bad) +RobotTest(1, doc, good, bad, request_rate, crawl_delay) # 2. doc = """ # robots.txt for http://www.example.com/ User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. @@ -83,8 +105,10 @@ Disallow: good = ['/','/test.html',('cybermapper','/cyberworld/map/index.html')] bad = ['/cyberworld/map/index.html'] +request_rate = None # The parameters should be equal to None since they +crawl_delay = None # don't apply to the cybermapper user agent -RobotTest(2, doc, good, bad) +RobotTest(2, doc, good, bad, request_rate, crawl_delay) # 3. doc = """ @@ -95,14 +119,18 @@ Disallow: / good = [] bad = ['/cyberworld/map/index.html','/','/tmp/'] +request_rate = None +crawl_delay = None -RobotTest(3, doc, good, bad) +RobotTest(3, doc, good, bad, request_rate, crawl_delay) # Examples from http://www.robotstxt.org/wc/norobots-rfc.html (fetched 2002) # 4. doc = """ User-agent: figtree +Crawl-delay: 3 +Request-rate: 9/30 Disallow: /tmp Disallow: /a%3cd.html Disallow: /a%2fb.html @@ -115,8 +143,17 @@ bad = ['/tmp','/tmp.html','/tmp/a.html', '/~joe/index.html' ] -RobotTest(4, doc, good, bad, 'figtree') -RobotTest(5, doc, good, bad, 'FigTree Robot libwww-perl/5.04') +request_rate = namedtuple('req_rate', 'requests seconds') +request_rate.requests = 9 +request_rate.seconds = 30 +crawl_delay = 3 +request_rate_bad = None # not actually tested, but we still need to parse it +crawl_delay_bad = None # in order to accommodate the input parameters + + +RobotTest(4, doc, good, bad, request_rate, crawl_delay, 'figtree' ) +RobotTest(5, doc, good, bad, request_rate_bad, crawl_delay_bad, + 'FigTree Robot libwww-perl/5.04') # 6. doc = """ @@ -125,14 +162,18 @@ Disallow: /tmp/ Disallow: /a%3Cd.html Disallow: /a/b.html Disallow: /%7ejoe/index.html +Crawl-delay: 3 +Request-rate: 9/banana """ good = ['/tmp',] # XFAIL: '/a%2fb.html' bad = ['/tmp/','/tmp/a.html', '/a%3cd.html','/a%3Cd.html',"/a/b.html", '/%7Ejoe/index.html'] +crawl_delay = 3 +request_rate = None # since request rate has invalid syntax, return None -RobotTest(6, doc, good, bad) +RobotTest(6, doc, good, bad, None, None) # From bug report #523041 @@ -140,12 +181,16 @@ RobotTest(6, doc, good, bad) doc = """ User-Agent: * Disallow: /. +Crawl-delay: pears """ good = ['/foo.html'] -bad = [] # Bug report says "/" should be denied, but that is not in the RFC +bad = [] # bug report says "/" should be denied, but that is not in the RFC + +crawl_delay = None # since crawl delay has invalid syntax, return None +request_rate = None -RobotTest(7, doc, good, bad) +RobotTest(7, doc, good, bad, crawl_delay, request_rate) # From Google: http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40364 @@ -154,12 +199,15 @@ doc = """ User-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ +Request-rate: whale/banana """ good = ['/folder1/myfile.html'] bad = ['/folder1/anotherfile.html'] +crawl_delay = None +request_rate = None # invalid syntax, return none -RobotTest(8, doc, good, bad, agent="Googlebot") +RobotTest(8, doc, good, bad, crawl_delay, request_rate, agent="Googlebot") # 9. This file is incorrect because "Googlebot" is a substring of # "Googlebot-Mobile", so test 10 works just like test 9. @@ -174,12 +222,12 @@ Allow: / good = [] bad = ['/something.jpg'] -RobotTest(9, doc, good, bad, agent="Googlebot") +RobotTest(9, doc, good, bad, None, None, agent="Googlebot") good = [] bad = ['/something.jpg'] -RobotTest(10, doc, good, bad, agent="Googlebot-Mobile") +RobotTest(10, doc, good, bad, None, None, agent="Googlebot-Mobile") # 11. Get the order correct. doc = """ @@ -193,12 +241,12 @@ Disallow: / good = [] bad = ['/something.jpg'] -RobotTest(11, doc, good, bad, agent="Googlebot") +RobotTest(11, doc, good, bad, None, None, agent="Googlebot") good = ['/something.jpg'] bad = [] -RobotTest(12, doc, good, bad, agent="Googlebot-Mobile") +RobotTest(12, doc, good, bad, None, None, agent="Googlebot-Mobile") # 13. Google also got the order wrong in #8. You need to specify the @@ -212,7 +260,7 @@ Disallow: /folder1/ good = ['/folder1/myfile.html'] bad = ['/folder1/anotherfile.html'] -RobotTest(13, doc, good, bad, agent="googlebot") +RobotTest(13, doc, good, bad, None, None, agent="googlebot") # 14. For issue #6325 (query string support) @@ -224,7 +272,7 @@ Disallow: /some/path?name=value good = ['/some/path'] bad = ['/some/path?name=value'] -RobotTest(14, doc, good, bad) +RobotTest(14, doc, good, bad, None, None) # 15. For issue #4108 (obey first * entry) doc = """ @@ -238,7 +286,7 @@ Disallow: /another/path good = ['/another/path'] bad = ['/some/path'] -RobotTest(15, doc, good, bad) +RobotTest(15, doc, good, bad, None, None) # 16. Empty query (issue #17403). Normalizing the url first. doc = """ @@ -250,7 +298,7 @@ Disallow: /another/path? good = ['/some/path?'] bad = ['/another/path?'] -RobotTest(16, doc, good, bad) +RobotTest(16, doc, good, bad, None, None) class RobotHandler(BaseHTTPRequestHandler): diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 4fbb0cb995..4ac553af20 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -10,7 +10,9 @@ http://www.robotstxt.org/norobots-rfc.txt """ -import urllib.parse, urllib.request +import collections +import urllib.parse +import urllib.request __all__ = ["RobotFileParser"] @@ -120,10 +122,29 @@ class RobotFileParser: if state != 0: entry.rulelines.append(RuleLine(line[1], True)) state = 2 + elif line[0] == "crawl-delay": + if state != 0: + # before trying to convert to int we need to make + # sure that robots.txt has valid syntax otherwise + # it will crash + if line[1].strip().isdigit(): + entry.delay = int(line[1]) + state = 2 + elif line[0] == "request-rate": + if state != 0: + numbers = line[1].split('/') + # check if all values are sane + if (len(numbers) == 2 and numbers[0].strip().isdigit() + and numbers[1].strip().isdigit()): + req_rate = collections.namedtuple('req_rate', + 'requests seconds') + entry.req_rate = req_rate + entry.req_rate.requests = int(numbers[0]) + entry.req_rate.seconds = int(numbers[1]) + state = 2 if state == 2: self._add_entry(entry) - def can_fetch(self, useragent, url): """using the parsed robots.txt decide if useragent can fetch url""" if self.disallow_all: @@ -153,6 +174,18 @@ class RobotFileParser: # agent not found ==> access granted return True + def crawl_delay(self, useragent): + for entry in self.entries: + if entry.applies_to(useragent): + return entry.delay + return None + + def request_rate(self, useragent): + for entry in self.entries: + if entry.applies_to(useragent): + return entry.req_rate + return None + def __str__(self): return ''.join([str(entry) + "\n" for entry in self.entries]) @@ -180,6 +213,8 @@ class Entry: def __init__(self): self.useragents = [] self.rulelines = [] + self.delay = None + self.req_rate = None def __str__(self): ret = [] -- cgit v1.2.1 From 98f6dc5fe36b41d1525e6222eba1c8eea9e84bf3 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 8 Oct 2015 09:05:36 -0700 Subject: Issue #23919: Prevents assert dialogs appearing in the test suite. --- Lib/test/libregrtest/cmdline.py | 4 ++++ Lib/test/libregrtest/setup.py | 12 +++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 6b18b3a9b7..21c3e66b54 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -304,6 +304,10 @@ def _parse_args(args, **kwargs): if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3): parser.error("--pgo/-v don't go together!") + if ns.nowindows: + print("Warning: the --nowindows (-n) option is deprecated. " + "Use -vv to display assertions in stderr.", file=sys.stderr) + if ns.quiet: ns.verbose = 0 if ns.timeout is not None: diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 6a1c308ecf..6e05c7e6ff 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -75,8 +75,11 @@ def setup_tests(ns): if ns.threshold is not None: gc.set_threshold(ns.threshold) - if ns.nowindows: + try: import msvcrt + except ImportError: + pass + else: msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| msvcrt.SEM_NOGPFAULTERRORBOX| @@ -88,8 +91,11 @@ def setup_tests(ns): pass else: for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + if ns.verbose and ns.verbose >= 2: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + else: + msvcrt.CrtSetReportMode(m, 0) support.use_resources = ns.use_resources -- cgit v1.2.1 From 11db3c5ee64a27add64933643404dd17fd15a1d5 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 8 Oct 2015 11:34:07 -0700 Subject: Fix missing import in libregrtest. --- Lib/test/libregrtest/cmdline.py | 1 + 1 file changed, 1 insertion(+) (limited to 'Lib') diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 21c3e66b54..a9cee6e79c 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -1,5 +1,6 @@ import argparse import os +import sys from test import support -- cgit v1.2.1 From 38cad1b32cb9388dbb463277b9964bfa4b32d57d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 03:17:30 +0200 Subject: Issue #25318: Avoid sprintf() in backslashreplace() Rewrite backslashreplace() to be closer to PyCodec_BackslashReplaceErrors(). Add also unit tests for non-BMP characters. --- Lib/test/test_codecs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 7b6883fcc5..ff314b10fb 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3155,7 +3155,8 @@ class ASCIITest(unittest.TestCase): ('[\x80\xff\u20ac]', 'ignore', b'[]'), ('[\x80\xff\u20ac]', 'replace', b'[???]'), ('[\x80\xff\u20ac]', 'xmlcharrefreplace', b'[€ÿ€]'), - ('[\x80\xff\u20ac]', 'backslashreplace', b'[\\x80\\xff\\u20ac]'), + ('[\x80\xff\u20ac\U000abcde]', 'backslashreplace', + b'[\\x80\\xff\\u20ac\\U000abcde]'), ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'), ): with self.subTest(data=data, error_handler=error_handler, @@ -3197,7 +3198,8 @@ class Latin1Test(unittest.TestCase): for data, error_handler, expected in ( ('[\u20ac\udc80]', 'ignore', b'[]'), ('[\u20ac\udc80]', 'replace', b'[??]'), - ('[\u20ac\udc80]', 'backslashreplace', b'[\\u20ac\\udc80]'), + ('[\u20ac\U000abcde]', 'backslashreplace', + b'[\\u20ac\\U000abcde]'), ('[\u20ac\udc80]', 'xmlcharrefreplace', b'[€�]'), ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'), ): -- cgit v1.2.1 From bd2799acec915f313ba0c8b69d2863a5f83d1121 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 9 Oct 2015 00:03:51 -0400 Subject: Issue #25298: Add lock and rlock weakref tests (Contributed by Nir Soffer). --- Lib/test/lock_tests.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'Lib') diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index afd6873683..055bf28565 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -7,6 +7,7 @@ import time from _thread import start_new_thread, TIMEOUT_MAX import threading import unittest +import weakref from test import support @@ -198,6 +199,17 @@ class BaseLockTests(BaseTestCase): self.assertFalse(results[0]) self.assertTimeout(results[1], 0.5) + def test_weakref_exists(self): + lock = self.locktype() + ref = weakref.ref(lock) + self.assertIsNotNone(ref()) + + def test_weakref_deleted(self): + lock = self.locktype() + ref = weakref.ref(lock) + del lock + self.assertIsNone(ref()) + class LockTests(BaseLockTests): """ -- cgit v1.2.1 From e3b25615d38fff5febb4df38906f97f60c61327a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 22:50:36 +0200 Subject: Issue #25349: Add fast path for b'%c' % int Optimize also %% formater. --- Lib/test/test_format.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 9b13632591..9924fde13e 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -300,6 +300,8 @@ class FormatTest(unittest.TestCase): testcommon(b"%c", 7, b"\x07") testcommon(b"%c", b"Z", b"Z") testcommon(b"%c", bytearray(b"Z"), b"Z") + testcommon(b"%5c", 65, b" A") + testcommon(b"%-5c", 65, b"A ") # %b will insert a series of bytes, either from a type that supports # the Py_buffer protocol, or something that has a __bytes__ method class FakeBytes(object): -- cgit v1.2.1 From d27a1d900d4ab8640d81864e9bf33ecf43663bd3 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 10 Oct 2015 22:42:18 +0300 Subject: Issue #24164: Objects that need calling ``__new__`` with keyword arguments, can now be pickled using pickle protocols older than protocol version 4. --- Lib/pickle.py | 17 ++++++++++++----- Lib/test/pickletester.py | 6 ++---- 2 files changed, 14 insertions(+), 9 deletions(-) (limited to 'Lib') diff --git a/Lib/pickle.py b/Lib/pickle.py index e93057a7f8..d41753d2d2 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -27,6 +27,7 @@ from types import FunctionType from copyreg import dispatch_table from copyreg import _extension_registry, _inverted_registry, _extension_cache from itertools import islice +from functools import partial import sys from sys import maxsize from struct import pack, unpack @@ -544,7 +545,7 @@ class _Pickler: write = self.write func_name = getattr(func, "__name__", "") - if self.proto >= 4 and func_name == "__newobj_ex__": + if self.proto >= 2 and func_name == "__newobj_ex__": cls, args, kwargs = args if not hasattr(cls, "__new__"): raise PicklingError("args[0] from {} args has no __new__" @@ -552,10 +553,16 @@ class _Pickler: if obj is not None and cls is not obj.__class__: raise PicklingError("args[0] from {} args has the wrong class" .format(func_name)) - save(cls) - save(args) - save(kwargs) - write(NEWOBJ_EX) + if self.proto >= 4: + save(cls) + save(args) + save(kwargs) + write(NEWOBJ_EX) + else: + func = partial(cls.__new__, cls, *args, **kwargs) + save(func) + save(()) + write(REDUCE) elif self.proto >= 2 and func_name == "__newobj__": # A __reduce__ implementation can direct protocol 2 or newer to # use the more efficient NEWOBJ opcode, while still diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 2ef48e64d7..fd641e3161 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1580,16 +1580,14 @@ class AbstractPickleTests(unittest.TestCase): x.abc = 666 for proto in protocols: with self.subTest(proto=proto): - if 2 <= proto < 4: - self.assertRaises(ValueError, self.dumps, x, proto) - continue s = self.dumps(x, proto) if proto < 1: self.assertIn(b'\nL64206', s) # LONG elif proto < 2: self.assertIn(b'M\xce\xfa', s) # BININT2 + elif proto < 4: + self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE else: - assert proto >= 4 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE self.assertFalse(opcode_in_pickle(pickle.NEWOBJ, s)) self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s), -- cgit v1.2.1 From 11aee159a061d48d2a4d756acb99b494b5d7384a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 09:47:17 +0200 Subject: Close #25368: Fix test_eintr when Python is compiled without thread support --- Lib/test/eintrdata/eintr_tester.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 443ccd5433..9de2b6b147 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -52,7 +52,8 @@ class EINTRBaseTest(unittest.TestCase): cls.signal_period) # Issue #25277: Use faulthandler to try to debug a hang on FreeBSD - faulthandler.dump_traceback_later(10 * 60, exit=True) + if hasattr(faulthandler, 'dump_traceback_later'): + faulthandler.dump_traceback_later(10 * 60, exit=True) @classmethod def stop_alarm(cls): @@ -62,7 +63,8 @@ class EINTRBaseTest(unittest.TestCase): def tearDownClass(cls): cls.stop_alarm() signal.signal(signal.SIGALRM, cls.orig_handler) - faulthandler.cancel_dump_traceback_later() + if hasattr(faulthandler, 'cancel_dump_traceback_later'): + faulthandler.cancel_dump_traceback_later() @classmethod def _sleep(cls): -- cgit v1.2.1 From b0811181cefb1fa9c84a5f070b70cbcedf45b993 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 10:03:28 +0200 Subject: Close #25369: Fix test_regrtest without thread support --- Lib/test/test_regrtest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 2e33555afe..0f0069836b 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,6 +7,7 @@ Note: test_regrtest cannot be run twice in parallel. import argparse import faulthandler import getopt +import io import os.path import platform import re @@ -433,7 +434,9 @@ class ProgramsTestCase(BaseTestCase): self.tests = [self.create_test() for index in range(self.NTEST)] self.python_args = ['-Wd', '-E', '-bb'] - self.regrtest_args = ['-uall', '-rwW', '--timeout', '3600', '-j4'] + self.regrtest_args = ['-uall', '-rwW'] + if hasattr(faulthandler, 'dump_traceback_later'): + self.regrtest_args.extend(('--timeout', '3600', '-j4')) if sys.platform == 'win32': self.regrtest_args.append('-n') -- cgit v1.2.1 From da645930483c87cd6fbd16f6651a2feee886d424 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 10:04:26 +0200 Subject: test_regrtest: catch stderr in test_nowindows() Check also that the deprecation warning is emited. --- Lib/test/test_regrtest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0f0069836b..16a29b1c9a 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -5,6 +5,7 @@ Note: test_regrtest cannot be run twice in parallel. """ import argparse +import contextlib import faulthandler import getopt import io @@ -247,8 +248,11 @@ class ParseArgsTestCase(unittest.TestCase): def test_nowindows(self): for opt in '-n', '--nowindows': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + with contextlib.redirect_stderr(io.StringIO()) as stderr: + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.nowindows) + err = stderr.getvalue() + self.assertIn('the --nowindows (-n) option is deprecated', err) def test_forever(self): for opt in '-F', '--forever': -- cgit v1.2.1 From 63fd8d3dbf7d73d91af7fc0f66403ab5ba09b081 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 10:37:25 +0200 Subject: Close #25373: Fix regrtest --slow with interrupted test * Fix accumulate_result(): don't use time on interrupted and failed test * Add unit test for interrupted test * Add unit test on --slow with interrupted test, with and without multiprocessing --- Lib/test/libregrtest/main.py | 12 ++++++++---- Lib/test/test_regrtest.py | 43 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 6 deletions(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index aa95b21562..82788ad941 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -7,10 +7,11 @@ import sys import sysconfig import tempfile import textwrap +from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( findtests, runtest, - STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) -from test.libregrtest.cmdline import _parse_args + STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, + INTERRUPTED, CHILD_ERROR) from test.libregrtest.setup import setup_tests from test import support try: @@ -87,7 +88,8 @@ class Regrtest: def accumulate_result(self, test, result): ok, test_time = result - self.test_times.append((test_time, test)) + if ok not in (CHILD_ERROR, INTERRUPTED): + self.test_times.append((test_time, test)) if ok == PASSED: self.good.append(test) elif ok == FAILED: @@ -291,10 +293,12 @@ class Regrtest: else: try: result = runtest(self.ns, test) - self.accumulate_result(test, result) except KeyboardInterrupt: + self.accumulate_result(test, (INTERRUPTED, None)) self.interrupted = True break + else: + self.accumulate_result(test, result) if self.ns.findleaks: gc.collect() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 16a29b1c9a..ab7741f3bf 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -24,6 +24,16 @@ Py_DEBUG = hasattr(sys, 'getobjects') ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) +TEST_INTERRUPTED = textwrap.dedent(""" + from signal import SIGINT + try: + from _testcapi import raise_signal + raise_signal(SIGINT) + except ImportError: + import os + os.kill(os.getpid(), SIGINT) + """) + class ParseArgsTestCase(unittest.TestCase): """ @@ -340,16 +350,19 @@ class BaseTestCase(unittest.TestCase): return list(match.group(1) for match in parser) def check_executed_tests(self, output, tests, skipped=(), failed=(), - randomize=False): + omitted=(), randomize=False): if isinstance(tests, str): tests = [tests] if isinstance(skipped, str): skipped = [skipped] if isinstance(failed, str): failed = [failed] + if isinstance(omitted, str): + omitted = [omitted] ntest = len(tests) nskipped = len(skipped) nfailed = len(failed) + nomitted = len(omitted) executed = self.parse_executed_tests(output) if randomize: @@ -375,7 +388,11 @@ class BaseTestCase(unittest.TestCase): regex = list_regex('%s test%s failed', failed) self.check_line(output, regex) - good = ntest - nskipped - nfailed + if omitted: + regex = list_regex('%s test%s omitted', omitted) + self.check_line(output, regex) + + good = ntest - nskipped - nfailed - nomitted if good: regex = r'%s test%s OK\.$' % (good, plural(good)) if not skipped and not failed and good > 1: @@ -607,6 +624,12 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests('--fromfile', filename) self.check_executed_tests(output, tests) + def test_interrupted(self): + code = TEST_INTERRUPTED + test = self.create_test("sigint", code=code) + output = self.run_tests(test, exitcode=1) + self.check_executed_tests(output, test, omitted=test) + def test_slow(self): # test --slow tests = [self.create_test() for index in range(3)] @@ -617,6 +640,22 @@ class ArgsTestCase(BaseTestCase): % (self.TESTNAME_REGEX, len(tests))) self.check_line(output, regex) + def test_slow_interrupted(self): + # Issue #25373: test --slow with an interrupted test + code = TEST_INTERRUPTED + test = self.create_test("sigint", code=code) + + for multiprocessing in (False, True): + if multiprocessing: + args = ("--slow", "-j2", test) + else: + args = ("--slow", test) + output = self.run_tests(*args, exitcode=1) + self.check_executed_tests(output, test, omitted=test) + regex = ('10 slowest tests:\n') + self.check_line(output, regex) + self.check_line(output, 'Test suite interrupted by signal SIGINT.') + def test_coverage(self): # test --coverage test = self.create_test() -- cgit v1.2.1 From af5459473f8c6309be0f59eb5898eba9c65f82f8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 11:01:02 +0200 Subject: Issue #25357: Add an optional newline paramer to binascii.b2a_base64(). base64.b64encode() uses it to avoid a memory copy. --- Lib/base64.py | 3 +-- Lib/test/test_binascii.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/base64.py b/Lib/base64.py index 640f787c73..eb925cd4c5 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -58,8 +58,7 @@ def b64encode(s, altchars=None): The encoded byte string is returned. """ - # Strip off the trailing newline - encoded = binascii.b2a_base64(s)[:-1] + encoded = binascii.b2a_base64(s, newline=False) if altchars is not None: assert len(altchars) == 2, repr(altchars) return encoded.translate(bytes.maketrans(b'+/', altchars)) diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 8367afe083..0ab46bc3f3 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -262,6 +262,16 @@ class BinASCIITest(unittest.TestCase): # non-ASCII string self.assertRaises(ValueError, a2b, "\x80") + def test_b2a_base64_newline(self): + # Issue #25357: test newline parameter + b = self.type2test(b'hello') + self.assertEqual(binascii.b2a_base64(b), + b'aGVsbG8=\n') + self.assertEqual(binascii.b2a_base64(b, newline=True), + b'aGVsbG8=\n') + self.assertEqual(binascii.b2a_base64(b, newline=False), + b'aGVsbG8=') + class ArrayBinASCIITest(BinASCIITest): def type2test(self, s): -- cgit v1.2.1 From b38b2aa196de961eb0751c0c6f77f27b94da85a0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 11 Oct 2015 17:52:09 +0300 Subject: Issue #24164: Fixed test_descr: __getnewargs_ex__ now is supported in protocols 2 and 3. --- Lib/test/test_descr.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index d096390110..d75109995e 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4738,11 +4738,8 @@ class PicklingTests(unittest.TestCase): return (args, kwargs) obj = C3() for proto in protocols: - if proto >= 4: + if proto >= 2: self._check_reduce(proto, obj, args, kwargs) - elif proto >= 2: - with self.assertRaises(ValueError): - obj.__reduce_ex__(proto) class C4: def __getnewargs_ex__(self): @@ -5061,10 +5058,6 @@ class PicklingTests(unittest.TestCase): kwargs = getattr(cls, 'KWARGS', {}) obj = cls(*cls.ARGS, **kwargs) proto = pickle_copier.proto - if 2 <= proto < 4 and hasattr(cls, '__getnewargs_ex__'): - with self.assertRaises(ValueError): - pickle_copier.dumps(obj, proto) - continue objcopy = pickle_copier.copy(obj) self._assert_is_copy(obj, objcopy) # For test classes that supports this, make sure we didn't go -- cgit v1.2.1 From cb0ffc7714f16ed3b817544206d948b9979d41a8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Oct 2015 14:38:24 +0200 Subject: Issue #24164: Fix test_pyclbr Ignore pickle.partial symbol which comes from functools.partial. --- Lib/test/test_pyclbr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index cab430b4bd..48bd72590f 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -156,7 +156,7 @@ class PyclbrTest(TestCase): # These were once about the 10 longest modules cm('random', ignore=('Random',)) # from _random import Random as CoreGenerator cm('cgi', ignore=('log',)) # set with = in module - cm('pickle') + cm('pickle', ignore=('partial',)) cm('aifc', ignore=('openfp', '_aifc_params')) # set with = in module cm('sre_parse', ignore=('dump', 'groups')) # from sre_constants import *; property cm('pdb') -- cgit v1.2.1 From e1dc8e119092efec4d67fc14b4a1e10e8349c00d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 13 Oct 2015 21:20:14 +0300 Subject: Issue #25382: pickletools.dis() now outputs implicit memo index for the MEMOIZE opcode. --- Lib/pickletools.py | 1 + 1 file changed, 1 insertion(+) (limited to 'Lib') diff --git a/Lib/pickletools.py b/Lib/pickletools.py index cf5df4158a..43dedb38a5 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -2440,6 +2440,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): if opcode.name in ("PUT", "BINPUT", "LONG_BINPUT", "MEMOIZE"): if opcode.name == "MEMOIZE": memo_idx = len(memo) + markmsg = "(as %d)" % memo_idx else: assert arg is not None memo_idx = arg -- cgit v1.2.1 From 3718ed36cd8d179737e79bfde3f2d5dae222347d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 00:21:35 +0200 Subject: Rewrite PyBytes_FromFormatV() using _PyBytesWriter API * Add much more unit tests on PyBytes_FromFormatV() * Remove the first loop to compute the length of the output string * Use _PyBytesWriter to handle the bytes buffer, use overallocation * Cleanup the code to make simpler and easier to review --- Lib/test/test_bytes.py | 92 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 12 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 53a80f4b47..5fe193e198 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -783,25 +783,93 @@ class BytesTest(BaseBytesTest, unittest.TestCase): # Test PyBytes_FromFormat() def test_from_format(self): test.support.import_module('ctypes') - from ctypes import pythonapi, py_object, c_int, c_char_p + _testcapi = test.support.import_module('_testcapi') + from ctypes import pythonapi, py_object + from ctypes import ( + c_int, c_uint, + c_long, c_ulong, + c_size_t, c_ssize_t, + c_char_p) + PyBytes_FromFormat = pythonapi.PyBytes_FromFormat PyBytes_FromFormat.restype = py_object + # basic tests self.assertEqual(PyBytes_FromFormat(b'format'), b'format') - + self.assertEqual(PyBytes_FromFormat(b'Hello %s !', b'world'), + b'Hello world !') + + # test formatters + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(0)), + b'c=\0') + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(ord('@'))), + b'c=@') + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(255)), + b'c=\xff') + self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', + c_int(1), c_long(2), + c_size_t(3)), + b'd=1 ld=2 zd=3') + self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', + c_int(-1), c_long(-2), + c_size_t(-3)), + b'd=-1 ld=-2 zd=-3') + self.assertEqual(PyBytes_FromFormat(b'u=%u lu=%lu zu=%zu', + c_uint(123), c_ulong(456), + c_size_t(789)), + b'u=123 lu=456 zu=789') + self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(123)), + b'i=123') + self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(-123)), + b'i=-123') + self.assertEqual(PyBytes_FromFormat(b'x=%x', c_int(0xabc)), + b'x=abc') + self.assertEqual(PyBytes_FromFormat(b'ptr=%p', + c_char_p(0xabcdef)), + b'ptr=0xabcdef') + self.assertEqual(PyBytes_FromFormat(b's=%s', c_char_p(b'cstr')), + b's=cstr') + + # test minimum and maximum integer values + size_max = c_size_t(-1).value + for formatstr, ctypes_type, value, py_formatter in ( + (b'%d', c_int, _testcapi.INT_MIN, str), + (b'%d', c_int, _testcapi.INT_MAX, str), + (b'%ld', c_long, _testcapi.LONG_MIN, str), + (b'%ld', c_long, _testcapi.LONG_MAX, str), + (b'%lu', c_ulong, _testcapi.ULONG_MAX, str), + (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MIN, str), + (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MAX, str), + (b'%zu', c_size_t, size_max, str), + (b'%p', c_char_p, size_max, lambda value: '%#x' % value), + ): + self.assertEqual(PyBytes_FromFormat(formatstr, ctypes_type(value)), + py_formatter(value).encode('ascii')), + + # width and precision (width is currently ignored) + self.assertEqual(PyBytes_FromFormat(b'%5s', b'a'), + b'a') + self.assertEqual(PyBytes_FromFormat(b'%.3s', b'abcdef'), + b'abc') + + # '%%' formatter + self.assertEqual(PyBytes_FromFormat(b'%%'), + b'%') + self.assertEqual(PyBytes_FromFormat(b'[%%]'), + b'[%]') + self.assertEqual(PyBytes_FromFormat(b'%%%c', c_int(ord('_'))), + b'%_') + self.assertEqual(PyBytes_FromFormat(b'%%s'), + b'%s') + + # Invalid formats and partial formatting self.assertEqual(PyBytes_FromFormat(b'%'), b'%') - self.assertEqual(PyBytes_FromFormat(b'%%'), b'%') - self.assertEqual(PyBytes_FromFormat(b'%%s'), b'%s') - self.assertEqual(PyBytes_FromFormat(b'[%%]'), b'[%]') - self.assertEqual(PyBytes_FromFormat(b'%%%c', c_int(ord('_'))), b'%_') - - self.assertEqual(PyBytes_FromFormat(b'c:%c', c_int(255)), - b'c:\xff') - self.assertEqual(PyBytes_FromFormat(b's:%s', c_char_p(b'cstr')), - b's:cstr') + self.assertEqual(PyBytes_FromFormat(b'x=%i y=%', c_int(2), c_int(3)), + b'x=2 y=%') - # Issue #19969 + # Issue #19969: %c must raise OverflowError for values + # not in the range [0; 255] self.assertRaises(OverflowError, PyBytes_FromFormat, b'%c', c_int(-1)) self.assertRaises(OverflowError, -- cgit v1.2.1 From de5c571232334cb071fbdc912d1de2f311e9f095 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 02:55:12 +0200 Subject: Fix test_bytes on Windows On Windows, sprintf("%p", 0xabcdef) formats hexadecimal in uppercase and pad to 16 characters (on 64-bit system) with zeros. --- Lib/test/test_bytes.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 5fe193e198..947d959e84 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -782,7 +782,7 @@ class BytesTest(BaseBytesTest, unittest.TestCase): # Test PyBytes_FromFormat() def test_from_format(self): - test.support.import_module('ctypes') + ctypes = test.support.import_module('ctypes') _testcapi = test.support.import_module('_testcapi') from ctypes import pythonapi, py_object from ctypes import ( @@ -825,9 +825,12 @@ class BytesTest(BaseBytesTest, unittest.TestCase): b'i=-123') self.assertEqual(PyBytes_FromFormat(b'x=%x', c_int(0xabc)), b'x=abc') - self.assertEqual(PyBytes_FromFormat(b'ptr=%p', - c_char_p(0xabcdef)), - b'ptr=0xabcdef') + ptr = 0xabcdef + expected = [b'ptr=%#x' % ptr] + win_format = 'ptr=0x%0{}X'.format(2 * ctypes.sizeof(c_char_p)) + expected.append((win_format % ptr).encode('ascii')) + self.assertIn(PyBytes_FromFormat(b'ptr=%p', c_char_p(ptr)), + expected) self.assertEqual(PyBytes_FromFormat(b's=%s', c_char_p(b'cstr')), b's=cstr') -- cgit v1.2.1 From ebccea7d65ef26d6bf89900a22a82bb9247c4ab4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 11:25:33 +0200 Subject: Optimize bytes.fromhex() and bytearray.fromhex() Issue #25401: Optimize bytes.fromhex() and bytearray.fromhex(): they are now between 2x and 3.5x faster. Changes: * Use a fast-path working on a char* string for ASCII string * Use a slow-path for non-ASCII string * Replace slow hex_digit_to_int() function with a O(1) lookup in _PyLong_DigitValue precomputed table * Use _PyBytesWriter API to handle the buffer * Add unit tests to check the error position in error messages --- Lib/test/test_bytes.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 947d959e84..0fe33b5cb3 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -301,6 +301,20 @@ class BaseBytesTest: self.assertRaises(ValueError, self.type2test.fromhex, '\x00') self.assertRaises(ValueError, self.type2test.fromhex, '12 \x00 34') + for data, pos in ( + # invalid first hexadecimal character + ('12 x4 56', 3), + # invalid second hexadecimal character + ('12 3x 56', 4), + # two invalid hexadecimal characters + ('12 xy 56', 3), + # test non-ASCII string + ('12 3\xff 56', 4), + ): + with self.assertRaises(ValueError) as cm: + self.type2test.fromhex(data) + self.assertIn('at position %s' % pos, str(cm.exception)) + def test_hex(self): self.assertRaises(TypeError, self.type2test.hex) self.assertRaises(TypeError, self.type2test.hex, 1) -- cgit v1.2.1 From 710596f4ade3ac43ec90bf4f741a2da46c1b7640 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 15:02:35 +0200 Subject: Issue #25384: Fix binascii.rledecode_hqx() Fix usage of _PyBytesWriter API. Use the new _PyBytesWriter_Resize() function instead of _PyBytesWriter_Prepare(). --- Lib/test/test_binascii.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 0ab46bc3f3..fbc933e4e6 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -159,11 +159,25 @@ class BinASCIITest(unittest.TestCase): # Then calculate the hexbin4 binary-to-ASCII translation rle = binascii.rlecode_hqx(self.data) a = binascii.b2a_hqx(self.type2test(rle)) + b, _ = binascii.a2b_hqx(self.type2test(a)) res = binascii.rledecode_hqx(b) - self.assertEqual(res, self.rawdata) + def test_rle(self): + # test repetition with a repetition longer than the limit of 255 + data = (b'a' * 100 + b'b' + b'c' * 300) + + encoded = binascii.rlecode_hqx(data) + self.assertEqual(encoded, + (b'a\x90d' # 'a' * 100 + b'b' # 'b' + b'c\x90\xff' # 'c' * 255 + b'c\x90-')) # 'c' * 45 + + decoded = binascii.rledecode_hqx(encoded) + self.assertEqual(decoded, data) + def test_hex(self): # test hexlification s = b'{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000' -- cgit v1.2.1 From a3558bfa01054b99609b958d433c225066b10677 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 15:28:59 +0200 Subject: test_bytes: new try to fix test on '%p' formatter on Windows --- Lib/test/test_bytes.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 0fe33b5cb3..87799dfd19 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -839,12 +839,22 @@ class BytesTest(BaseBytesTest, unittest.TestCase): b'i=-123') self.assertEqual(PyBytes_FromFormat(b'x=%x', c_int(0xabc)), b'x=abc') + + sizeof_ptr = ctypes.sizeof(c_char_p) + + if os.name == 'nt': + # Windows (MSCRT) + ptr_format = '0x%0{}X'.format(2 * sizeof_ptr) + def ptr_formatter(ptr): + return (ptr_format % ptr) + else: + # UNIX (glibc) + def ptr_formatter(ptr): + return '%#x' % ptr + ptr = 0xabcdef - expected = [b'ptr=%#x' % ptr] - win_format = 'ptr=0x%0{}X'.format(2 * ctypes.sizeof(c_char_p)) - expected.append((win_format % ptr).encode('ascii')) - self.assertIn(PyBytes_FromFormat(b'ptr=%p', c_char_p(ptr)), - expected) + self.assertEqual(PyBytes_FromFormat(b'ptr=%p', c_char_p(ptr)), + ('ptr=' + ptr_formatter(ptr)).encode('ascii')) self.assertEqual(PyBytes_FromFormat(b's=%s', c_char_p(b'cstr')), b's=cstr') @@ -859,7 +869,7 @@ class BytesTest(BaseBytesTest, unittest.TestCase): (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MIN, str), (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MAX, str), (b'%zu', c_size_t, size_max, str), - (b'%p', c_char_p, size_max, lambda value: '%#x' % value), + (b'%p', c_char_p, size_max, ptr_formatter), ): self.assertEqual(PyBytes_FromFormat(formatstr, ctypes_type(value)), py_formatter(value).encode('ascii')), -- cgit v1.2.1 From e5dfe12ff11d1bea1bc69d5eb87273648c15cf7e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 16 Oct 2015 12:21:37 -0700 Subject: Upgrade the imp module's deprecation to DeprecationWarning. --- Lib/imp.py | 2 +- Lib/pkgutil.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/imp.py b/Lib/imp.py index f6fff44201..b33995267b 100644 --- a/Lib/imp.py +++ b/Lib/imp.py @@ -30,7 +30,7 @@ import warnings warnings.warn("the imp module is deprecated in favour of importlib; " "see the module's documentation for alternative uses", - PendingDeprecationWarning, stacklevel=2) + DeprecationWarning, stacklevel=2) # DEPRECATED SEARCH_ERROR = 0 diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index fc4a074f5b..203d515e5e 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -180,7 +180,7 @@ iter_importer_modules.register( def _import_imp(): global imp with warnings.catch_warnings(): - warnings.simplefilter('ignore', PendingDeprecationWarning) + warnings.simplefilter('ignore', DeprecationWarning) imp = importlib.import_module('imp') class ImpImporter: -- cgit v1.2.1 From f33fb769b89dee6f38108cfc5cd40baa8ea0791a Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Fri, 16 Oct 2015 20:45:53 -0400 Subject: Issue 25422: Add tests for multi-line string tokenization. Also remove truncated tokens. --- Lib/test/test_tokenize.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 3b17ca6329..b74396f1be 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -24,8 +24,7 @@ class TokenizeTest(TestCase): if type == ENDMARKER: break type = tok_name[type] - result.append(" %(type)-10.10s %(token)-13.13r %(start)s %(end)s" % - locals()) + result.append(f" {type:10} {token!r:13} {start} {end}") self.assertEqual(result, [" ENCODING 'utf-8' (0, 0) (0, 0)"] + expected.rstrip().splitlines()) @@ -132,18 +131,18 @@ def k(x): self.check_tokenize("x = 0xfffffffffff", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) - NUMBER '0xffffffffff (1, 4) (1, 17) + NUMBER '0xfffffffffff' (1, 4) (1, 17) """) self.check_tokenize("x = 123141242151251616110", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) - NUMBER '123141242151 (1, 4) (1, 25) + NUMBER '123141242151251616110' (1, 4) (1, 25) """) self.check_tokenize("x = -15921590215012591", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '-' (1, 4) (1, 5) - NUMBER '159215902150 (1, 5) (1, 22) + NUMBER '15921590215012591' (1, 5) (1, 22) """) def test_float(self): @@ -306,6 +305,33 @@ def k(x): STRING 'Rb"abc"' (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING 'RB"abc"' (1, 30) (1, 37) + """) + # Check 0, 1, and 2 character string prefixes. + self.check_tokenize(r'"a\ +de\ +fg"', """\ + STRING '"a\\\\\\nde\\\\\\nfg"\' (1, 0) (3, 3) + """) + self.check_tokenize(r'u"a\ +de"', """\ + STRING 'u"a\\\\\\nde"\' (1, 0) (2, 3) + """) + self.check_tokenize(r'rb"a\ +d"', """\ + STRING 'rb"a\\\\\\nd"\' (1, 0) (2, 2) + """) + self.check_tokenize(r'"""a\ +b"""', """\ + STRING '\"\""a\\\\\\nb\"\""' (1, 0) (2, 4) + """) + self.check_tokenize(r'u"""a\ +b"""', """\ + STRING 'u\"\""a\\\\\\nb\"\""' (1, 0) (2, 4) + """) + self.check_tokenize(r'rb"""a\ +b\ +c"""', """\ + STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4) """) def test_function(self): @@ -505,7 +531,7 @@ def k(x): # Methods self.check_tokenize("@staticmethod\ndef foo(x,y): pass", """\ OP '@' (1, 0) (1, 1) - NAME 'staticmethod (1, 1) (1, 13) + NAME 'staticmethod' (1, 1) (1, 13) NEWLINE '\\n' (1, 13) (1, 14) NAME 'def' (2, 0) (2, 3) NAME 'foo' (2, 4) (2, 7) -- cgit v1.2.1 From 25527ac85478fac8430dc72d89b5c8300237445c Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 22 Oct 2015 07:49:36 +0300 Subject: Issue #25210: Add some basic tests for the new exception message --- Lib/test/test_richcmp.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_richcmp.py b/Lib/test/test_richcmp.py index 1582caad97..58729a9fea 100644 --- a/Lib/test/test_richcmp.py +++ b/Lib/test/test_richcmp.py @@ -253,6 +253,31 @@ class MiscTest(unittest.TestCase): self.assertTrue(a != b) self.assertTrue(a < b) + def test_exception_message(self): + class Spam: + pass + + tests = [ + (lambda: 42 < None, r"'<' .* of 'int' and 'NoneType'"), + (lambda: None < 42, r"'<' .* of 'NoneType' and 'int'"), + (lambda: 42 > None, r"'>' .* of 'int' and 'NoneType'"), + (lambda: "foo" < None, r"'<' .* of 'str' and 'NoneType'"), + (lambda: "foo" >= 666, r"'>=' .* of 'str' and 'int'"), + (lambda: 42 <= None, r"'<=' .* of 'int' and 'NoneType'"), + (lambda: 42 >= None, r"'>=' .* of 'int' and 'NoneType'"), + (lambda: 42 < [], r"'<' .* of 'int' and 'list'"), + (lambda: () > [], r"'>' .* of 'tuple' and 'list'"), + (lambda: None >= None, r"'>=' .* of 'NoneType' and 'NoneType'"), + (lambda: Spam() < 42, r"'<' .* of 'Spam' and 'int'"), + (lambda: 42 < Spam(), r"'<' .* of 'int' and 'Spam'"), + (lambda: Spam() <= Spam(), r"'<=' .* of 'Spam' and 'Spam'"), + ] + for i, test in enumerate(tests): + with self.subTest(test=i): + with self.assertRaisesRegex(TypeError, test[1]): + test[0]() + + class DictTest(unittest.TestCase): def test_dicts(self): -- cgit v1.2.1 From a9254c552a389703124141ebffae2519cd07ac08 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Mon, 26 Oct 2015 04:37:55 -0400 Subject: Issue 25311: Add support for f-strings to tokenize.py. Also added some comments to explain what's happening, since it's not so obvious. --- Lib/test/test_tokenize.py | 17 +++++++ Lib/tokenize.py | 118 ++++++++++++++++++++++++++-------------------- 2 files changed, 84 insertions(+), 51 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index b74396f1be..90438e7d30 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -332,6 +332,23 @@ b"""', """\ b\ c"""', """\ STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4) + """) + self.check_tokenize('f"abc"', """\ + STRING 'f"abc"' (1, 0) (1, 6) + """) + self.check_tokenize('fR"a{b}c"', """\ + STRING 'fR"a{b}c"' (1, 0) (1, 9) + """) + self.check_tokenize('f"""abc"""', """\ + STRING 'f\"\"\"abc\"\"\"' (1, 0) (1, 10) + """) + self.check_tokenize(r'f"abc\ +def"', """\ + STRING 'f"abc\\\\\\ndef"' (1, 0) (2, 4) + """) + self.check_tokenize(r'Rf"abc\ +def"', """\ + STRING 'Rf"abc\\\\\\ndef"' (1, 0) (2, 4) """) def test_function(self): diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 65d06e53f3..2237c3a554 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -29,6 +29,7 @@ from codecs import lookup, BOM_UTF8 import collections from io import TextIOWrapper from itertools import chain +import itertools as _itertools import re import sys from token import * @@ -131,7 +132,28 @@ Floatnumber = group(Pointfloat, Expfloat) Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) -StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU])?' +# Return the empty string, plus all of the valid string prefixes. +def _all_string_prefixes(): + # The valid string prefixes. Only contain the lower case versions, + # and don't contain any permuations (include 'fr', but not + # 'rf'). The various permutations will be generated. + _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] + # if we add binary f-strings, add: ['fb', 'fbr'] + result = set(['']) + for prefix in _valid_string_prefixes: + for t in _itertools.permutations(prefix): + # create a list with upper and lower versions of each + # character + for u in _itertools.product(*[(c, c.upper()) for c in t]): + result.add(''.join(u)) + return result + +def _compile(expr): + return re.compile(expr, re.UNICODE) + +# Note that since _all_string_prefixes includes the empty string, +# StringPrefix can be the empty string (making it optional). +StringPrefix = group(*_all_string_prefixes()) # Tail end of ' string. Single = r"[^'\\]*(?:\\.[^'\\]*)*'" @@ -169,50 +191,25 @@ ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) -def _compile(expr): - return re.compile(expr, re.UNICODE) - -endpats = {"'": Single, '"': Double, - "'''": Single3, '"""': Double3, - "r'''": Single3, 'r"""': Double3, - "b'''": Single3, 'b"""': Double3, - "R'''": Single3, 'R"""': Double3, - "B'''": Single3, 'B"""': Double3, - "br'''": Single3, 'br"""': Double3, - "bR'''": Single3, 'bR"""': Double3, - "Br'''": Single3, 'Br"""': Double3, - "BR'''": Single3, 'BR"""': Double3, - "rb'''": Single3, 'rb"""': Double3, - "Rb'''": Single3, 'Rb"""': Double3, - "rB'''": Single3, 'rB"""': Double3, - "RB'''": Single3, 'RB"""': Double3, - "u'''": Single3, 'u"""': Double3, - "U'''": Single3, 'U"""': Double3, - 'r': None, 'R': None, 'b': None, 'B': None, - 'u': None, 'U': None} - -triple_quoted = {} -for t in ("'''", '"""', - "r'''", 'r"""', "R'''", 'R"""', - "b'''", 'b"""', "B'''", 'B"""', - "br'''", 'br"""', "Br'''", 'Br"""', - "bR'''", 'bR"""', "BR'''", 'BR"""', - "rb'''", 'rb"""', "rB'''", 'rB"""', - "Rb'''", 'Rb"""', "RB'''", 'RB"""', - "u'''", 'u"""', "U'''", 'U"""', - ): - triple_quoted[t] = t -single_quoted = {} -for t in ("'", '"', - "r'", 'r"', "R'", 'R"', - "b'", 'b"', "B'", 'B"', - "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"' , - "rb'", 'rb"', "rB'", 'rB"', - "Rb'", 'Rb"', "RB'", 'RB"' , - "u'", 'u"', "U'", 'U"', - ): - single_quoted[t] = t +# For a given string prefix plus quotes, endpats maps it to a regex +# to match the remainder of that string. _prefix can be empty, for +# a normal single or triple quoted string (with no prefix). +endpats = {} +for _prefix in _all_string_prefixes(): + endpats[_prefix + "'"] = Single + endpats[_prefix + '"'] = Double + endpats[_prefix + "'''"] = Single3 + endpats[_prefix + '"""'] = Double3 + +# A set of all of the single and triple quoted string prefixes, +# including the opening quotes. +single_quoted = set() +triple_quoted = set() +for t in _all_string_prefixes(): + for u in (t + '"', t + "'"): + single_quoted.add(u) + for u in (t + '"""', t + "'''"): + triple_quoted.add(u) tabsize = 8 @@ -626,6 +623,7 @@ def _tokenize(readline, encoding): yield stashed stashed = None yield TokenInfo(COMMENT, token, spos, epos, line) + elif token in triple_quoted: endprog = _compile(endpats[token]) endmatch = endprog.match(line, pos) @@ -638,19 +636,37 @@ def _tokenize(readline, encoding): contstr = line[start:] contline = line break - elif initial in single_quoted or \ - token[:2] in single_quoted or \ - token[:3] in single_quoted: + + # Check up to the first 3 chars of the token to see if + # they're in the single_quoted set. If so, they start + # a string. + # We're using the first 3, because we're looking for + # "rb'" (for example) at the start of the token. If + # we switch to longer prefixes, this needs to be + # adjusted. + # Note that initial == token[:1]. + # Also note that single quote checking must come afer + # triple quote checking (above). + elif (initial in single_quoted or + token[:2] in single_quoted or + token[:3] in single_quoted): if token[-1] == '\n': # continued string strstart = (lnum, start) - endprog = _compile(endpats[initial] or - endpats[token[1]] or - endpats[token[2]]) + # Again, using the first 3 chars of the + # token. This is looking for the matching end + # regex for the correct type of quote + # character. So it's really looking for + # endpats["'"] or endpats['"'], by trying to + # skip string prefix characters, if any. + endprog = _compile(endpats.get(initial) or + endpats.get(token[1]) or + endpats.get(token[2])) contstr, needcont = line[start:], 1 contline = line break else: # ordinary string yield TokenInfo(STRING, token, spos, epos, line) + elif initial.isidentifier(): # ordinary name if token in ('async', 'await'): if async_def: -- cgit v1.2.1 From aaaba68ae029727fdad6318ac242c3bbadc12b68 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 30 Oct 2015 14:41:06 -0700 Subject: Issue #25487: Fix tests not updated when the imp module moved to a DeprecationWarning. Thanks to Martin Panter for finding the tests. --- Lib/modulefinder.py | 2 +- Lib/test/test_imp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index 50f2462da0..4a2f1b5491 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -10,7 +10,7 @@ import types import struct import warnings with warnings.catch_warnings(): - warnings.simplefilter('ignore', PendingDeprecationWarning) + warnings.simplefilter('ignore', DeprecationWarning) import imp # XXX Clean up once str8's cstor matches bytes. diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index ee9ee1ad8c..efb03840c4 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -12,7 +12,7 @@ from test import support import unittest import warnings with warnings.catch_warnings(): - warnings.simplefilter('ignore', PendingDeprecationWarning) + warnings.simplefilter('ignore', DeprecationWarning) import imp -- cgit v1.2.1 From 0e2c6cf78b68c744893f5d29834784710de83092 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 1 Nov 2015 17:14:27 +0200 Subject: Issue #18973: Command-line interface of the calendar module now uses argparse instead of optparse. --- Lib/calendar.py | 81 ++++++++++++++++++++++++++--------------------- Lib/test/test_calendar.py | 8 ++--- 2 files changed, 49 insertions(+), 40 deletions(-) (limited to 'Lib') diff --git a/Lib/calendar.py b/Lib/calendar.py index 02050ea457..4ff154cb6e 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -605,51 +605,63 @@ def timegm(tuple): def main(args): - import optparse - parser = optparse.OptionParser(usage="usage: %prog [options] [year [month]]") - parser.add_option( + import argparse + parser = argparse.ArgumentParser() + textgroup = parser.add_argument_group('text only arguments') + htmlgroup = parser.add_argument_group('html only arguments') + textgroup.add_argument( "-w", "--width", - dest="width", type="int", default=2, - help="width of date column (default 2, text only)" + type=int, default=2, + help="width of date column (default 2)" ) - parser.add_option( + textgroup.add_argument( "-l", "--lines", - dest="lines", type="int", default=1, - help="number of lines for each week (default 1, text only)" + type=int, default=1, + help="number of lines for each week (default 1)" ) - parser.add_option( + textgroup.add_argument( "-s", "--spacing", - dest="spacing", type="int", default=6, - help="spacing between months (default 6, text only)" + type=int, default=6, + help="spacing between months (default 6)" ) - parser.add_option( + textgroup.add_argument( "-m", "--months", - dest="months", type="int", default=3, - help="months per row (default 3, text only)" + type=int, default=3, + help="months per row (default 3)" ) - parser.add_option( + htmlgroup.add_argument( "-c", "--css", - dest="css", default="calendar.css", - help="CSS to use for page (html only)" + default="calendar.css", + help="CSS to use for page" ) - parser.add_option( + parser.add_argument( "-L", "--locale", - dest="locale", default=None, + default=None, help="locale to be used from month and weekday names" ) - parser.add_option( + parser.add_argument( "-e", "--encoding", - dest="encoding", default=None, - help="Encoding to use for output." + default=None, + help="encoding to use for output" ) - parser.add_option( + parser.add_argument( "-t", "--type", - dest="type", default="text", + default="text", choices=("text", "html"), help="output type (text or html)" ) + parser.add_argument( + "year", + nargs='?', type=int, + help="year number (1-9999)" + ) + parser.add_argument( + "month", + nargs='?', type=int, + help="month number (1-12, text only)" + ) - (options, args) = parser.parse_args(args) + options = parser.parse_args(args[1:]) if options.locale and not options.encoding: parser.error("if --locale is specified --encoding is required") @@ -667,10 +679,10 @@ def main(args): encoding = sys.getdefaultencoding() optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write - if len(args) == 1: + if options.year is None: write(cal.formatyearpage(datetime.date.today().year, **optdict)) - elif len(args) == 2: - write(cal.formatyearpage(int(args[1]), **optdict)) + elif options.month is None: + write(cal.formatyearpage(options.year, **optdict)) else: parser.error("incorrect number of arguments") sys.exit(1) @@ -680,18 +692,15 @@ def main(args): else: cal = TextCalendar() optdict = dict(w=options.width, l=options.lines) - if len(args) != 3: + if options.month is None: optdict["c"] = options.spacing optdict["m"] = options.months - if len(args) == 1: + if options.year is None: result = cal.formatyear(datetime.date.today().year, **optdict) - elif len(args) == 2: - result = cal.formatyear(int(args[1]), **optdict) - elif len(args) == 3: - result = cal.formatmonth(int(args[1]), int(args[2]), **optdict) + elif options.month is None: + result = cal.formatyear(options.year, **optdict) else: - parser.error("incorrect number of arguments") - sys.exit(1) + result = cal.formatmonth(options.year, options.month, **optdict) write = sys.stdout.write if options.encoding: result = result.encode(options.encoding) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 80ed632588..d9d3128ea8 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -702,19 +702,19 @@ class CommandLineTestCase(unittest.TestCase): def assertFailure(self, *args): rc, stdout, stderr = assert_python_failure('-m', 'calendar', *args) - self.assertIn(b'Usage:', stderr) + self.assertIn(b'usage:', stderr) self.assertEqual(rc, 2) def test_help(self): stdout = self.run_ok('-h') - self.assertIn(b'Usage:', stdout) + self.assertIn(b'usage:', stdout) self.assertIn(b'calendar.py', stdout) self.assertIn(b'--help', stdout) def test_illegal_arguments(self): self.assertFailure('-z') - #self.assertFailure('spam') - #self.assertFailure('2004', 'spam') + self.assertFailure('spam') + self.assertFailure('2004', 'spam') self.assertFailure('-t', 'html', '2004', '1') def test_output_current_year(self): -- cgit v1.2.1 From c38bea2ca0cb9280027ec446627ed2c8276007c5 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 2 Nov 2015 00:39:56 -0500 Subject: Issue #24379: Revert the operator.subscript patch (dccc4e63aef5) pending resolution of the related refcnt leak. --- Lib/operator.py | 28 +--------------------------- Lib/test/test_operator.py | 33 --------------------------------- 2 files changed, 1 insertion(+), 60 deletions(-) (limited to 'Lib') diff --git a/Lib/operator.py b/Lib/operator.py index bc2a9478b8..0e2e53efc6 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -17,7 +17,7 @@ __all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', - 'setitem', 'sub', 'subscript', 'truediv', 'truth', 'xor'] + 'setitem', 'sub', 'truediv', 'truth', 'xor'] from builtins import abs as _abs @@ -408,32 +408,6 @@ def ixor(a, b): return a -@object.__new__ # create a singleton instance -class subscript: - """ - A helper to turn subscript notation into indexing objects. This can be - used to create item access patterns ahead of time to pass them into - various subscriptable objects. - - For example: - subscript[5] == 5 - subscript[3:7:2] == slice(3, 7, 2) - subscript[5, 8] == (5, 8) - """ - __slots__ = () - - def __new__(cls): - raise TypeError("cannot create '{}' instances".format(cls.__name__)) - - @staticmethod - def __getitem__(key): - return key - - @staticmethod - def __reduce__(): - return 'subscript' - - try: from _operator import * except ImportError: diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 27501c2466..54fd1f4e52 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -596,38 +596,5 @@ class CCOperatorPickleTestCase(OperatorPickleTestCase, unittest.TestCase): module2 = c_operator -class SubscriptTestCase: - def test_subscript(self): - subscript = self.module.subscript - self.assertIsNone(subscript[None]) - self.assertEqual(subscript[0], 0) - self.assertEqual(subscript[0:1:2], slice(0, 1, 2)) - self.assertEqual( - subscript[0, ..., :2, ...], - (0, Ellipsis, slice(2), Ellipsis), - ) - - def test_pickle(self): - from operator import subscript - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(proto=proto): - self.assertIs( - pickle.loads(pickle.dumps(subscript, proto)), - subscript, - ) - - def test_singleton(self): - with self.assertRaises(TypeError): - type(self.module.subscript)() - - def test_immutable(self): - with self.assertRaises(AttributeError): - self.module.subscript.attr = None - - -class PySubscriptTestCase(SubscriptTestCase, PyOperatorTestCase): - pass - - if __name__ == "__main__": unittest.main() -- cgit v1.2.1 From 33a07588c44980e713c7e0f10b7a8bd5c6d5046d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 3 Nov 2015 14:34:51 +0100 Subject: locale.delocalize(): only call localeconv() once --- Lib/locale.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/locale.py b/Lib/locale.py index ceaa6d8ff7..074f6e02fa 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -303,12 +303,16 @@ def str(val): def delocalize(string): "Parses a string as a normalized number according to the locale settings." + + conv = localeconv() + #First, get rid of the grouping - ts = localeconv()['thousands_sep'] + ts = conv['thousands_sep'] if ts: string = string.replace(ts, '') + #next, replace the decimal point with a dot - dd = localeconv()['decimal_point'] + dd = conv['decimal_point'] if dd: string = string.replace(dd, '.') return string -- cgit v1.2.1 From 008499d3f1761db65815faebb31bf95ab7ef9cd4 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Tue, 3 Nov 2015 12:45:05 -0500 Subject: Issue 25483: Add an opcode to make f-string formatting more robust. --- Lib/importlib/_bootstrap_external.py | 3 ++- Lib/opcode.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index e0a1200547..c58a62a90d 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -223,12 +223,13 @@ _code_type = type(_write_atomic.__code__) # Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations) # Python 3.5b2 3340 (fix dictionary display evaluation order #11205) # Python 3.5b2 3350 (add GET_YIELD_FROM_ITER opcode #24400) +# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) # # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually # due to the addition of new opcodes). -MAGIC_NUMBER = (3350).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3360).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff --git a/Lib/opcode.py b/Lib/opcode.py index 4c826a7730..71ce67233b 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -214,4 +214,6 @@ def_op('BUILD_MAP_UNPACK_WITH_CALL', 151) def_op('BUILD_TUPLE_UNPACK', 152) def_op('BUILD_SET_UNPACK', 153) +def_op('FORMAT_VALUE', 155) + del def_op, name_op, jrel_op, jabs_op -- cgit v1.2.1 From 03fdef3cfb4b98ce8e532d8eda757516bd833724 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Nov 2015 09:03:53 +0100 Subject: regrtest: display progress every 30 seconds (instead of 60 seconds) when running tests in multiprocessing mode (-jN). --- Lib/test/libregrtest/runtest_mp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 4473db4549..0ca7dd7a4f 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -21,7 +21,7 @@ from test.libregrtest.setup import setup_tests PROGRESS_MIN_TIME = 30.0 # seconds # Display the running tests if nothing happened last N seconds -PROGRESS_UPDATE = 60.0 # seconds +PROGRESS_UPDATE = 30.0 # seconds def run_test_in_subprocess(testname, ns): -- cgit v1.2.1 From b3de672499523d3949e8f15a8477ee1bb8ccf5db Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 7 Nov 2015 18:06:24 +0200 Subject: Issue #25263: Trying to fix test_use on Windows. --- Lib/tkinter/test/test_tkinter/test_widgets.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/tkinter/test/test_tkinter/test_widgets.py b/Lib/tkinter/test/test_tkinter/test_widgets.py index 7171667cc8..5a01a5a2d8 100644 --- a/Lib/tkinter/test/test_tkinter/test_widgets.py +++ b/Lib/tkinter/test/test_tkinter/test_widgets.py @@ -91,9 +91,10 @@ class ToplevelTest(AbstractToplevelTest, unittest.TestCase): widget = self.create() self.assertEqual(widget['use'], '') parent = self.create(container=True) - wid = parent.winfo_id() - widget2 = self.create(use=wid) - self.assertEqual(int(widget2['use']), wid) + wid = hex(parent.winfo_id()) + with self.subTest(wid=wid): + widget2 = self.create(use=wid) + self.assertEqual(widget2['use'], wid) @add_standard_options(StandardOptionsTests) -- cgit v1.2.1 From ae9b0efa053bba441a3b0f8971df300545b8425c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 9 Nov 2015 14:43:31 +0200 Subject: Issue #25263: Trying to fix test_use on Windows. Avoid possible weird behavior of WideInt convertion. "winfo id" always returns string hexadecimal representation. --- Lib/tkinter/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 12085a9bd7..46f86f9d8c 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -845,8 +845,7 @@ class Misc: self.tk.call('winfo', 'height', self._w)) def winfo_id(self): """Return identifier ID for this widget.""" - return self.tk.getint( - self.tk.call('winfo', 'id', self._w)) + return int(self.tk.call('winfo', 'id', self._w), 0) def winfo_interps(self, displayof=0): """Return the name of all Tcl interpreters for this display.""" args = ('winfo', 'interps') + self._displayof(displayof) -- cgit v1.2.1 From 04d70b51387edb7b650b3542d818e52f2851a690 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Fri, 13 Nov 2015 23:54:02 +0000 Subject: Issue #25590: Complete attribute names even if they are not yet created --- Lib/rlcompleter.py | 8 +++++--- Lib/test/test_rlcompleter.py | 8 ++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index d368876b0c..02e1fa50cc 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -160,12 +160,14 @@ class Completer: for word in words: if (word[:n] == attr and not (noprefix and word[:n+1] == noprefix)): + match = "%s.%s" % (expr, word) try: val = getattr(thisobject, word) except Exception: - continue # Exclude properties that are not set - word = self._callable_postfix(val, "%s.%s" % (expr, word)) - matches.append(word) + pass # Include even if attribute not set + else: + match = self._callable_postfix(val, match) + matches.append(match) if matches or not noprefix: break if noprefix == '_': diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index fee39bc434..8ff75c75c5 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -92,6 +92,14 @@ class TestRlcompleter(unittest.TestCase): self.assertEqual(completer.complete('f.b', 0), 'f.bar') self.assertEqual(f.calls, 1) + def test_uncreated_attr(self): + # Attributes like properties and slots should be completed even when + # they haven't been created on an instance + class Foo: + __slots__ = ("bar",) + completer = rlcompleter.Completer(dict(f=Foo())) + self.assertEqual(completer.complete('f.', 0), 'f.bar') + def test_complete(self): completer = rlcompleter.Completer() self.assertEqual(completer.complete('', 0), '\t') -- cgit v1.2.1 From b56989225d01c3caa06188e22ab5f5b3ea37ffc8 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 14 Nov 2015 08:54:30 +0000 Subject: Issue #25168: Temporary timezone and cache debugging --- Lib/test/datetimetester.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 11deffccd7..332d94b64f 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1979,7 +1979,16 @@ class TestDateTime(TestDate): seconds = tzseconds hours, minutes = divmod(seconds//60, 60) dtstr = "{}{:02d}{:02d} {}".format(sign, hours, minutes, tzname) - dt = strptime(dtstr, "%z %Z") + try: + dt = strptime(dtstr, "%z %Z") + except ValueError: + import os + self.fail( + "Issue #25168 strptime() failure info:\n" + f"_TimeRE_cache['Z']={_strptime._TimeRE_cache['Z']!r}\n" + f"TZ={os.environ.get('TZ')!r}, or {os.getenv('TZ')!r} via getenv()\n" + f"_regex_cache={_strptime._regex_cache!r}\n" + ) self.assertEqual(dt.utcoffset(), timedelta(seconds=tzseconds)) self.assertEqual(dt.tzname(), tzname) # Can produce inconsistent datetime -- cgit v1.2.1 From d4b32c4332f9c1e6cdca5eb24bcf1b7c13ce57ba Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 14 Nov 2015 11:47:00 +0000 Subject: Issue #23883: Add test.support.check__all__() and test gettext.__all__ Patches by Jacek Ko?odziej. --- Lib/test/support/__init__.py | 61 ++++++++++++++++++++++++++++++++++++++++++++ Lib/test/test_gettext.py | 6 +++++ Lib/test/test_support.py | 22 ++++++++++++++++ 3 files changed, 89 insertions(+) (limited to 'Lib') diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 359d6dd0f5..728b459751 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -26,6 +26,7 @@ import sys import sysconfig import tempfile import time +import types import unittest import urllib.error import warnings @@ -89,6 +90,7 @@ __all__ = [ "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", "requires_IEEE_754", "skip_unless_xattr", "requires_zlib", "anticipate_failure", "load_package_tests", "detect_api_mismatch", + "check__all__", # sys "is_jython", "check_impl_detail", # network @@ -2199,6 +2201,65 @@ def detect_api_mismatch(ref_api, other_api, *, ignore=()): return missing_items +def check__all__(test_case, module, name_of_module=None, extra=(), + blacklist=()): + """Assert that the __all__ variable of 'module' contains all public names. + + The module's public names (its API) are detected automatically based on + whether they match the public name convention and were defined in + 'module'. + + The 'name_of_module' argument can specify (as a string or tuple thereof) + what module(s) an API could be defined in in order to be detected as a + public API. One case for this is when 'module' imports part of its public + API from other modules, possibly a C backend (like 'csv' and its '_csv'). + + The 'extra' argument can be a set of names that wouldn't otherwise be + automatically detected as "public", like objects without a proper + '__module__' attriubute. If provided, it will be added to the + automatically detected ones. + + The 'blacklist' argument can be a set of names that must not be treated + as part of the public API even though their names indicate otherwise. + + Usage: + import bar + import foo + import unittest + from test import support + + class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, foo) + + class OtherTestCase(unittest.TestCase): + def test__all__(self): + extra = {'BAR_CONST', 'FOO_CONST'} + blacklist = {'baz'} # Undocumented name. + # bar imports part of its API from _bar. + support.check__all__(self, bar, ('bar', '_bar'), + extra=extra, blacklist=blacklist) + + """ + + if name_of_module is None: + name_of_module = (module.__name__, ) + elif isinstance(name_of_module, str): + name_of_module = (name_of_module, ) + + expected = set(extra) + + for name in dir(module): + if name.startswith('_') or name in blacklist: + continue + obj = getattr(module, name) + if (getattr(obj, '__module__', None) in name_of_module or + (not hasattr(obj, '__module__') and + not isinstance(obj, types.ModuleType))): + expected.add(name) + test_case.assertCountEqual(module.__all__, expected) + + class SuppressCrashReport: """Try to prevent a crash report from popping up. diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index de610c752a..3a94383103 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -440,6 +440,12 @@ class GettextCacheTestCase(GettextBaseTest): self.assertEqual(t.__class__, DummyGNUTranslations) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'c2py', 'ENOENT'} + support.check__all__(self, gettext, blacklist=blacklist) + + def test_main(): support.run_unittest(__name__) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 2c00417414..a9ba460a2a 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -312,6 +312,28 @@ class TestSupport(unittest.TestCase): self.OtherClass, self.RefClass, ignore=ignore) self.assertEqual(set(), missing_items) + def test_check__all__(self): + extra = {'tempdir'} + blacklist = {'template'} + support.check__all__(self, + tempfile, + extra=extra, + blacklist=blacklist) + + extra = {'TextTestResult', 'installHandler'} + blacklist = {'load_tests', "TestProgram", "BaseTestSuite"} + + support.check__all__(self, + unittest, + ("unittest.result", "unittest.case", + "unittest.suite", "unittest.loader", + "unittest.main", "unittest.runner", + "unittest.signals"), + extra=extra, + blacklist=blacklist) + + self.assertRaises(AssertionError, support.check__all__, self, unittest) + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled -- cgit v1.2.1 From 47bfb98000c9ebaa9f64d9b9f80e80a456279c6b Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 14 Nov 2015 12:46:42 +0000 Subject: Issue #23883: Add missing APIs to __all__; patch by Jacek Ko?odziej --- Lib/csv.py | 11 ++++++----- Lib/enum.py | 2 +- Lib/ftplib.py | 3 ++- Lib/logging/__init__.py | 5 +++-- Lib/optparse.py | 3 ++- Lib/test/test_csv.py | 6 ++++++ Lib/test/test_enum.py | 7 +++++++ Lib/test/test_ftplib.py | 11 ++++++++++- Lib/test/test_logging.py | 14 +++++++++++++- Lib/test/test_optparse.py | 7 +++++++ Lib/test/test_pickletools.py | 33 +++++++++++++++++++++++++++++++++ Lib/test/test_threading.py | 8 ++++++++ Lib/test/test_wave.py | 7 +++++++ Lib/threading.py | 8 +++++--- Lib/wave.py | 2 +- 15 files changed, 111 insertions(+), 16 deletions(-) (limited to 'Lib') diff --git a/Lib/csv.py b/Lib/csv.py index ca40e5e0ef..90461dbe1e 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -13,11 +13,12 @@ from _csv import Dialect as _Dialect from io import StringIO -__all__ = [ "QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", - "Error", "Dialect", "__doc__", "excel", "excel_tab", - "field_size_limit", "reader", "writer", - "register_dialect", "get_dialect", "list_dialects", "Sniffer", - "unregister_dialect", "__version__", "DictReader", "DictWriter" ] +__all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", + "Error", "Dialect", "__doc__", "excel", "excel_tab", + "field_size_limit", "reader", "writer", + "register_dialect", "get_dialect", "list_dialects", "Sniffer", + "unregister_dialect", "__version__", "DictReader", "DictWriter", + "unix_dialect"] class Dialect: """Describe a CSV dialect. diff --git a/Lib/enum.py b/Lib/enum.py index 8d04e2d718..35a9c77935 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -8,7 +8,7 @@ except ImportError: from collections import OrderedDict -__all__ = ['Enum', 'IntEnum', 'unique'] +__all__ = ['EnumMeta', 'Enum', 'IntEnum', 'unique'] def _is_descriptor(obj): diff --git a/Lib/ftplib.py b/Lib/ftplib.py index c416d8562b..2afa19de43 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -42,7 +42,8 @@ import socket import warnings from socket import _GLOBAL_DEFAULT_TIMEOUT -__all__ = ["FTP"] +__all__ = ["FTP", "error_reply", "error_temp", "error_perm", "error_proto", + "all_errors"] # Magic number from MSG_OOB = 0x1 # Process data out of band diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 104b0be8d0..369d2c342a 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -33,8 +33,9 @@ __all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR', 'StreamHandler', 'WARN', 'WARNING', 'addLevelName', 'basicConfig', 'captureWarnings', 'critical', 'debug', 'disable', 'error', 'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass', - 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'warn', 'warning', - 'getLogRecordFactory', 'setLogRecordFactory', 'lastResort'] + 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown', + 'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory', + 'lastResort', 'raiseExceptions'] try: import threading diff --git a/Lib/optparse.py b/Lib/optparse.py index 432a2eb9b6..d239ea27d9 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -38,7 +38,8 @@ __all__ = ['Option', 'OptionError', 'OptionConflictError', 'OptionValueError', - 'BadOptionError'] + 'BadOptionError', + 'check_choice'] __copyright__ = """ Copyright (c) 2001-2006 Gregory P. Ward. All rights reserved. diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 8e9c2b479a..4763bbbfb7 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1084,5 +1084,11 @@ class TestUnicode(unittest.TestCase): self.assertEqual(fileobj.read(), expected) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + extra = {'__doc__', '__version__'} + support.check__all__(self, csv, ('csv', '_csv'), extra=extra) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 0f7b769b7a..e4e6c2b51a 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -6,6 +6,7 @@ from collections import OrderedDict from enum import Enum, IntEnum, EnumMeta, unique from io import StringIO from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL +from test import support # for pickle tests try: @@ -1708,5 +1709,11 @@ class TestStdLib(unittest.TestCase): if failed: self.fail("result does not equal expected, see print above") + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, enum) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index aef66da98a..9d8de211df 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -1049,10 +1049,19 @@ class TestTimeouts(TestCase): ftp.close() +class MiscTestCase(TestCase): + def test__all__(self): + blacklist = {'MSG_OOB', 'FTP_PORT', 'MAXLINE', 'CRLF', 'B_CRLF', + 'Error', 'parse150', 'parse227', 'parse229', 'parse257', + 'print_line', 'ftpcp', 'test'} + support.check__all__(self, ftplib, blacklist=blacklist) + + def test_main(): tests = [TestFTPClass, TestTimeouts, TestIPv6Environment, - TestTLS_FTPClassMixin, TestTLS_FTPClass] + TestTLS_FTPClassMixin, TestTLS_FTPClass, + MiscTestCase] thread_info = support.threading_setup() try: diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 95575bf56a..9c4344f99a 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4159,6 +4159,17 @@ class NTEventLogHandlerTest(BaseTest): msg = 'Record not found in event log, went back %d records' % GO_BACK self.assertTrue(found, msg=msg) + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'logThreads', 'logMultiprocessing', + 'logProcesses', 'currentframe', + 'PercentStyle', 'StrFormatStyle', 'StringTemplateStyle', + 'Filterer', 'PlaceHolder', 'Manager', 'RootLogger', + 'root'} + support.check__all__(self, logging, blacklist=blacklist) + + # Set the locale to the platform-dependent default. I have no idea # why the test does this, but in any case we save the current locale # first and restore it at the end. @@ -4175,7 +4186,8 @@ def test_main(): RotatingFileHandlerTest, LastResortTest, LogRecordTest, ExceptionTest, SysLogHandlerTest, HTTPHandlerTest, NTEventLogHandlerTest, TimedRotatingFileHandlerTest, - UnixSocketHandlerTest, UnixDatagramHandlerTest, UnixSysLogHandlerTest) + UnixSocketHandlerTest, UnixDatagramHandlerTest, UnixSysLogHandlerTest, + MiscTestCase) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index 7621c24305..91a0319a73 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -16,6 +16,7 @@ from io import StringIO from test import support +import optparse from optparse import make_option, Option, \ TitledHelpFormatter, OptionParser, OptionGroup, \ SUPPRESS_USAGE, OptionError, OptionConflictError, \ @@ -1650,6 +1651,12 @@ class TestParseNumber(BaseTest): "option -l: invalid integer value: '0x12x'") +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'check_builtin', 'AmbiguousOptionError', 'NO_DEFAULT'} + support.check__all__(self, optparse, blacklist=blacklist) + + def test_main(): support.run_unittest(__name__) diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index bbe6875545..80221f005c 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -4,6 +4,7 @@ import pickletools from test import support from test.pickletester import AbstractPickleTests from test.pickletester import AbstractPickleModuleTests +import unittest class OptimizedPickleTests(AbstractPickleTests, AbstractPickleModuleTests): @@ -59,8 +60,40 @@ class OptimizedPickleTests(AbstractPickleTests, AbstractPickleModuleTests): self.assertNotIn(pickle.BINPUT, pickled2) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'bytes_types', + 'UP_TO_NEWLINE', 'TAKEN_FROM_ARGUMENT1', + 'TAKEN_FROM_ARGUMENT4', 'TAKEN_FROM_ARGUMENT4U', + 'TAKEN_FROM_ARGUMENT8U', 'ArgumentDescriptor', + 'read_uint1', 'read_uint2', 'read_int4', 'read_uint4', + 'read_uint8', 'read_stringnl', 'read_stringnl_noescape', + 'read_stringnl_noescape_pair', 'read_string1', + 'read_string4', 'read_bytes1', 'read_bytes4', + 'read_bytes8', 'read_unicodestringnl', + 'read_unicodestring1', 'read_unicodestring4', + 'read_unicodestring8', 'read_decimalnl_short', + 'read_decimalnl_long', 'read_floatnl', 'read_float8', + 'read_long1', 'read_long4', + 'uint1', 'uint2', 'int4', 'uint4', 'uint8', 'stringnl', + 'stringnl_noescape', 'stringnl_noescape_pair', 'string1', + 'string4', 'bytes1', 'bytes4', 'bytes8', + 'unicodestringnl', 'unicodestring1', 'unicodestring4', + 'unicodestring8', 'decimalnl_short', 'decimalnl_long', + 'floatnl', 'float8', 'long1', 'long4', + 'StackObject', + 'pyint', 'pylong', 'pyinteger_or_bool', 'pybool', 'pyfloat', + 'pybytes_or_str', 'pystring', 'pybytes', 'pyunicode', + 'pynone', 'pytuple', 'pylist', 'pydict', 'pyset', + 'pyfrozenset', 'anyobject', 'markobject', 'stackslice', + 'OpcodeInfo', 'opcodes', 'code2op', + } + support.check__all__(self, pickletools, blacklist=blacklist) + + def test_main(): support.run_unittest(OptimizedPickleTests) + support.run_unittest(MiscTestCase) support.run_doctest(pickletools) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 3b11bf6508..45564f71ef 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -18,6 +18,7 @@ import os import subprocess from test import lock_tests +from test import support # Between fork() and exec(), only async-safe functions are allowed (issues @@ -1098,5 +1099,12 @@ class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): class BarrierTests(lock_tests.BarrierTests): barriertype = staticmethod(threading.Barrier) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + extra = {"ThreadError"} + blacklist = {'currentThread', 'activeCount'} + support.check__all__(self, threading, ('threading', '_thread'), + extra=extra, blacklist=blacklist) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index 3eff773bca..a67a8b009e 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -1,6 +1,7 @@ from test.support import TESTFN import unittest from test import audiotests +from test import support from audioop import byteswap import sys import wave @@ -103,5 +104,11 @@ class WavePCM32Test(WaveTest, unittest.TestCase): frames = byteswap(frames, 4) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'WAVE_FORMAT_PCM'} + support.check__all__(self, wave, blacklist=blacklist) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/threading.py b/Lib/threading.py index 828019d441..2bf33c12ae 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -22,9 +22,11 @@ except ImportError: # with the multiprocessing module, which doesn't provide the old # Java inspired names. -__all__ = ['active_count', 'Condition', 'current_thread', 'enumerate', 'Event', - 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', 'Barrier', - 'Timer', 'ThreadError', 'setprofile', 'settrace', 'local', 'stack_size'] +__all__ = ['get_ident', 'active_count', 'Condition', 'current_thread', + 'enumerate', 'main_thread', 'TIMEOUT_MAX', + 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', + 'Barrier', 'BrokenBarrierError', 'Timer', 'ThreadError', + 'setprofile', 'settrace', 'local', 'stack_size'] # Rename some stuff so "from threading import *" is safe _start_new_thread = _thread.start_new_thread diff --git a/Lib/wave.py b/Lib/wave.py index 8a101e320b..f71f7e5bf9 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -73,7 +73,7 @@ is destroyed. import builtins -__all__ = ["open", "openfp", "Error"] +__all__ = ["open", "openfp", "Error", "Wave_read", "Wave_write"] class Error(Exception): pass -- cgit v1.2.1 From c62097e1719ca9df0498cbeb1fd2d72e53ceabe4 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Mon, 16 Nov 2015 09:22:19 +0000 Subject: Issue #20220: Revert time zone test debugging, revision 139c18943d9b --- Lib/test/datetimetester.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) (limited to 'Lib') diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index f814f23d23..7374608a66 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2006,16 +2006,7 @@ class TestDateTime(TestDate): seconds = tzseconds hours, minutes = divmod(seconds//60, 60) dtstr = "{}{:02d}{:02d} {}".format(sign, hours, minutes, tzname) - try: - dt = strptime(dtstr, "%z %Z") - except ValueError: - import os - self.fail( - "Issue #25168 strptime() failure info:\n" - f"_TimeRE_cache['Z']={_strptime._TimeRE_cache['Z']!r}\n" - f"TZ={os.environ.get('TZ')!r}, or {os.getenv('TZ')!r} via getenv()\n" - f"_regex_cache={_strptime._regex_cache!r}\n" - ) + dt = strptime(dtstr, "%z %Z") self.assertEqual(dt.utcoffset(), timedelta(seconds=tzseconds)) self.assertEqual(dt.tzname(), tzname) # Can produce inconsistent datetime -- cgit v1.2.1 From 35cd28d57125356d69cac0be1dce1387b64599ab Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 22 Nov 2015 15:18:54 +0100 Subject: Issue #25694: Fix test_regrtest for installed Python --- Lib/test/test_regrtest.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ab7741f3bf..59f8c9d033 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -14,6 +14,7 @@ import platform import re import subprocess import sys +import sysconfig import textwrap import unittest from test import libregrtest @@ -306,7 +307,7 @@ class BaseTestCase(unittest.TestCase): TESTNAME_REGEX = r'test_[a-z0-9_]+' def setUp(self): - self.testdir = os.path.join(ROOT_DIR, 'Lib', 'test') + self.testdir = os.path.realpath(os.path.dirname(__file__)) # When test_regrtest is interrupted by CTRL+c, it can leave # temporary test files @@ -330,8 +331,13 @@ class BaseTestCase(unittest.TestCase): self.addCleanup(support.unlink, path) # Use 'x' mode to ensure that we do not override existing tests - with open(path, 'x', encoding='utf-8') as fp: - fp.write(code) + try: + with open(path, 'x', encoding='utf-8') as fp: + fp.write(code) + except PermissionError as exc: + if not sysconfig.is_python_build(): + self.skipTest("cannot write %s: %s" % (path, exc)) + raise return name def regex_search(self, regex, output): @@ -471,7 +477,7 @@ class ProgramsTestCase(BaseTestCase): def test_script_regrtest(self): # Lib/test/regrtest.py - script = os.path.join(ROOT_DIR, 'Lib', 'test', 'regrtest.py') + script = os.path.join(self.testdir, 'regrtest.py') args = [*self.python_args, script, *self.regrtest_args, *self.tests] self.run_tests(args) @@ -503,10 +509,12 @@ class ProgramsTestCase(BaseTestCase): def test_script_autotest(self): # Lib/test/autotest.py - script = os.path.join(ROOT_DIR, 'Lib', 'test', 'autotest.py') + script = os.path.join(self.testdir, 'autotest.py') args = [*self.python_args, script, *self.regrtest_args, *self.tests] self.run_tests(args) + @unittest.skipUnless(sysconfig.is_python_build(), + 'run_tests.py script is not installed') def test_tools_script_run_tests(self): # Tools/scripts/run_tests.py script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') @@ -516,6 +524,8 @@ class ProgramsTestCase(BaseTestCase): proc = self.run_command(args) self.check_output(proc.stdout) + @unittest.skipUnless(sysconfig.is_python_build(), + 'test.bat script is not installed') @unittest.skipUnless(sys.platform == 'win32', 'Windows only') def test_tools_buildbot_test(self): # Tools\buildbot\test.bat -- cgit v1.2.1 From a86dffe49ce6ebfcb6ad5dc86f8b21affd25b99c Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Tue, 24 Nov 2015 00:20:00 +0000 Subject: Issue #25663: Update rlcompleter test for new 3.6 behaviour --- Lib/test/test_rlcompleter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 34a5eff7f7..d9e4de6deb 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -124,9 +124,10 @@ class TestRlcompleter(unittest.TestCase): completer = rlcompleter.Completer(namespace) self.assertEqual(completer.complete('False', 0), 'False') self.assertIsNone(completer.complete('False', 1)) # No duplicates - self.assertEqual(completer.complete('assert', 0), 'assert') + # Space or colon added due to being a reserved keyword + self.assertEqual(completer.complete('assert', 0), 'assert ') self.assertIsNone(completer.complete('assert', 1)) - self.assertEqual(completer.complete('try', 0), 'try') + self.assertEqual(completer.complete('try', 0), 'try:') self.assertIsNone(completer.complete('try', 1)) # No opening bracket "(" because we overrode the built-in class self.assertEqual(completer.complete('memoryview', 0), 'memoryview') -- cgit v1.2.1 From cd93b15005e9d8120c722580173be049def3acfe Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Tue, 24 Nov 2015 22:12:05 +0000 Subject: Issue #25695: Defer creation of TESTDIRN until the test case is run --- Lib/test/test_support.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index a9ba460a2a..f86ea918e1 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -9,13 +9,11 @@ import errno from test import support TESTFN = support.TESTFN -TESTDIRN = os.path.basename(tempfile.mkdtemp(dir='.')) class TestSupport(unittest.TestCase): def setUp(self): support.unlink(TESTFN) - support.rmtree(TESTDIRN) tearDown = setUp def test_import_module(self): @@ -48,6 +46,10 @@ class TestSupport(unittest.TestCase): support.unlink(TESTFN) def test_rmtree(self): + TESTDIRN = os.path.basename(tempfile.mkdtemp(dir='.')) + self.addCleanup(support.rmtree, TESTDIRN) + support.rmtree(TESTDIRN) + os.mkdir(TESTDIRN) os.mkdir(os.path.join(TESTDIRN, TESTDIRN)) support.rmtree(TESTDIRN) -- cgit v1.2.1 From 5b9914ed5262a882872108edab1ff16ff7648e55 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Tue, 24 Nov 2015 23:00:37 +0000 Subject: Issue #25576: Apply fix to new urlopen() doc string --- Lib/urllib/request.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'Lib') diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index e6abf34fa0..57d0dea075 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -149,13 +149,8 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, *data* should be a buffer in the standard application/x-www-form-urlencoded format. The urllib.parse.urlencode() function takes a mapping or sequence - of 2-tuples and returns a string in this format. It should be encoded to - bytes before being used as the data parameter. The charset parameter in - Content-Type header may be used to specify the encoding. If charset - parameter is not sent with the Content-Type header, the server following - the HTTP 1.1 recommendation may assume that the data is encoded in - ISO-8859-1 encoding. It is advisable to use charset parameter with encoding - used in Content-Type header with the Request. + of 2-tuples and returns an ASCII text string in this format. It should be + encoded to bytes before being used as the data parameter. urllib.request module uses HTTP/1.1 and includes a "Connection:close" header in its HTTP requests. -- cgit v1.2.1 From 822041ddbdaad1d66641fae961f3da48bf5c140d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 25 Nov 2015 15:28:13 +0200 Subject: Issue #7990: dir() on ElementTree.Element now lists properties: "tag", "text", "tail" and "attrib". Original patch by Santoso Wijaya. --- Lib/test/test_xml_etree.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 57d8e4d2de..029320153e 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -182,10 +182,12 @@ class ElementTreeTest(unittest.TestCase): def check_element(element): self.assertTrue(ET.iselement(element), msg="not an element") - self.assertTrue(hasattr(element, "tag"), msg="no tag member") - self.assertTrue(hasattr(element, "attrib"), msg="no attrib member") - self.assertTrue(hasattr(element, "text"), msg="no text member") - self.assertTrue(hasattr(element, "tail"), msg="no tail member") + direlem = dir(element) + for attr in 'tag', 'attrib', 'text', 'tail': + self.assertTrue(hasattr(element, attr), + msg='no %s member' % attr) + self.assertIn(attr, direlem, + msg='no %s visible by dir' % attr) check_string(element.tag) check_mapping(element.attrib) -- cgit v1.2.1 From 5e8b2c1d8d1725557255df8cc4858157c9a3fbb6 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Thu, 26 Nov 2015 02:36:26 +0000 Subject: Issue #25622: Rename to PythonValuesTestCase and enable for non-Windows --- Lib/ctypes/test/test_values.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/ctypes/test/test_values.py b/Lib/ctypes/test/test_values.py index 9551e7ae9f..c7c78ce4e5 100644 --- a/Lib/ctypes/test/test_values.py +++ b/Lib/ctypes/test/test_values.py @@ -28,8 +28,7 @@ class ValuesTestCase(unittest.TestCase): ctdll = CDLL(_ctypes_test.__file__) self.assertRaises(ValueError, c_int.in_dll, ctdll, "Undefined_Symbol") -@unittest.skipUnless(sys.platform == 'win32', 'Windows-specific test') -class Win_ValuesTestCase(unittest.TestCase): +class PythonValuesTestCase(unittest.TestCase): """This test only works when python itself is a dll/shared library""" def test_optimizeflag(self): @@ -76,7 +75,7 @@ class Win_ValuesTestCase(unittest.TestCase): if entry.name in bootstrap_expected: bootstrap_seen.append(entry.name) self.assertTrue(entry.size, - "{} was reported as having no size".format(entry.name)) + "{!r} was reported as having no size".format(entry.name)) continue items.append((entry.name, entry.size)) -- cgit v1.2.1 From b00885aeb66804b25d8ae492e0608ea214a0d421 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sat, 28 Nov 2015 12:24:52 -0500 Subject: #25485: Add context manager support to Telnet class. Patch by St?phane Wirtel. --- Lib/telnetlib.py | 15 ++++++++++----- Lib/test/test_telnetlib.py | 5 +++++ 2 files changed, 15 insertions(+), 5 deletions(-) (limited to 'Lib') diff --git a/Lib/telnetlib.py b/Lib/telnetlib.py index 72dabc76e0..b0863b1cbd 100644 --- a/Lib/telnetlib.py +++ b/Lib/telnetlib.py @@ -637,6 +637,12 @@ class Telnet: raise EOFError return (-1, None, text) + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + def test(): """Test program for telnetlib. @@ -660,11 +666,10 @@ def test(): port = int(portstr) except ValueError: port = socket.getservbyname(portstr, 'tcp') - tn = Telnet() - tn.set_debuglevel(debuglevel) - tn.open(host, port, timeout=0.5) - tn.interact() - tn.close() + with Telnet() as tn: + tn.set_debuglevel(debuglevel) + tn.open(host, port, timeout=0.5) + tn.interact() if __name__ == '__main__': test() diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py index 524bba37b3..23029e0c6f 100644 --- a/Lib/test/test_telnetlib.py +++ b/Lib/test/test_telnetlib.py @@ -42,6 +42,11 @@ class GeneralTests(TestCase): telnet = telnetlib.Telnet(HOST, self.port) telnet.sock.close() + def testContextManager(self): + with telnetlib.Telnet(HOST, self.port) as tn: + self.assertIsNotNone(tn.get_socket()) + self.assertIsNone(tn.get_socket()) + def testTimeoutDefault(self): self.assertTrue(socket.getdefaulttimeout() is None) socket.setdefaulttimeout(30) -- cgit v1.2.1 From ad946e74b8d6bd4e7cdacd6e075da0c54b5d99c2 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 28 Nov 2015 22:38:24 +0000 Subject: Issue #25754: Allow test_rlcompleter to be run multiple times --- Lib/test/test_rlcompleter.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index d9e4de6deb..208c0545c4 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -1,4 +1,5 @@ import unittest +from unittest.mock import patch import builtins import rlcompleter @@ -72,12 +73,12 @@ class TestRlcompleter(unittest.TestCase): self.assertIn('CompleteMe.__name__', matches) self.assertIn('CompleteMe.__new__(', matches) - CompleteMe.me = CompleteMe - self.assertEqual(self.completer.attr_matches('CompleteMe.me.me.sp'), - ['CompleteMe.me.me.spam']) - self.assertEqual(self.completer.attr_matches('egg.s'), - ['egg.{}('.format(x) for x in dir(str) - if x.startswith('s')]) + with patch.object(CompleteMe, "me", CompleteMe, create=True): + self.assertEqual(self.completer.attr_matches('CompleteMe.me.me.sp'), + ['CompleteMe.me.me.spam']) + self.assertEqual(self.completer.attr_matches('egg.s'), + ['egg.{}('.format(x) for x in dir(str) + if x.startswith('s')]) def test_excessive_getattr(self): # Ensure getattr() is invoked no more than once per attribute -- cgit v1.2.1 From 0b2be356380b91c945a044ddd261b5c8c3c264b5 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Mon, 30 Nov 2015 03:18:29 +0000 Subject: Issue #5319: New Py_FinalizeEx() API to exit with status 120 on failure --- Lib/test/test_cmd_line.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 0feb63fd4e..b4106082cf 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -348,8 +348,9 @@ class CmdLineTest(unittest.TestCase): test.support.SuppressCrashReport().__enter__() sys.stdout.write('x') os.close(sys.stdout.fileno())""" - rc, out, err = assert_python_ok('-c', code) + rc, out, err = assert_python_failure('-c', code) self.assertEqual(b'', out) + self.assertEqual(120, rc) self.assertRegex(err.decode('ascii', 'ignore'), 'Exception ignored in.*\nOSError: .*') -- cgit v1.2.1 From 8297f2cb959262f4bb131bfad5a7830a63e0e1b1 Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Tue, 1 Dec 2015 19:59:53 +1100 Subject: Issue #25177: Fixed problem with the mean of very small and very large numbers. --- Lib/statistics.py | 185 +++++++++++++--------- Lib/test/test_statistics.py | 363 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 429 insertions(+), 119 deletions(-) (limited to 'Lib') diff --git a/Lib/statistics.py b/Lib/statistics.py index 9203cf1082..518f546544 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -104,6 +104,8 @@ import math from fractions import Fraction from decimal import Decimal +from itertools import groupby + # === Exceptions === @@ -115,86 +117,102 @@ class StatisticsError(ValueError): # === Private utilities === def _sum(data, start=0): - """_sum(data [, start]) -> value + """_sum(data [, start]) -> (type, sum, count) + + Return a high-precision sum of the given numeric data as a fraction, + together with the type to be converted to and the count of items. - Return a high-precision sum of the given numeric data. If optional - argument ``start`` is given, it is added to the total. If ``data`` is - empty, ``start`` (defaulting to 0) is returned. + If optional argument ``start`` is given, it is added to the total. + If ``data`` is empty, ``start`` (defaulting to 0) is returned. Examples -------- >>> _sum([3, 2.25, 4.5, -0.5, 1.0], 0.75) - 11.0 + (, Fraction(11, 1), 5) Some sources of round-off error will be avoided: >>> _sum([1e50, 1, -1e50] * 1000) # Built-in sum returns zero. - 1000.0 + (, Fraction(1000, 1), 3000) Fractions and Decimals are also supported: >>> from fractions import Fraction as F >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) - Fraction(63, 20) + (, Fraction(63, 20), 4) >>> from decimal import Decimal as D >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] >>> _sum(data) - Decimal('0.6963') + (, Fraction(6963, 10000), 4) Mixed types are currently treated as an error, except that int is allowed. """ - # We fail as soon as we reach a value that is not an int or the type of - # the first value which is not an int. E.g. _sum([int, int, float, int]) - # is okay, but sum([int, int, float, Fraction]) is not. - allowed_types = {int, type(start)} + count = 0 n, d = _exact_ratio(start) - partials = {d: n} # map {denominator: sum of numerators} - # Micro-optimizations. - exact_ratio = _exact_ratio + partials = {d: n} partials_get = partials.get - # Add numerators for each denominator. - for x in data: - _check_type(type(x), allowed_types) - n, d = exact_ratio(x) - partials[d] = partials_get(d, 0) + n - # Find the expected result type. If allowed_types has only one item, it - # will be int; if it has two, use the one which isn't int. - assert len(allowed_types) in (1, 2) - if len(allowed_types) == 1: - assert allowed_types.pop() is int - T = int - else: - T = (allowed_types - {int}).pop() + T = _coerce(int, type(start)) + for typ, values in groupby(data, type): + T = _coerce(T, typ) # or raise TypeError + for n,d in map(_exact_ratio, values): + count += 1 + partials[d] = partials_get(d, 0) + n if None in partials: - assert issubclass(T, (float, Decimal)) - assert not math.isfinite(partials[None]) - return T(partials[None]) - total = Fraction() - for d, n in sorted(partials.items()): - total += Fraction(n, d) - if issubclass(T, int): - assert total.denominator == 1 - return T(total.numerator) - if issubclass(T, Decimal): - return T(total.numerator)/total.denominator - return T(total) - - -def _check_type(T, allowed): - if T not in allowed: - if len(allowed) == 1: - allowed.add(T) - else: - types = ', '.join([t.__name__ for t in allowed] + [T.__name__]) - raise TypeError("unsupported mixed types: %s" % types) + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + total = partials[None] + assert not _isfinite(total) + else: + # Sum all the partial sums using builtin sum. + # FIXME is this faster if we sum them in order of the denominator? + total = sum(Fraction(n, d) for d, n in sorted(partials.items())) + return (T, total, count) + + +def _isfinite(x): + try: + return x.is_finite() # Likely a Decimal. + except AttributeError: + return math.isfinite(x) # Coerces to float first. + + +def _coerce(T, S): + """Coerce types T and S to a common type, or raise TypeError. + + Coercion rules are currently an implementation detail. See the CoerceTest + test class in test_statistics for details. + """ + # See http://bugs.python.org/issue24068. + assert T is not bool, "initial type T is bool" + # If the types are the same, no need to coerce anything. Put this + # first, so that the usual case (no coercion needed) happens as soon + # as possible. + if T is S: return T + # Mixed int & other coerce to the other type. + if S is int or S is bool: return T + if T is int: return S + # If one is a (strict) subclass of the other, coerce to the subclass. + if issubclass(S, T): return S + if issubclass(T, S): return T + # Ints coerce to the other type. + if issubclass(T, int): return S + if issubclass(S, int): return T + # Mixed fraction & float coerces to float (or float subclass). + if issubclass(T, Fraction) and issubclass(S, float): + return S + if issubclass(T, float) and issubclass(S, Fraction): + return T + # Any other combination is disallowed. + msg = "don't know how to coerce %s and %s" + raise TypeError(msg % (T.__name__, S.__name__)) def _exact_ratio(x): - """Convert Real number x exactly to (numerator, denominator) pair. + """Return Real number x to exact (numerator, denominator) pair. >>> _exact_ratio(0.25) (1, 4) @@ -202,29 +220,31 @@ def _exact_ratio(x): x is expected to be an int, Fraction, Decimal or float. """ try: + # Optimise the common case of floats. We expect that the most often + # used numeric type will be builtin floats, so try to make this as + # fast as possible. + if type(x) is float: + return x.as_integer_ratio() try: - # int, Fraction + # x may be an int, Fraction, or Integral ABC. return (x.numerator, x.denominator) except AttributeError: - # float try: + # x may be a float subclass. return x.as_integer_ratio() except AttributeError: - # Decimal try: + # x may be a Decimal. return _decimal_to_ratio(x) except AttributeError: - msg = "can't convert type '{}' to numerator/denominator" - raise TypeError(msg.format(type(x).__name__)) from None + # Just give up? + pass except (OverflowError, ValueError): - # INF or NAN - if __debug__: - # Decimal signalling NANs cannot be converted to float :-( - if isinstance(x, Decimal): - assert not x.is_finite() - else: - assert not math.isfinite(x) + # float NAN or INF. + assert not math.isfinite(x) return (x, None) + msg = "can't convert type '{}' to numerator/denominator" + raise TypeError(msg.format(type(x).__name__)) # FIXME This is faster than Fraction.from_decimal, but still too slow. @@ -239,7 +259,7 @@ def _decimal_to_ratio(d): sign, digits, exp = d.as_tuple() if exp in ('F', 'n', 'N'): # INF, NAN, sNAN assert not d.is_finite() - raise ValueError + return (d, None) num = 0 for digit in digits: num = num*10 + digit @@ -253,6 +273,24 @@ def _decimal_to_ratio(d): return (num, den) +def _convert(value, T): + """Convert value to given numeric type T.""" + if type(value) is T: + # This covers the cases where T is Fraction, or where value is + # a NAN or INF (Decimal or float). + return value + if issubclass(T, int) and value.denominator != 1: + T = float + try: + # FIXME: what do we do if this overflows? + return T(value) + except TypeError: + if issubclass(T, Decimal): + return T(value.numerator)/T(value.denominator) + else: + raise + + def _counts(data): # Generate a table of sorted (value, frequency) pairs. table = collections.Counter(iter(data)).most_common() @@ -290,7 +328,9 @@ def mean(data): n = len(data) if n < 1: raise StatisticsError('mean requires at least one data point') - return _sum(data)/n + T, total, count = _sum(data) + assert count == n + return _convert(total/n, T) # FIXME: investigate ways to calculate medians without sorting? Quickselect? @@ -460,12 +500,14 @@ def _ss(data, c=None): """ if c is None: c = mean(data) - ss = _sum((x-c)**2 for x in data) + T, total, count = _sum((x-c)**2 for x in data) # The following sum should mathematically equal zero, but due to rounding # error may not. - ss -= _sum((x-c) for x in data)**2/len(data) - assert not ss < 0, 'negative sum of square deviations: %f' % ss - return ss + U, total2, count2 = _sum((x-c) for x in data) + assert T == U and count == count2 + total -= total2**2/len(data) + assert not total < 0, 'negative sum of square deviations: %f' % total + return (T, total) def variance(data, xbar=None): @@ -511,8 +553,8 @@ def variance(data, xbar=None): n = len(data) if n < 2: raise StatisticsError('variance requires at least two data points') - ss = _ss(data, xbar) - return ss/(n-1) + T, ss = _ss(data, xbar) + return _convert(ss/(n-1), T) def pvariance(data, mu=None): @@ -560,7 +602,8 @@ def pvariance(data, mu=None): if n < 1: raise StatisticsError('pvariance requires at least one data point') ss = _ss(data, mu) - return ss/n + T, ss = _ss(data, mu) + return _convert(ss/n, T) def stdev(data, xbar=None): diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 758a481fe7..0089ae8dc6 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -21,6 +21,37 @@ import statistics # === Helper functions and class === +def _nan_equal(a, b): + """Return True if a and b are both the same kind of NAN. + + >>> _nan_equal(Decimal('NAN'), Decimal('NAN')) + True + >>> _nan_equal(Decimal('sNAN'), Decimal('sNAN')) + True + >>> _nan_equal(Decimal('NAN'), Decimal('sNAN')) + False + >>> _nan_equal(Decimal(42), Decimal('NAN')) + False + + >>> _nan_equal(float('NAN'), float('NAN')) + True + >>> _nan_equal(float('NAN'), 0.5) + False + + >>> _nan_equal(float('NAN'), Decimal('NAN')) + False + + NAN payloads are not compared. + """ + if type(a) is not type(b): + return False + if isinstance(a, float): + return math.isnan(a) and math.isnan(b) + aexp = a.as_tuple()[2] + bexp = b.as_tuple()[2] + return (aexp == bexp) and (aexp in ('n', 'N')) # Both NAN or both sNAN. + + def _calc_errors(actual, expected): """Return the absolute and relative errors between two numbers. @@ -675,15 +706,60 @@ class ExactRatioTest(unittest.TestCase): self.assertEqual(_exact_ratio(D("12.345")), (12345, 1000)) self.assertEqual(_exact_ratio(D("-1.98")), (-198, 100)) + def test_inf(self): + INF = float("INF") + class MyFloat(float): + pass + class MyDecimal(Decimal): + pass + for inf in (INF, -INF): + for type_ in (float, MyFloat, Decimal, MyDecimal): + x = type_(inf) + ratio = statistics._exact_ratio(x) + self.assertEqual(ratio, (x, None)) + self.assertEqual(type(ratio[0]), type_) + self.assertTrue(math.isinf(ratio[0])) + + def test_float_nan(self): + NAN = float("NAN") + class MyFloat(float): + pass + for nan in (NAN, MyFloat(NAN)): + ratio = statistics._exact_ratio(nan) + self.assertTrue(math.isnan(ratio[0])) + self.assertIs(ratio[1], None) + self.assertEqual(type(ratio[0]), type(nan)) + + def test_decimal_nan(self): + NAN = Decimal("NAN") + sNAN = Decimal("sNAN") + class MyDecimal(Decimal): + pass + for nan in (NAN, MyDecimal(NAN), sNAN, MyDecimal(sNAN)): + ratio = statistics._exact_ratio(nan) + self.assertTrue(_nan_equal(ratio[0], nan)) + self.assertIs(ratio[1], None) + self.assertEqual(type(ratio[0]), type(nan)) + class DecimalToRatioTest(unittest.TestCase): # Test _decimal_to_ratio private function. - def testSpecialsRaise(self): - # Test that NANs and INFs raise ValueError. - # Non-special values are covered by _exact_ratio above. - for d in (Decimal('NAN'), Decimal('sNAN'), Decimal('INF')): - self.assertRaises(ValueError, statistics._decimal_to_ratio, d) + def test_infinity(self): + # Test that INFs are handled correctly. + inf = Decimal('INF') + self.assertEqual(statistics._decimal_to_ratio(inf), (inf, None)) + self.assertEqual(statistics._decimal_to_ratio(-inf), (-inf, None)) + + def test_nan(self): + # Test that NANs are handled correctly. + for nan in (Decimal('NAN'), Decimal('sNAN')): + num, den = statistics._decimal_to_ratio(nan) + # Because NANs always compare non-equal, we cannot use assertEqual. + # Nor can we use an identity test, as we don't guarantee anything + # about the object identity. + self.assertTrue(_nan_equal(num, nan)) + self.assertIs(den, None) def test_sign(self): # Test sign is calculated correctly. @@ -718,25 +794,181 @@ class DecimalToRatioTest(unittest.TestCase): self.assertEqual(t, (147000, 1)) -class CheckTypeTest(unittest.TestCase): - # Test _check_type private function. +class IsFiniteTest(unittest.TestCase): + # Test _isfinite private function. + + def test_finite(self): + # Test that finite numbers are recognised as finite. + for x in (5, Fraction(1, 3), 2.5, Decimal("5.5")): + self.assertTrue(statistics._isfinite(x)) - def test_allowed(self): - # Test that a type which should be allowed is allowed. - allowed = set([int, float]) - statistics._check_type(int, allowed) - statistics._check_type(float, allowed) + def test_infinity(self): + # Test that INFs are not recognised as finite. + for x in (float("inf"), Decimal("inf")): + self.assertFalse(statistics._isfinite(x)) + + def test_nan(self): + # Test that NANs are not recognised as finite. + for x in (float("nan"), Decimal("NAN"), Decimal("sNAN")): + self.assertFalse(statistics._isfinite(x)) + + +class CoerceTest(unittest.TestCase): + # Test that private function _coerce correctly deals with types. + + # The coercion rules are currently an implementation detail, although at + # some point that should change. The tests and comments here define the + # correct implementation. + + # Pre-conditions of _coerce: + # + # - The first time _sum calls _coerce, the + # - coerce(T, S) will never be called with bool as the first argument; + # this is a pre-condition, guarded with an assertion. + + # + # - coerce(T, T) will always return T; we assume T is a valid numeric + # type. Violate this assumption at your own risk. + # + # - Apart from as above, bool is treated as if it were actually int. + # + # - coerce(int, X) and coerce(X, int) return X. + # - + def test_bool(self): + # bool is somewhat special, due to the pre-condition that it is + # never given as the first argument to _coerce, and that it cannot + # be subclassed. So we test it specially. + for T in (int, float, Fraction, Decimal): + self.assertIs(statistics._coerce(T, bool), T) + class MyClass(T): pass + self.assertIs(statistics._coerce(MyClass, bool), MyClass) + + def assertCoerceTo(self, A, B): + """Assert that type A coerces to B.""" + self.assertIs(statistics._coerce(A, B), B) + self.assertIs(statistics._coerce(B, A), B) + + def check_coerce_to(self, A, B): + """Checks that type A coerces to B, including subclasses.""" + # Assert that type A is coerced to B. + self.assertCoerceTo(A, B) + # Subclasses of A are also coerced to B. + class SubclassOfA(A): pass + self.assertCoerceTo(SubclassOfA, B) + # A, and subclasses of A, are coerced to subclasses of B. + class SubclassOfB(B): pass + self.assertCoerceTo(A, SubclassOfB) + self.assertCoerceTo(SubclassOfA, SubclassOfB) + + def assertCoerceRaises(self, A, B): + """Assert that coercing A to B, or vice versa, raises TypeError.""" + self.assertRaises(TypeError, statistics._coerce, (A, B)) + self.assertRaises(TypeError, statistics._coerce, (B, A)) + + def check_type_coercions(self, T): + """Check that type T coerces correctly with subclasses of itself.""" + assert T is not bool + # Coercing a type with itself returns the same type. + self.assertIs(statistics._coerce(T, T), T) + # Coercing a type with a subclass of itself returns the subclass. + class U(T): pass + class V(T): pass + class W(U): pass + for typ in (U, V, W): + self.assertCoerceTo(T, typ) + self.assertCoerceTo(U, W) + # Coercing two subclasses that aren't parent/child is an error. + self.assertCoerceRaises(U, V) + self.assertCoerceRaises(V, W) - def test_not_allowed(self): - # Test that a type which should not be allowed raises. - allowed = set([int, float]) - self.assertRaises(TypeError, statistics._check_type, Decimal, allowed) + def test_int(self): + # Check that int coerces correctly. + self.check_type_coercions(int) + for typ in (float, Fraction, Decimal): + self.check_coerce_to(int, typ) - def test_add_to_allowed(self): - # Test that a second type will be added to the allowed set. - allowed = set([int]) - statistics._check_type(float, allowed) - self.assertEqual(allowed, set([int, float])) + def test_fraction(self): + # Check that Fraction coerces correctly. + self.check_type_coercions(Fraction) + self.check_coerce_to(Fraction, float) + + def test_decimal(self): + # Check that Decimal coerces correctly. + self.check_type_coercions(Decimal) + + def test_float(self): + # Check that float coerces correctly. + self.check_type_coercions(float) + + def test_non_numeric_types(self): + for bad_type in (str, list, type(None), tuple, dict): + for good_type in (int, float, Fraction, Decimal): + self.assertCoerceRaises(good_type, bad_type) + + def test_incompatible_types(self): + # Test that incompatible types raise. + for T in (float, Fraction): + class MySubclass(T): pass + self.assertCoerceRaises(T, Decimal) + self.assertCoerceRaises(MySubclass, Decimal) + + +class ConvertTest(unittest.TestCase): + # Test private _convert function. + + def check_exact_equal(self, x, y): + """Check that x equals y, and has the same type as well.""" + self.assertEqual(x, y) + self.assertIs(type(x), type(y)) + + def test_int(self): + # Test conversions to int. + x = statistics._convert(Fraction(71), int) + self.check_exact_equal(x, 71) + class MyInt(int): pass + x = statistics._convert(Fraction(17), MyInt) + self.check_exact_equal(x, MyInt(17)) + + def test_fraction(self): + # Test conversions to Fraction. + x = statistics._convert(Fraction(95, 99), Fraction) + self.check_exact_equal(x, Fraction(95, 99)) + class MyFraction(Fraction): + def __truediv__(self, other): + return self.__class__(super().__truediv__(other)) + x = statistics._convert(Fraction(71, 13), MyFraction) + self.check_exact_equal(x, MyFraction(71, 13)) + + def test_float(self): + # Test conversions to float. + x = statistics._convert(Fraction(-1, 2), float) + self.check_exact_equal(x, -0.5) + class MyFloat(float): + def __truediv__(self, other): + return self.__class__(super().__truediv__(other)) + x = statistics._convert(Fraction(9, 8), MyFloat) + self.check_exact_equal(x, MyFloat(1.125)) + + def test_decimal(self): + # Test conversions to Decimal. + x = statistics._convert(Fraction(1, 40), Decimal) + self.check_exact_equal(x, Decimal("0.025")) + class MyDecimal(Decimal): + def __truediv__(self, other): + return self.__class__(super().__truediv__(other)) + x = statistics._convert(Fraction(-15, 16), MyDecimal) + self.check_exact_equal(x, MyDecimal("-0.9375")) + + def test_inf(self): + for INF in (float('inf'), Decimal('inf')): + for inf in (INF, -INF): + x = statistics._convert(inf, type(inf)) + self.check_exact_equal(x, inf) + + def test_nan(self): + for nan in (float('nan'), Decimal('NAN'), Decimal('sNAN')): + x = statistics._convert(nan, type(nan)) + self.assertTrue(_nan_equal(x, nan)) # === Tests for public functions === @@ -874,52 +1106,71 @@ class UnivariateTypeMixin: self.assertIs(type(result), kind) -class TestSum(NumericTestCase, UnivariateCommonMixin, UnivariateTypeMixin): +class TestSumCommon(UnivariateCommonMixin, UnivariateTypeMixin): + # Common test cases for statistics._sum() function. + + # This test suite looks only at the numeric value returned by _sum, + # after conversion to the appropriate type. + def setUp(self): + def simplified_sum(*args): + T, value, n = statistics._sum(*args) + return statistics._coerce(value, T) + self.func = simplified_sum + + +class TestSum(NumericTestCase): # Test cases for statistics._sum() function. + # These tests look at the entire three value tuple returned by _sum. + def setUp(self): self.func = statistics._sum def test_empty_data(self): # Override test for empty data. for data in ([], (), iter([])): - self.assertEqual(self.func(data), 0) - self.assertEqual(self.func(data, 23), 23) - self.assertEqual(self.func(data, 2.3), 2.3) + self.assertEqual(self.func(data), (int, Fraction(0), 0)) + self.assertEqual(self.func(data, 23), (int, Fraction(23), 0)) + self.assertEqual(self.func(data, 2.3), (float, Fraction(2.3), 0)) def test_ints(self): - self.assertEqual(self.func([1, 5, 3, -4, -8, 20, 42, 1]), 60) - self.assertEqual(self.func([4, 2, 3, -8, 7], 1000), 1008) + self.assertEqual(self.func([1, 5, 3, -4, -8, 20, 42, 1]), + (int, Fraction(60), 8)) + self.assertEqual(self.func([4, 2, 3, -8, 7], 1000), + (int, Fraction(1008), 5)) def test_floats(self): - self.assertEqual(self.func([0.25]*20), 5.0) - self.assertEqual(self.func([0.125, 0.25, 0.5, 0.75], 1.5), 3.125) + self.assertEqual(self.func([0.25]*20), + (float, Fraction(5.0), 20)) + self.assertEqual(self.func([0.125, 0.25, 0.5, 0.75], 1.5), + (float, Fraction(3.125), 4)) def test_fractions(self): - F = Fraction - self.assertEqual(self.func([Fraction(1, 1000)]*500), Fraction(1, 2)) + self.assertEqual(self.func([Fraction(1, 1000)]*500), + (Fraction, Fraction(1, 2), 500)) def test_decimals(self): D = Decimal data = [D("0.001"), D("5.246"), D("1.702"), D("-0.025"), D("3.974"), D("2.328"), D("4.617"), D("2.843"), ] - self.assertEqual(self.func(data), Decimal("20.686")) + self.assertEqual(self.func(data), + (Decimal, Decimal("20.686"), 8)) def test_compare_with_math_fsum(self): # Compare with the math.fsum function. # Ideally we ought to get the exact same result, but sometimes # we differ by a very slight amount :-( data = [random.uniform(-100, 1000) for _ in range(1000)] - self.assertApproxEqual(self.func(data), math.fsum(data), rel=2e-16) + self.assertApproxEqual(float(self.func(data)[1]), math.fsum(data), rel=2e-16) def test_start_argument(self): # Test that the optional start argument works correctly. data = [random.uniform(1, 1000) for _ in range(100)] - t = self.func(data) - self.assertEqual(t+42, self.func(data, 42)) - self.assertEqual(t-23, self.func(data, -23)) - self.assertEqual(t+1e20, self.func(data, 1e20)) + t = self.func(data)[1] + self.assertEqual(t+42, self.func(data, 42)[1]) + self.assertEqual(t-23, self.func(data, -23)[1]) + self.assertEqual(t+Fraction(1e20), self.func(data, 1e20)[1]) def test_strings_fail(self): # Sum of strings should fail. @@ -934,7 +1185,7 @@ class TestSum(NumericTestCase, UnivariateCommonMixin, UnivariateTypeMixin): def test_mixed_sum(self): # Mixed input types are not (currently) allowed. # Check that mixed data types fail. - self.assertRaises(TypeError, self.func, [1, 2.0, Fraction(1, 2)]) + self.assertRaises(TypeError, self.func, [1, 2.0, Decimal(1)]) # And so does mixed start argument. self.assertRaises(TypeError, self.func, [1, 2.0], Decimal(1)) @@ -942,11 +1193,14 @@ class TestSum(NumericTestCase, UnivariateCommonMixin, UnivariateTypeMixin): class SumTortureTest(NumericTestCase): def test_torture(self): # Tim Peters' torture test for sum, and variants of same. - self.assertEqual(statistics._sum([1, 1e100, 1, -1e100]*10000), 20000.0) - self.assertEqual(statistics._sum([1e100, 1, 1, -1e100]*10000), 20000.0) - self.assertApproxEqual( - statistics._sum([1e-100, 1, 1e-100, -1]*10000), 2.0e-96, rel=5e-16 - ) + self.assertEqual(statistics._sum([1, 1e100, 1, -1e100]*10000), + (float, Fraction(20000.0), 40000)) + self.assertEqual(statistics._sum([1e100, 1, 1, -1e100]*10000), + (float, Fraction(20000.0), 40000)) + T, num, count = statistics._sum([1e-100, 1, 1e-100, -1]*10000) + self.assertIs(T, float) + self.assertEqual(count, 40000) + self.assertApproxEqual(float(num), 2.0e-96, rel=5e-16) class SumSpecialValues(NumericTestCase): @@ -955,7 +1209,7 @@ class SumSpecialValues(NumericTestCase): def test_nan(self): for type_ in (float, Decimal): nan = type_('nan') - result = statistics._sum([1, nan, 2]) + result = statistics._sum([1, nan, 2])[1] self.assertIs(type(result), type_) self.assertTrue(math.isnan(result)) @@ -968,10 +1222,10 @@ class SumSpecialValues(NumericTestCase): def do_test_inf(self, inf): # Adding a single infinity gives infinity. - result = statistics._sum([1, 2, inf, 3]) + result = statistics._sum([1, 2, inf, 3])[1] self.check_infinity(result, inf) # Adding two infinities of the same sign also gives infinity. - result = statistics._sum([1, 2, inf, 3, inf, 4]) + result = statistics._sum([1, 2, inf, 3, inf, 4])[1] self.check_infinity(result, inf) def test_float_inf(self): @@ -987,7 +1241,7 @@ class SumSpecialValues(NumericTestCase): def test_float_mismatched_infs(self): # Test that adding two infinities of opposite sign gives a NAN. inf = float('inf') - result = statistics._sum([1, 2, inf, 3, -inf, 4]) + result = statistics._sum([1, 2, inf, 3, -inf, 4])[1] self.assertTrue(math.isnan(result)) def test_decimal_extendedcontext_mismatched_infs_to_nan(self): @@ -995,7 +1249,7 @@ class SumSpecialValues(NumericTestCase): inf = Decimal('inf') data = [1, 2, inf, 3, -inf, 4] with decimal.localcontext(decimal.ExtendedContext): - self.assertTrue(math.isnan(statistics._sum(data))) + self.assertTrue(math.isnan(statistics._sum(data)[1])) def test_decimal_basiccontext_mismatched_infs_to_nan(self): # Test adding Decimal INFs with opposite sign raises InvalidOperation. @@ -1111,6 +1365,19 @@ class TestMean(NumericTestCase, AverageMixin, UnivariateTypeMixin): d = Decimal('1e4') self.assertEqual(statistics.mean([d]), d) + def test_regression_25177(self): + # Regression test for issue 25177. + # Ensure very big and very small floats don't overflow. + # See http://bugs.python.org/issue25177. + self.assertEqual(statistics.mean( + [8.988465674311579e+307, 8.98846567431158e+307]), + 8.98846567431158e+307) + big = 8.98846567431158e+307 + tiny = 5e-324 + for n in (2, 3, 5, 200): + self.assertEqual(statistics.mean([big]*n), big) + self.assertEqual(statistics.mean([tiny]*n), tiny) + class TestMedian(NumericTestCase, AverageMixin): # Common tests for median and all median.* functions. -- cgit v1.2.1 From 10993aa8d5fc513a6ac88831fcdc8d4ff2a972da Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 4 Dec 2015 15:19:42 -0800 Subject: Issue #25771: Tweak ValueError message when package isn't specified for importlib.util.resolve_name() but is needed. Thanks to Martin Panter for the bug report. --- Lib/importlib/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 1dbff2605e..39cb0f74fc 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -22,8 +22,8 @@ def resolve_name(name, package): if not name.startswith('.'): return name elif not package: - raise ValueError('{!r} is not a relative name ' - '(no leading dot)'.format(name)) + raise ValueError(f'no package specified for {repr(name)} ' + '(required for relative module names)') level = 0 for character in name: if character != '.': -- cgit v1.2.1 From ae9c6f526589ae064628647ce02bd09b62ce629f Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 5 Dec 2015 04:16:45 +0000 Subject: Issue #25764: Attempt to debug and skip OS X setrlimit() failure --- Lib/test/test_subprocess.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 0448d643cf..b68def50b3 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1516,10 +1516,16 @@ class POSIXProcessTestCase(BaseTestCase): # The internal code did not preserve the previous exception when # re-enabling garbage collection try: - from resource import getrlimit, setrlimit, RLIMIT_NPROC + from resource import getrlimit, setrlimit, RLIMIT_NPROC, RLIM_INFINITY except ImportError as err: self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD limits = getrlimit(RLIMIT_NPROC) + try: + setrlimit(RLIMIT_NPROC, limits) + except ValueError as err: + # Seems to happen on AMD64 Snow Leop and x86-64 Yosemite buildbots + print(f"Setting NPROC to {limits!r}: {err!r}, RLIM_INFINITY={RLIM_INFINITY!r}") + self.skipTest("Setting existing NPROC limit failed") [_, hard] = limits setrlimit(RLIMIT_NPROC, (0, hard)) self.addCleanup(setrlimit, RLIMIT_NPROC, limits) -- cgit v1.2.1 From 9a6e5a7ff536dbbd168b0cd5f1a7c13fb5ad5721 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 5 Dec 2015 05:42:18 +0000 Subject: Issue #25764: OS X now failing on the second setrlimit() call --- Lib/test/test_subprocess.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index b68def50b3..ed1b213fad 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1520,14 +1520,17 @@ class POSIXProcessTestCase(BaseTestCase): except ImportError as err: self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD limits = getrlimit(RLIMIT_NPROC) + [_, hard] = limits try: setrlimit(RLIMIT_NPROC, limits) + setrlimit(RLIMIT_NPROC, (0, hard)) except ValueError as err: - # Seems to happen on AMD64 Snow Leop and x86-64 Yosemite buildbots - print(f"Setting NPROC to {limits!r}: {err!r}, RLIM_INFINITY={RLIM_INFINITY!r}") - self.skipTest("Setting existing NPROC limit failed") - [_, hard] = limits - setrlimit(RLIMIT_NPROC, (0, hard)) + # Seems to happen on various OS X buildbots + print( + f"Setting NPROC failed: {err!r}, limits={limits!r}, " + f"RLIM_INFINITY={RLIM_INFINITY!r}, " + f"getrlimit() -> {getrlimit(RLIMIT_NPROC)!r}") + self.skipTest("Setting NPROC limit failed") self.addCleanup(setrlimit, RLIMIT_NPROC, limits) # Forking should raise EAGAIN, translated to BlockingIOError with self.assertRaises(BlockingIOError): -- cgit v1.2.1 From 4b80147acf86860abf1b7dde8a96d9c5d354f436 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 5 Dec 2015 10:18:25 +0000 Subject: Issue #25764: Remove test debugging --- Lib/test/test_subprocess.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 81c2672c59..b32ef9788c 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1518,21 +1518,12 @@ class POSIXProcessTestCase(BaseTestCase): # The internal code did not preserve the previous exception when # re-enabling garbage collection try: - from resource import getrlimit, setrlimit, RLIMIT_NPROC, RLIM_INFINITY + from resource import getrlimit, setrlimit, RLIMIT_NPROC except ImportError as err: self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD limits = getrlimit(RLIMIT_NPROC) [_, hard] = limits - try: - setrlimit(RLIMIT_NPROC, limits) - setrlimit(RLIMIT_NPROC, (0, hard)) - except ValueError as err: - # Seems to happen on various OS X buildbots - print( - f"Setting NPROC failed: {err!r}, limits={limits!r}, " - f"RLIM_INFINITY={RLIM_INFINITY!r}, " - f"getrlimit() -> {getrlimit(RLIMIT_NPROC)!r}") - self.skipTest("Setting NPROC limit failed") + setrlimit(RLIMIT_NPROC, (0, hard)) self.addCleanup(setrlimit, RLIMIT_NPROC, limits) # Forking should raise EAGAIN, translated to BlockingIOError with self.assertRaises(BlockingIOError): -- cgit v1.2.1 From 22306ca43756345c142028d51dc420dd2b08f677 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 6 Dec 2015 22:01:35 +0200 Subject: Issue #25761: Improved detecting errors in broken pickle data. --- Lib/pickle.py | 88 ++++++++++++++++++++---------------------------- Lib/test/pickletester.py | 17 ++++------ Lib/test/test_pickle.py | 5 --- 3 files changed, 42 insertions(+), 68 deletions(-) (limited to 'Lib') diff --git a/Lib/pickle.py b/Lib/pickle.py index 53978fbd65..a60b1b74c5 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -1031,7 +1031,7 @@ class _Unpickler: self._unframer = _Unframer(self._file_read, self._file_readline) self.read = self._unframer.read self.readline = self._unframer.readline - self.mark = object() # any new unique object + self.metastack = [] self.stack = [] self.append = self.stack.append self.proto = 0 @@ -1047,20 +1047,12 @@ class _Unpickler: except _Stop as stopinst: return stopinst.value - # Return largest index k such that self.stack[k] is self.mark. - # If the stack doesn't contain a mark, eventually raises IndexError. - # This could be sped by maintaining another stack, of indices at which - # the mark appears. For that matter, the latter stack would suffice, - # and we wouldn't need to push mark objects on self.stack at all. - # Doing so is probably a good thing, though, since if the pickle is - # corrupt (or hostile) we may get a clue from finding self.mark embedded - # in unpickled objects. - def marker(self): - stack = self.stack - mark = self.mark - k = len(stack)-1 - while stack[k] is not mark: k = k-1 - return k + # Return a list of items pushed in the stack after last MARK instruction. + def pop_mark(self): + items = self.stack + self.stack = self.metastack.pop() + self.append = self.stack.append + return items def persistent_load(self, pid): raise UnpicklingError("unsupported persistent id encountered") @@ -1237,8 +1229,8 @@ class _Unpickler: dispatch[SHORT_BINUNICODE[0]] = load_short_binunicode def load_tuple(self): - k = self.marker() - self.stack[k:] = [tuple(self.stack[k+1:])] + items = self.pop_mark() + self.append(tuple(items)) dispatch[TUPLE[0]] = load_tuple def load_empty_tuple(self): @@ -1270,21 +1262,20 @@ class _Unpickler: dispatch[EMPTY_SET[0]] = load_empty_set def load_frozenset(self): - k = self.marker() - self.stack[k:] = [frozenset(self.stack[k+1:])] + items = self.pop_mark() + self.append(frozenset(items)) dispatch[FROZENSET[0]] = load_frozenset def load_list(self): - k = self.marker() - self.stack[k:] = [self.stack[k+1:]] + items = self.pop_mark() + self.append(items) dispatch[LIST[0]] = load_list def load_dict(self): - k = self.marker() - items = self.stack[k+1:] + items = self.pop_mark() d = {items[i]: items[i+1] for i in range(0, len(items), 2)} - self.stack[k:] = [d] + self.append(d) dispatch[DICT[0]] = load_dict # INST and OBJ differ only in how they get a class object. It's not @@ -1292,9 +1283,7 @@ class _Unpickler: # previously diverged and grew different bugs. # klass is the class to instantiate, and k points to the topmost mark # object, following which are the arguments for klass.__init__. - def _instantiate(self, klass, k): - args = tuple(self.stack[k+1:]) - del self.stack[k:] + def _instantiate(self, klass, args): if (args or not isinstance(klass, type) or hasattr(klass, "__getinitargs__")): try: @@ -1310,14 +1299,14 @@ class _Unpickler: module = self.readline()[:-1].decode("ascii") name = self.readline()[:-1].decode("ascii") klass = self.find_class(module, name) - self._instantiate(klass, self.marker()) + self._instantiate(klass, self.pop_mark()) dispatch[INST[0]] = load_inst def load_obj(self): # Stack is ... markobject classobject arg1 arg2 ... - k = self.marker() - klass = self.stack.pop(k+1) - self._instantiate(klass, k) + args = self.pop_mark() + cls = args.pop(0) + self._instantiate(cls, args) dispatch[OBJ[0]] = load_obj def load_newobj(self): @@ -1402,12 +1391,14 @@ class _Unpickler: dispatch[REDUCE[0]] = load_reduce def load_pop(self): - del self.stack[-1] + if self.stack: + del self.stack[-1] + else: + self.pop_mark() dispatch[POP[0]] = load_pop def load_pop_mark(self): - k = self.marker() - del self.stack[k:] + self.pop_mark() dispatch[POP_MARK[0]] = load_pop_mark def load_dup(self): @@ -1463,17 +1454,14 @@ class _Unpickler: dispatch[APPEND[0]] = load_append def load_appends(self): - stack = self.stack - mark = self.marker() - list_obj = stack[mark - 1] - items = stack[mark + 1:] + items = self.pop_mark() + list_obj = self.stack[-1] if isinstance(list_obj, list): list_obj.extend(items) else: append = list_obj.append for item in items: append(item) - del stack[mark:] dispatch[APPENDS[0]] = load_appends def load_setitem(self): @@ -1485,27 +1473,21 @@ class _Unpickler: dispatch[SETITEM[0]] = load_setitem def load_setitems(self): - stack = self.stack - mark = self.marker() - dict = stack[mark - 1] - for i in range(mark + 1, len(stack), 2): - dict[stack[i]] = stack[i + 1] - - del stack[mark:] + items = self.pop_mark() + dict = self.stack[-1] + for i in range(0, len(items), 2): + dict[items[i]] = items[i + 1] dispatch[SETITEMS[0]] = load_setitems def load_additems(self): - stack = self.stack - mark = self.marker() - set_obj = stack[mark - 1] - items = stack[mark + 1:] + items = self.pop_mark() + set_obj = self.stack[-1] if isinstance(set_obj, set): set_obj.update(items) else: add = set_obj.add for item in items: add(item) - del stack[mark:] dispatch[ADDITEMS[0]] = load_additems def load_build(self): @@ -1533,7 +1515,9 @@ class _Unpickler: dispatch[BUILD[0]] = load_build def load_mark(self): - self.append(self.mark) + self.metastack.append(self.stack) + self.stack = [] + self.append = self.stack.append dispatch[MARK[0]] = load_mark def load_stop(self): diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 608c35ac72..217aa3db0c 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1000,7 +1000,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'0', # POP b'1', # POP_MARK b'2', # DUP - # b'(2', # PyUnpickler doesn't raise + b'(2', b'R', # REDUCE b')R', b'a', # APPEND @@ -1009,7 +1009,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'Nb', b'd', # DICT b'e', # APPENDS - # b'(e', # PyUnpickler raises AttributeError + b'(e', b'ibuiltins\nlist\n', # INST b'l', # LIST b'o', # OBJ @@ -1022,7 +1022,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'NNs', b't', # TUPLE b'u', # SETITEMS - # b'(u', # PyUnpickler doesn't raise + b'(u', b'}(Nu', b'\x81', # NEWOBJ b')\x81', @@ -1033,7 +1033,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'N\x87', b'NN\x87', b'\x90', # ADDITEMS - # b'(\x90', # PyUnpickler raises AttributeError + b'(\x90', b'\x91', # FROZENSET b'\x92', # NEWOBJ_EX b')}\x92', @@ -1046,7 +1046,7 @@ class AbstractUnpickleTests(unittest.TestCase): def test_bad_mark(self): badpickles = [ - # b'N(.', # STOP + b'N(.', # STOP b'N(2', # DUP b'cbuiltins\nlist\n)(R', # REDUCE b'cbuiltins\nlist\n()R', @@ -1081,7 +1081,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'N(\x94', # MEMOIZE ] for p in badpickles: - self.check_unpickling_error(self.bad_mark_errors, p) + self.check_unpickling_error(self.bad_stack_errors, p) def test_truncated_data(self): self.check_unpickling_error(EOFError, b'') @@ -2581,11 +2581,6 @@ class AbstractPickleModuleTests(unittest.TestCase): self.assertRaises(pickle.PicklingError, BadPickler().dump, 0) self.assertRaises(pickle.UnpicklingError, BadUnpickler().load) - def test_bad_input(self): - # Test issue4298 - s = bytes([0x58, 0, 0, 0, 0x54]) - self.assertRaises(EOFError, pickle.loads, s) - class AbstractPersistentPicklerTests(unittest.TestCase): diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index bd38cfbea9..6b97315438 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -33,8 +33,6 @@ class PyUnpicklerTests(AbstractUnpickleTests): unpickler = pickle._Unpickler bad_stack_errors = (IndexError,) - bad_mark_errors = (IndexError, pickle.UnpicklingError, - TypeError, AttributeError, EOFError) truncated_errors = (pickle.UnpicklingError, EOFError, AttributeError, ValueError, struct.error, IndexError, ImportError) @@ -69,8 +67,6 @@ class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, pickler = pickle._Pickler unpickler = pickle._Unpickler bad_stack_errors = (pickle.UnpicklingError, IndexError) - bad_mark_errors = (pickle.UnpicklingError, IndexError, - TypeError, AttributeError, EOFError) truncated_errors = (pickle.UnpicklingError, EOFError, AttributeError, ValueError, struct.error, IndexError, ImportError) @@ -132,7 +128,6 @@ if has_c_implementation: class CUnpicklerTests(PyUnpicklerTests): unpickler = _pickle.Unpickler bad_stack_errors = (pickle.UnpicklingError,) - bad_mark_errors = (EOFError,) truncated_errors = (pickle.UnpicklingError, EOFError, AttributeError, ValueError) -- cgit v1.2.1 From 8b5502dee0a559ea1975ff4aa76197c501a2ac33 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 7 Dec 2015 02:31:11 +0200 Subject: Issue #25638: Optimized ElementTree.iterparse(); it is now 2x faster. ElementTree.XMLParser._setevents now accepts any objects with the append method, not just a list. --- Lib/xml/etree/ElementTree.py | 92 +++++++++++++++----------------------------- 1 file changed, 31 insertions(+), 61 deletions(-) (limited to 'Lib') diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 62b5d3aeec..a58ef31812 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -95,6 +95,7 @@ import sys import re import warnings import io +import collections import contextlib from . import ElementPath @@ -1198,16 +1199,37 @@ def iterparse(source, events=None, parser=None): Returns an iterator providing (event, elem) pairs. """ + # Use the internal, undocumented _parser argument for now; When the + # parser argument of iterparse is removed, this can be killed. + pullparser = XMLPullParser(events=events, _parser=parser) + def iterator(): + try: + while True: + yield from pullparser.read_events() + # load event buffer + data = source.read(16 * 1024) + if not data: + break + pullparser.feed(data) + root = pullparser._close_and_return_root() + yield from pullparser.read_events() + it.root = root + finally: + if close_source: + source.close() + + class IterParseIterator(collections.Iterator): + __next__ = iterator().__next__ + it = IterParseIterator() + it.root = None + del iterator, IterParseIterator + close_source = False if not hasattr(source, "read"): source = open(source, "rb") close_source = True - try: - return _IterParseIterator(source, events, parser, close_source) - except: - if close_source: - source.close() - raise + + return it class XMLPullParser: @@ -1217,9 +1239,7 @@ class XMLPullParser: # upon in user code. It will be removed in a future release. # See http://bugs.python.org/issue17741 for more details. - # _elementtree.c expects a list, not a deque - self._events_queue = [] - self._index = 0 + self._events_queue = collections.deque() self._parser = _parser or XMLParser(target=TreeBuilder()) # wire up the parser for event reporting if events is None: @@ -1257,64 +1277,14 @@ class XMLPullParser: retrieved from the iterator. """ events = self._events_queue - while True: - index = self._index - try: - event = events[self._index] - # Avoid retaining references to past events - events[self._index] = None - except IndexError: - break - index += 1 - # Compact the list in a O(1) amortized fashion - # As noted above, _elementree.c needs a list, not a deque - if index * 2 >= len(events): - events[:index] = [] - self._index = 0 - else: - self._index = index + while events: + event = events.popleft() if isinstance(event, Exception): raise event else: yield event -class _IterParseIterator: - - def __init__(self, source, events, parser, close_source=False): - # Use the internal, undocumented _parser argument for now; When the - # parser argument of iterparse is removed, this can be killed. - self._parser = XMLPullParser(events=events, _parser=parser) - self._file = source - self._close_file = close_source - self.root = self._root = None - - def __next__(self): - try: - while 1: - for event in self._parser.read_events(): - return event - if self._parser._parser is None: - break - # load event buffer - data = self._file.read(16 * 1024) - if data: - self._parser.feed(data) - else: - self._root = self._parser._close_and_return_root() - self.root = self._root - except: - if self._close_file: - self._file.close() - raise - if self._close_file: - self._file.close() - raise StopIteration - - def __iter__(self): - return self - - def XML(text, parser=None): """Parse XML document from string constant. -- cgit v1.2.1 From c68ea57f1f4c9d560dda01b942627803fdc2e838 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Dec 2015 11:29:59 +0100 Subject: Issue #25868: Try to make test_eintr.test_sigwaitinfo() more reliable especially on slow buildbots Use a pipe to synchronize the parent and the child processes. --- Lib/test/eintrdata/eintr_tester.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'Lib') diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index ee6e75bb99..531d576df6 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -377,10 +377,10 @@ class SignalEINTRTest(EINTRBaseTest): @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), 'need signal.sigwaitinfo()') def test_sigwaitinfo(self): - # Issue #25277: The sleep is a weak synchronization between the parent - # and the child process. If the sleep is too low, the test hangs on - # slow or highly loaded systems. - self.sleep_time = 2.0 + # Issue #25277, #25868: give a few miliseconds to the parent process + # between os.write() and signal.sigwaitinfo() to works around a race + # condition + self.sleep_time = 0.100 signum = signal.SIGUSR1 pid = os.getpid() @@ -388,18 +388,28 @@ class SignalEINTRTest(EINTRBaseTest): old_handler = signal.signal(signum, lambda *args: None) self.addCleanup(signal.signal, signum, old_handler) + rpipe, wpipe = os.pipe() + code = '\n'.join(( 'import os, time', 'pid = %s' % os.getpid(), 'signum = %s' % int(signum), 'sleep_time = %r' % self.sleep_time, + 'rpipe = %r' % rpipe, + 'os.read(rpipe, 1)', + 'os.close(rpipe)', 'time.sleep(sleep_time)', 'os.kill(pid, signum)', )) t0 = time.monotonic() - proc = self.subprocess(code) + proc = self.subprocess(code, pass_fds=(rpipe,)) + os.close(rpipe) with kill_on_error(proc): + # sync child-parent + os.write(wpipe, b'x') + os.close(wpipe) + # parent signal.sigwaitinfo([signum]) dt = time.monotonic() - t0 -- cgit v1.2.1 From 5b42b55545fd62ab8a507848c4c3fd498521bede Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 19 Dec 2015 09:43:14 +0200 Subject: Issue #22227: The TarFile iterator is reimplemented using generator. This implementation is simpler that using class. --- Lib/tarfile.py | 68 ++++++++++++++++++++++------------------------------------ 1 file changed, 26 insertions(+), 42 deletions(-) (limited to 'Lib') diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 9d22c8e37f..80b6e35d42 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2372,9 +2372,32 @@ class TarFile(object): """Provide an iterator object. """ if self._loaded: - return iter(self.members) - else: - return TarIter(self) + yield from self.members + return + + # Yield items using TarFile's next() method. + # When all members have been read, set TarFile as _loaded. + index = 0 + # Fix for SF #1100429: Under rare circumstances it can + # happen that getmembers() is called during iteration, + # which will have already exhausted the next() method. + if self.firstmember is not None: + tarinfo = self.next() + index += 1 + yield tarinfo + + while True: + if index < len(self.members): + tarinfo = self.members[index] + elif not self._loaded: + tarinfo = self.next() + if not tarinfo: + self._loaded = True + return + else: + return + index += 1 + yield tarinfo def _dbg(self, level, msg): """Write debugging output to sys.stderr. @@ -2395,45 +2418,6 @@ class TarFile(object): if not self._extfileobj: self.fileobj.close() self.closed = True -# class TarFile - -class TarIter: - """Iterator Class. - - for tarinfo in TarFile(...): - suite... - """ - - def __init__(self, tarfile): - """Construct a TarIter object. - """ - self.tarfile = tarfile - self.index = 0 - def __iter__(self): - """Return iterator object. - """ - return self - def __next__(self): - """Return the next item using TarFile's next() method. - When all members have been read, set TarFile as _loaded. - """ - # Fix for SF #1100429: Under rare circumstances it can - # happen that getmembers() is called during iteration, - # which will cause TarIter to stop prematurely. - - if self.index == 0 and self.tarfile.firstmember is not None: - tarinfo = self.tarfile.next() - elif self.index < len(self.tarfile.members): - tarinfo = self.tarfile.members[self.index] - elif not self.tarfile._loaded: - tarinfo = self.tarfile.next() - if not tarinfo: - self.tarfile._loaded = True - raise StopIteration - else: - raise StopIteration - self.index += 1 - return tarinfo #-------------------- # exported functions -- cgit v1.2.1 From 881583881808cb2123cb1c7d9b03f278dd5df37d Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 27 Dec 2015 13:17:04 -0800 Subject: Issue #25768: Make compileall functions return booleans and document the return values as well as test them. Thanks to Nicholas Chammas for the bug report and initial patch. --- Lib/compileall.py | 16 ++++++++-------- Lib/test/test_compileall.py | 26 ++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 10 deletions(-) (limited to 'Lib') diff --git a/Lib/compileall.py b/Lib/compileall.py index 0cc0c1d530..67c5f5ac72 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -68,7 +68,7 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, """ files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, ddir=ddir) - success = 1 + success = True if workers is not None and workers != 1 and ProcessPoolExecutor is not None: if workers < 0: raise ValueError('workers must be greater or equal to 0') @@ -81,12 +81,12 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, legacy=legacy, optimize=optimize), files) - success = min(results, default=1) + success = min(results, default=True) else: for file in files: if not compile_file(file, ddir, force, rx, quiet, legacy, optimize): - success = 0 + success = False return success def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, @@ -104,7 +104,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy: if True, produce legacy pyc paths instead of PEP 3147 paths optimize: optimization level or -1 for level of the interpreter """ - success = 1 + success = True name = os.path.basename(fullname) if ddir is not None: dfile = os.path.join(ddir, name) @@ -144,7 +144,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, ok = py_compile.compile(fullname, cfile, dfile, True, optimize=optimize) except py_compile.PyCompileError as err: - success = 0 + success = False if quiet >= 2: return success elif quiet: @@ -157,7 +157,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, msg = msg.decode(sys.stdout.encoding) print(msg) except (SyntaxError, UnicodeError, OSError) as e: - success = 0 + success = False if quiet >= 2: return success elif quiet: @@ -167,7 +167,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, print(e.__class__.__name__ + ':', e) else: if ok == 0: - success = 0 + success = False return success def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, @@ -183,7 +183,7 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, legacy: as for compile_dir() (default False) optimize: as for compile_dir() (default -1) """ - success = 1 + success = True for dir in sys.path: if (not dir or dir == os.curdir) and skip_curdir: if quiet < 2: diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 2ce8a61e39..439c125edb 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -1,6 +1,7 @@ import sys import compileall import importlib.util +import test.test_importlib.util import os import pathlib import py_compile @@ -40,6 +41,11 @@ class CompileallTests(unittest.TestCase): def tearDown(self): shutil.rmtree(self.directory) + def add_bad_source_file(self): + self.bad_source_path = os.path.join(self.directory, '_test_bad.py') + with open(self.bad_source_path, 'w') as file: + file.write('x (\n') + def data(self): with open(self.bc_path, 'rb') as file: data = file.read(8) @@ -78,15 +84,31 @@ class CompileallTests(unittest.TestCase): os.unlink(fn) except: pass - compileall.compile_file(self.source_path, force=False, quiet=True) + self.assertTrue(compileall.compile_file(self.source_path, + force=False, quiet=True)) self.assertTrue(os.path.isfile(self.bc_path) and not os.path.isfile(self.bc_path2)) os.unlink(self.bc_path) - compileall.compile_dir(self.directory, force=False, quiet=True) + self.assertTrue(compileall.compile_dir(self.directory, force=False, + quiet=True)) self.assertTrue(os.path.isfile(self.bc_path) and os.path.isfile(self.bc_path2)) os.unlink(self.bc_path) os.unlink(self.bc_path2) + # Test against bad files + self.add_bad_source_file() + self.assertFalse(compileall.compile_file(self.bad_source_path, + force=False, quiet=2)) + self.assertFalse(compileall.compile_dir(self.directory, + force=False, quiet=2)) + + def test_compile_path(self): + self.assertTrue(compileall.compile_path(quiet=2)) + + with test.test_importlib.util.import_state(path=[self.directory]): + self.add_bad_source_file() + self.assertFalse(compileall.compile_path(skip_curdir=False, + force=True, quiet=2)) def test_no_pycache_in_non_package(self): # Bug 8563 reported that __pycache__ directories got created by -- cgit v1.2.1 From c03d52305867f81c40034f830c514e20bc12047f Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Mon, 28 Dec 2015 23:50:19 +0200 Subject: Remove duplicate method in test_pathlib. Initial patch by Navneet Suman. --- Lib/test/test_pathlib.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 1c53ab78ea..80cb97e97a 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1437,14 +1437,14 @@ class _BasePathTest(object): self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") }) self.assertEqual(set(p.glob("../xyzzy")), set()) - def _check_resolve_relative(self, p, expected): - q = p.resolve() - self.assertEqual(q, expected) - def _check_resolve_absolute(self, p, expected): + def _check_resolve(self, p, expected): q = p.resolve() self.assertEqual(q, expected) + # this can be used to check both relative and absolute resolutions + _check_resolve_relative = _check_resolve_absolute = _check_resolve + @with_symlinks def test_resolve_common(self): P = self.cls -- cgit v1.2.1 From 4f9ad6267df821336864a4f9dd83d7d47ef7c34a Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Mon, 28 Dec 2015 23:02:02 +0100 Subject: Issue #25928: Add Decimal.as_integer_ratio(). Python parts and docs by Mark Dickinson. --- Lib/_pydecimal.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ Lib/test/test_decimal.py | 33 ++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) (limited to 'Lib') diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 05ba4eeee3..eb7bba8e93 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -1010,6 +1010,58 @@ class Decimal(object): """ return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp) + def as_integer_ratio(self): + """Express a finite Decimal instance in the form n / d. + + Returns a pair (n, d) of integers. When called on an infinity + or NaN, raises OverflowError or ValueError respectively. + + >>> Decimal('3.14').as_integer_ratio() + (157, 50) + >>> Decimal('-123e5').as_integer_ratio() + (-12300000, 1) + >>> Decimal('0.00').as_integer_ratio() + (0, 1) + + """ + if self._is_special: + if self.is_nan(): + raise ValueError("Cannot pass NaN " + "to decimal.as_integer_ratio.") + else: + raise OverflowError("Cannot pass infinity " + "to decimal.as_integer_ratio.") + + if not self: + return 0, 1 + + # Find n, d in lowest terms such that abs(self) == n / d; + # we'll deal with the sign later. + n = int(self._int) + if self._exp >= 0: + # self is an integer. + n, d = n * 10**self._exp, 1 + else: + # Find d2, d5 such that abs(self) = n / (2**d2 * 5**d5). + d5 = -self._exp + while d5 > 0 and n % 5 == 0: + n //= 5 + d5 -= 1 + + # (n & -n).bit_length() - 1 counts trailing zeros in binary + # representation of n (provided n is nonzero). + d2 = -self._exp + shift2 = min((n & -n).bit_length() - 1, d2) + if shift2: + n >>= shift2 + d2 -= shift2 + + d = 5**d5 << d2 + + if self._sign: + n = -n + return n, d + def __repr__(self): """Represents the number as an instance of Decimal.""" # Invariant: eval(repr(d)) == d diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 137aaa537e..f6d58ff53d 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -2047,6 +2047,39 @@ class UsabilityTest(unittest.TestCase): d = Decimal( (1, (0, 2, 7, 1), 'F') ) self.assertEqual(d.as_tuple(), (1, (0,), 'F')) + def test_as_integer_ratio(self): + Decimal = self.decimal.Decimal + + # exceptional cases + self.assertRaises(OverflowError, + Decimal.as_integer_ratio, Decimal('inf')) + self.assertRaises(OverflowError, + Decimal.as_integer_ratio, Decimal('-inf')) + self.assertRaises(ValueError, + Decimal.as_integer_ratio, Decimal('-nan')) + self.assertRaises(ValueError, + Decimal.as_integer_ratio, Decimal('snan123')) + + for exp in range(-4, 2): + for coeff in range(1000): + for sign in '+', '-': + d = Decimal('%s%dE%d' % (sign, coeff, exp)) + pq = d.as_integer_ratio() + p, q = pq + + # check return type + self.assertIsInstance(pq, tuple) + self.assertIsInstance(p, int) + self.assertIsInstance(q, int) + + # check normalization: q should be positive; + # p should be relatively prime to q. + self.assertGreater(q, 0) + self.assertEqual(math.gcd(p, q), 1) + + # check that p/q actually gives the correct value + self.assertEqual(Decimal(p) / Decimal(q), d) + def test_subclassing(self): # Different behaviours when subclassing Decimal Decimal = self.decimal.Decimal -- cgit v1.2.1 From 3f8a4221ad6bbbe4891b5247de4967964f2c1cf1 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Tue, 29 Dec 2015 00:11:36 +0100 Subject: Issue #25928: Temporarily disable some tests in test_statistics in order to sort out its assumptions about the as_integer_ratio() interface. --- Lib/test/test_statistics.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 0089ae8dc6..9a54fe12b8 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -699,6 +699,7 @@ class ExactRatioTest(unittest.TestCase): num, den = statistics._exact_ratio(x) self.assertEqual(x, num/den) + @unittest.skipIf(True, "temporarily disabled: see #25928") def test_decimal(self): D = Decimal _exact_ratio = statistics._exact_ratio @@ -730,6 +731,7 @@ class ExactRatioTest(unittest.TestCase): self.assertIs(ratio[1], None) self.assertEqual(type(ratio[0]), type(nan)) + @unittest.skipIf(True, "temporarily disabled: see #25928") def test_decimal_nan(self): NAN = Decimal("NAN") sNAN = Decimal("sNAN") @@ -1258,6 +1260,7 @@ class SumSpecialValues(NumericTestCase): with decimal.localcontext(decimal.BasicContext): self.assertRaises(decimal.InvalidOperation, statistics._sum, data) + @unittest.skipIf(True, "temporarily disabled: see #25928") def test_decimal_snan_raises(self): # Adding sNAN should raise InvalidOperation. sNAN = Decimal('sNAN') -- cgit v1.2.1 From 5254ee14c323e4d26d1c871376a68d031eb68e3e Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Tue, 29 Dec 2015 02:15:35 +0100 Subject: Issue #25940: Make buildbots usable again until a solution is found. --- Lib/test/test_ssl.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 548feebb82..d1a908826d 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1342,6 +1342,7 @@ class MemoryBIOTests(unittest.TestCase): self.assertRaises(TypeError, bio.write, 1) +@unittest.skipIf(True, "temporarily disabled: see #25940") class NetworkedTests(unittest.TestCase): def test_connect(self): @@ -1711,6 +1712,7 @@ class NetworkedBIOTests(unittest.TestCase): % (count, func.__name__)) return ret + @unittest.skipIf(True, "temporarily disabled: see #25940") def test_handshake(self): with support.transient_internet("svn.python.org"): sock = socket.socket(socket.AF_INET) -- cgit v1.2.1 From 10f0daab9e1d56d5b1558e5a0bf273f10cfb5074 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 28 Dec 2015 17:17:58 -0800 Subject: Issue #24725: Skip the test_socket.testFDPassEmpty on OS X. In OS X 10.11, the test fails consistently due to a platform change since 10.10. Thanks to Jeff Ramnani for the patch. --- Lib/test/test_socket.py | 1 + 1 file changed, 1 insertion(+) (limited to 'Lib') diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 1e355ea8fe..ad1efb2818 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -2809,6 +2809,7 @@ class SCMRightsTest(SendrecvmsgServerTimeoutBase): nbytes = self.sendmsgToServer([msg]) self.assertEqual(nbytes, len(msg)) + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #24725") def testFDPassEmpty(self): # Try to pass an empty FD array. Can receive either no array # or an empty array. -- cgit v1.2.1 From ba9190d78cb46d1f2aa1e2fae5479463e6cbfc32 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 28 Dec 2015 17:21:44 -0800 Subject: Tweak skipping message --- Lib/test/test_socket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index ad1efb2818..99707cec9b 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -2809,7 +2809,7 @@ class SCMRightsTest(SendrecvmsgServerTimeoutBase): nbytes = self.sendmsgToServer([msg]) self.assertEqual(nbytes, len(msg)) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #24725") + @unittest.skipIf(sys.platform == "darwin", "see issue #24725") def testFDPassEmpty(self): # Try to pass an empty FD array. Can receive either no array # or an empty array. -- cgit v1.2.1 From a899861e906afcc4cd7c7b4c22a6bb633cc81376 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 28 Dec 2015 17:23:35 -0800 Subject: Issue #25234: Skip test_eintr.test_os_open under OS X. Test inconsistently hangs. --- Lib/test/eintrdata/eintr_tester.py | 1 + 1 file changed, 1 insertion(+) (limited to 'Lib') diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 531d576df6..e1ed3da828 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -345,6 +345,7 @@ class SocketEINTRTest(EINTRBaseTest): fd = os.open(path, os.O_WRONLY) os.close(fd) + @unittest.skipIf(sys.platform == "darwin", "hangs under OS X; see issue #25234") def test_os_open(self): self._test_open("fd = os.open(path, os.O_RDONLY)\nos.close(fd)", self.os_open) -- cgit v1.2.1 From eb8ef4db8ad7af0f2f71223afd5c4b2200afa7c8 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 28 Dec 2015 17:55:27 -0800 Subject: Issue #25802: Deprecate load_module() on importlib.machinery.SourceFileLoader and SourcelessFileLoader. They were the only remaining implementations of load_module() not documented as deprecated. --- Lib/importlib/_bootstrap_external.py | 1 + 1 file changed, 1 insertion(+) (limited to 'Lib') diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index c58a62a90d..f27450153e 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -655,6 +655,7 @@ class _LoaderBasics: _bootstrap._call_with_frames_removed(exec, code, module.__dict__) def load_module(self, fullname): + """This module is deprecated.""" return _bootstrap._load_module_shim(self, fullname) -- cgit v1.2.1 From e1c37015c636e57cffb3a1ae8721cb213880f65b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 29 Dec 2015 22:34:23 +0200 Subject: Issue #25971: Optimized creating Fractions from floats by 2 times and from Decimals by 3 times. Unified error messages in float.as_integer_ratio(), Decimal.as_integer_ratio(), and Fraction constructors. --- Lib/_pydecimal.py | 6 ++---- Lib/fractions.py | 32 ++++---------------------------- Lib/test/test_fractions.py | 14 +++++++------- 3 files changed, 13 insertions(+), 39 deletions(-) (limited to 'Lib') diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index eb7bba8e93..02365cad2c 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -1026,11 +1026,9 @@ class Decimal(object): """ if self._is_special: if self.is_nan(): - raise ValueError("Cannot pass NaN " - "to decimal.as_integer_ratio.") + raise ValueError("cannot convert NaN to integer ratio") else: - raise OverflowError("Cannot pass infinity " - "to decimal.as_integer_ratio.") + raise OverflowError("cannot convert Infinity to integer ratio") if not self: return 0, 1 diff --git a/Lib/fractions.py b/Lib/fractions.py index 60b0728807..64d746b842 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -125,17 +125,9 @@ class Fraction(numbers.Rational): self._denominator = numerator.denominator return self - elif isinstance(numerator, float): - # Exact conversion from float - value = Fraction.from_float(numerator) - self._numerator = value._numerator - self._denominator = value._denominator - return self - - elif isinstance(numerator, Decimal): - value = Fraction.from_decimal(numerator) - self._numerator = value._numerator - self._denominator = value._denominator + elif isinstance(numerator, (float, Decimal)): + # Exact conversion + self._numerator, self._denominator = numerator.as_integer_ratio() return self elif isinstance(numerator, str): @@ -210,10 +202,6 @@ class Fraction(numbers.Rational): elif not isinstance(f, float): raise TypeError("%s.from_float() only takes floats, not %r (%s)" % (cls.__name__, f, type(f).__name__)) - if math.isnan(f): - raise ValueError("Cannot convert %r to %s." % (f, cls.__name__)) - if math.isinf(f): - raise OverflowError("Cannot convert %r to %s." % (f, cls.__name__)) return cls(*f.as_integer_ratio()) @classmethod @@ -226,19 +214,7 @@ class Fraction(numbers.Rational): raise TypeError( "%s.from_decimal() only takes Decimals, not %r (%s)" % (cls.__name__, dec, type(dec).__name__)) - if dec.is_infinite(): - raise OverflowError( - "Cannot convert %s to %s." % (dec, cls.__name__)) - if dec.is_nan(): - raise ValueError("Cannot convert %s to %s." % (dec, cls.__name__)) - sign, digits, exp = dec.as_tuple() - digits = int(''.join(map(str, digits))) - if sign: - digits = -digits - if exp >= 0: - return cls(digits * 10 ** exp) - else: - return cls(digits, 10 ** -exp) + return cls(*dec.as_integer_ratio()) def limit_denominator(self, max_denominator=1000000): """Closest Fraction to self with denominator at most max_denominator. diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 1699852216..73d2dd3031 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -263,13 +263,13 @@ class FractionTest(unittest.TestCase): nan = inf - inf # bug 16469: error types should be consistent with float -> int self.assertRaisesMessage( - OverflowError, "Cannot convert inf to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_float, inf) self.assertRaisesMessage( - OverflowError, "Cannot convert -inf to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_float, -inf) self.assertRaisesMessage( - ValueError, "Cannot convert nan to Fraction.", + ValueError, "cannot convert NaN to integer ratio", F.from_float, nan) def testFromDecimal(self): @@ -284,16 +284,16 @@ class FractionTest(unittest.TestCase): # bug 16469: error types should be consistent with decimal -> int self.assertRaisesMessage( - OverflowError, "Cannot convert Infinity to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_decimal, Decimal("inf")) self.assertRaisesMessage( - OverflowError, "Cannot convert -Infinity to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_decimal, Decimal("-inf")) self.assertRaisesMessage( - ValueError, "Cannot convert NaN to Fraction.", + ValueError, "cannot convert NaN to integer ratio", F.from_decimal, Decimal("nan")) self.assertRaisesMessage( - ValueError, "Cannot convert sNaN to Fraction.", + ValueError, "cannot convert NaN to integer ratio", F.from_decimal, Decimal("snan")) def testLimitDenominator(self): -- cgit v1.2.1 From 926b7445d9df428b51d1ae9a55ee729dde6bfdc7 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sat, 2 Jan 2016 17:18:34 -0500 Subject: #21815: violate IMAP RFC to be compatible with, e.g., gmail and others, including imaplib's own behavior. I'm applying this only to 3.6 because there's a potential backward compatibility concern: if there are servers that include ] characters in the 'text' portion of their imap responses, this code change could introduce a new bug. Patch by Lita Cho, reviewed by Jessica McKellar, Berker Peksag, Maciej Szulik, silentghost, and me (I fleshed out the comments with the additional info/concerns.) --- Lib/imaplib.py | 10 +++++++++- Lib/test/test_imaplib.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 4e8a4bb6fa..a63ba8dbe0 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -111,7 +111,15 @@ InternalDate = re.compile(br'.*INTERNALDATE "' # Literal is no longer used; kept for backward compatibility. Literal = re.compile(br'.*{(?P\d+)}$', re.ASCII) MapCRLF = re.compile(br'\r\n|\r|\n') -Response_code = re.compile(br'\[(?P[A-Z-]+)( (?P[^\]]*))?\]') +# We no longer exclude the ']' character from the data portion of the response +# code, even though it violates the RFC. Popular IMAP servers such as Gmail +# allow flags with ']', and there are programs (including imaplib!) that can +# produce them. The problem with this is if the 'text' portion of the response +# includes a ']' we'll parse the response wrong (which is the point of the RFC +# restriction). However, that seems less likely to be a problem in practice +# than being unable to correctly parse flags that include ']' chars, which +# was reported as a real-world problem in issue #21815. +Response_code = re.compile(br'\[(?P[A-Z-]+)( (?P.*))?\]') Untagged_response = re.compile(br'\* (?P[A-Z-]+)( (?P.*))?') # Untagged_status is no longer used; kept for backward compatibility Untagged_status = re.compile( diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 07157f52d6..8e4990b3cf 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -242,6 +242,55 @@ class ThreadedNetworkedTests(unittest.TestCase): client = self.imap_class(*server.server_address) client.shutdown() + @reap_threads + def test_bracket_flags(self): + + # This violates RFC 3501, which disallows ']' characters in tag names, + # but imaplib has allowed producing such tags forever, other programs + # also produce them (eg: OtherInbox's Organizer app as of 20140716), + # and Gmail, for example, accepts them and produces them. So we + # support them. See issue #21815. + + class BracketFlagHandler(SimpleIMAPHandler): + + def handle(self): + self.flags = ['Answered', 'Flagged', 'Deleted', 'Seen', 'Draft'] + super().handle() + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + + def cmd_SELECT(self, tag, args): + flag_msg = ' \\'.join(self.flags) + self._send_line(('* FLAGS (%s)' % flag_msg).encode('ascii')) + self._send_line(b'* 2 EXISTS') + self._send_line(b'* 0 RECENT') + msg = ('* OK [PERMANENTFLAGS %s \\*)] Flags permitted.' + % flag_msg) + self._send_line(msg.encode('ascii')) + self._send_tagged(tag, 'OK', '[READ-WRITE] SELECT completed.') + + def cmd_STORE(self, tag, args): + new_flags = args[2].strip('(').strip(')').split() + self.flags.extend(new_flags) + flags_msg = '(FLAGS (%s))' % ' \\'.join(self.flags) + msg = '* %s FETCH %s' % (args[0], flags_msg) + self._send_line(msg.encode('ascii')) + self._send_tagged(tag, 'OK', 'STORE completed.') + + with self.reaped_pair(BracketFlagHandler) as (server, client): + code, data = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, b'ZmFrZQ==\r\n') + client.select('test') + typ, [data] = client.store(b'1', "+FLAGS", "[test]") + self.assertIn(b'[test]', data) + client.select('test') + typ, [data] = client.response('PERMANENTFLAGS') + self.assertIn(b'[test]', data) + @reap_threads def test_issue5949(self): -- cgit v1.2.1 From 57959d9706981516c162419ee8a2e5ef1e739c8d Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Mon, 11 Jan 2016 07:09:42 -0800 Subject: Issue #26069: Remove the deprecated apis in the trace module. --- Lib/test/test_trace.py | 48 +----------------------------------------------- Lib/trace.py | 43 ------------------------------------------- 2 files changed, 1 insertion(+), 90 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 03dff8432d..55a3bfa789 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,11 +1,10 @@ import os -import io import sys from test.support import TESTFN, rmtree, unlink, captured_stdout import unittest import trace -from trace import CoverageResults, Trace +from trace import Trace from test.tracedmodules import testmod @@ -366,50 +365,5 @@ class Test_Ignore(unittest.TestCase): self.assertTrue(ignore.names(jn('bar', 'baz.py'), 'baz')) -class TestDeprecatedMethods(unittest.TestCase): - - def test_deprecated_usage(self): - sio = io.StringIO() - with self.assertWarns(DeprecationWarning): - trace.usage(sio) - self.assertIn('Usage:', sio.getvalue()) - - def test_deprecated_Ignore(self): - with self.assertWarns(DeprecationWarning): - trace.Ignore() - - def test_deprecated_modname(self): - with self.assertWarns(DeprecationWarning): - self.assertEqual("spam", trace.modname("spam")) - - def test_deprecated_fullmodname(self): - with self.assertWarns(DeprecationWarning): - self.assertEqual("spam", trace.fullmodname("spam")) - - def test_deprecated_find_lines_from_code(self): - with self.assertWarns(DeprecationWarning): - def foo(): - pass - trace.find_lines_from_code(foo.__code__, ["eggs"]) - - def test_deprecated_find_lines(self): - with self.assertWarns(DeprecationWarning): - def foo(): - pass - trace.find_lines(foo.__code__, ["eggs"]) - - def test_deprecated_find_strings(self): - with open(TESTFN, 'w') as fd: - self.addCleanup(unlink, TESTFN) - with self.assertWarns(DeprecationWarning): - trace.find_strings(fd.name) - - def test_deprecated_find_executable_linenos(self): - with open(TESTFN, 'w') as fd: - self.addCleanup(unlink, TESTFN) - with self.assertWarns(DeprecationWarning): - trace.find_executable_linenos(fd.name) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/trace.py b/Lib/trace.py index f108266816..b768829e9b 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -58,7 +58,6 @@ import inspect import gc import dis import pickle -from warnings import warn as _warn from time import monotonic as _time try: @@ -810,47 +809,5 @@ def main(argv=None): if not no_report: results.write_results(missing, summary=summary, coverdir=coverdir) -# Deprecated API -def usage(outfile): - _warn("The trace.usage() function is deprecated", - DeprecationWarning, 2) - _usage(outfile) - -class Ignore(_Ignore): - def __init__(self, modules=None, dirs=None): - _warn("The class trace.Ignore is deprecated", - DeprecationWarning, 2) - _Ignore.__init__(self, modules, dirs) - -def modname(path): - _warn("The trace.modname() function is deprecated", - DeprecationWarning, 2) - return _modname(path) - -def fullmodname(path): - _warn("The trace.fullmodname() function is deprecated", - DeprecationWarning, 2) - return _fullmodname(path) - -def find_lines_from_code(code, strs): - _warn("The trace.find_lines_from_code() function is deprecated", - DeprecationWarning, 2) - return _find_lines_from_code(code, strs) - -def find_lines(code, strs): - _warn("The trace.find_lines() function is deprecated", - DeprecationWarning, 2) - return _find_lines(code, strs) - -def find_strings(filename, encoding=None): - _warn("The trace.find_strings() function is deprecated", - DeprecationWarning, 2) - return _find_strings(filename, encoding=None) - -def find_executable_linenos(filename): - _warn("The trace.find_executable_linenos() function is deprecated", - DeprecationWarning, 2) - return _find_executable_linenos(filename) - if __name__=='__main__': main() -- cgit v1.2.1 From d507d724fee8dbcce5a22e10c1ea0e11f107c30e Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Mon, 11 Jan 2016 15:15:01 -0500 Subject: Issue #25486: Resurrect inspect.getargspec in 3.6. Backout a565aad5d6e1. The decision is that we shouldn't remove popular APIs (however long they are depreacted) from Python 3, while 2.7 is still around and supported. --- Lib/inspect.py | 25 +++++++++++++++++++++++++ Lib/test/test_inspect.py | 40 +++++++++++++++++++++++++++++++++++----- 2 files changed, 60 insertions(+), 5 deletions(-) (limited to 'Lib') diff --git a/Lib/inspect.py b/Lib/inspect.py index 7615e5277f..74f3b3c3bb 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1004,6 +1004,31 @@ def _getfullargs(co): varkw = co.co_varnames[nargs] return args, varargs, kwonlyargs, varkw + +ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') + +def getargspec(func): + """Get the names and default values of a function's arguments. + + A tuple of four things is returned: (args, varargs, keywords, defaults). + 'args' is a list of the argument names, including keyword-only argument names. + 'varargs' and 'keywords' are the names of the * and ** arguments or None. + 'defaults' is an n-tuple of the default values of the last n arguments. + + Use the getfullargspec() API for Python 3 code, as annotations + and keyword arguments are supported. getargspec() will raise ValueError + if the func has either annotations or keyword arguments. + """ + warnings.warn("inspect.getargspec() is deprecated, " + "use inspect.signature() instead", DeprecationWarning, + stacklevel=2) + args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ + getfullargspec(func) + if kwonlyargs or ann: + raise ValueError("Function has keyword-only arguments or annotations" + ", use getfullargspec() API which can support them") + return ArgSpec(args, varargs, varkw, defaults) + FullArgSpec = namedtuple('FullArgSpec', 'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations') diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index a88e7fdbd8..283c92268c 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -628,6 +628,18 @@ class TestClassesAndFunctions(unittest.TestCase): got = inspect.getmro(D) self.assertEqual(expected, got) + def assertArgSpecEquals(self, routine, args_e, varargs_e=None, + varkw_e=None, defaults_e=None, formatted=None): + with self.assertWarns(DeprecationWarning): + args, varargs, varkw, defaults = inspect.getargspec(routine) + self.assertEqual(args, args_e) + self.assertEqual(varargs, varargs_e) + self.assertEqual(varkw, varkw_e) + self.assertEqual(defaults, defaults_e) + if formatted is not None: + self.assertEqual(inspect.formatargspec(args, varargs, varkw, defaults), + formatted) + def assertFullArgSpecEquals(self, routine, args_e, varargs_e=None, varkw_e=None, defaults_e=None, kwonlyargs_e=[], kwonlydefaults_e=None, @@ -646,6 +658,23 @@ class TestClassesAndFunctions(unittest.TestCase): kwonlyargs, kwonlydefaults, ann), formatted) + def test_getargspec(self): + self.assertArgSpecEquals(mod.eggs, ['x', 'y'], formatted='(x, y)') + + self.assertArgSpecEquals(mod.spam, + ['a', 'b', 'c', 'd', 'e', 'f'], + 'g', 'h', (3, 4, 5), + '(a, b, c, d=3, e=4, f=5, *g, **h)') + + self.assertRaises(ValueError, self.assertArgSpecEquals, + mod2.keyworded, []) + + self.assertRaises(ValueError, self.assertArgSpecEquals, + mod2.annotated, []) + self.assertRaises(ValueError, self.assertArgSpecEquals, + mod2.keyword_only_arg, []) + + def test_getfullargspec(self): self.assertFullArgSpecEquals(mod2.keyworded, [], varargs_e='arg1', kwonlyargs_e=['arg2'], @@ -659,19 +688,20 @@ class TestClassesAndFunctions(unittest.TestCase): kwonlyargs_e=['arg'], formatted='(*, arg)') - def test_fullargspec_api_ignores_wrapped(self): + def test_argspec_api_ignores_wrapped(self): # Issue 20684: low level introspection API must ignore __wrapped__ @functools.wraps(mod.spam) def ham(x, y): pass # Basic check + self.assertArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)') self.assertFullArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)') self.assertFullArgSpecEquals(functools.partial(ham), ['x', 'y'], formatted='(x, y)') # Other variants def check_method(f): - self.assertFullArgSpecEquals(f, ['self', 'x', 'y'], - formatted='(self, x, y)') + self.assertArgSpecEquals(f, ['self', 'x', 'y'], + formatted='(self, x, y)') class C: @functools.wraps(mod.spam) def ham(self, x, y): @@ -749,11 +779,11 @@ class TestClassesAndFunctions(unittest.TestCase): with self.assertRaises(TypeError): inspect.getfullargspec(builtin) - def test_getfullargspec_method(self): + def test_getargspec_method(self): class A(object): def m(self): pass - self.assertFullArgSpecEquals(A.m, ['self']) + self.assertArgSpecEquals(A.m, ['self']) def test_classify_newstyle(self): class A(object): -- cgit v1.2.1 From dcfa7176485eb6c78afc9eb0da67ca56dc1f4ca9 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Tue, 12 Jan 2016 06:18:32 -0800 Subject: Issue25347 - Format the error message output of mock's assert_has_calls method. Patch contributed by Robert Zimmerman. --- Lib/unittest/mock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 976f663c0a..21f49fab1b 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -820,7 +820,7 @@ class NonCallableMock(Base): if expected not in all_calls: raise AssertionError( 'Calls not found.\nExpected: %r\n' - 'Actual: %r' % (calls, self.mock_calls) + 'Actual: %r' % (_CallList(calls), self.mock_calls) ) from cause return -- cgit v1.2.1 From bb1c8bc3ad4fd1258a24489d0660232354236352 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Wed, 13 Jan 2016 07:46:54 -0800 Subject: Issue22642 - Convert trace module's option handling mechanism from getopt to argparse. Patch contributed by SilentGhost. --- Lib/test/test_trace.py | 22 ++++ Lib/trace.py | 339 +++++++++++++++++++------------------------------ 2 files changed, 156 insertions(+), 205 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 55a3bfa789..1d894aaf84 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,6 +1,7 @@ import os import sys from test.support import TESTFN, rmtree, unlink, captured_stdout +from test.support.script_helper import assert_python_ok, assert_python_failure import unittest import trace @@ -364,6 +365,27 @@ class Test_Ignore(unittest.TestCase): # Matched before. self.assertTrue(ignore.names(jn('bar', 'baz.py'), 'baz')) +class TestCommandLine(unittest.TestCase): + + def test_failures(self): + _errors = ( + (b'filename is missing: required with the main options', '-l', '-T'), + (b'cannot specify both --listfuncs and (--trace or --count)', '-lc'), + (b'argument -R/--no-report: not allowed with argument -r/--report', '-rR'), + (b'must specify one of --trace, --count, --report, --listfuncs, or --trackcalls', '-g'), + (b'-r/--report requires -f/--file', '-r'), + (b'--summary can only be used with --count or --report', '-sT'), + (b'unrecognized arguments: -y', '-y')) + for message, *args in _errors: + *_, stderr = assert_python_failure('-m', 'trace', *args) + self.assertIn(message, stderr) + + def test_listfuncs_flag_success(self): + with open(TESTFN, 'w') as fd: + self.addCleanup(unlink, TESTFN) + fd.write("a = 1\n") + status, stdout, stderr = assert_python_ok('-m', 'trace', '-l', TESTFN) + self.assertIn(b'functions called:', stdout) if __name__ == '__main__': unittest.main() diff --git a/Lib/trace.py b/Lib/trace.py index b768829e9b..7afbe76713 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -48,6 +48,7 @@ Sample use, programmatically r.write_results(show_missing=True, coverdir="/tmp") """ __all__ = ['Trace', 'CoverageResults'] +import argparse import linecache import os import re @@ -76,51 +77,6 @@ else: sys.settrace(None) threading.settrace(None) -def _usage(outfile): - outfile.write("""Usage: %s [OPTIONS] [ARGS] - -Meta-options: ---help Display this help then exit. ---version Output version information then exit. - -Otherwise, exactly one of the following three options must be given: --t, --trace Print each line to sys.stdout before it is executed. --c, --count Count the number of times each line is executed - and write the counts to .cover for each - module executed, in the module's directory. - See also `--coverdir', `--file', `--no-report' below. --l, --listfuncs Keep track of which functions are executed at least - once and write the results to sys.stdout after the - program exits. --T, --trackcalls Keep track of caller/called pairs and write the - results to sys.stdout after the program exits. --r, --report Generate a report from a counts file; do not execute - any code. `--file' must specify the results file to - read, which must have been created in a previous run - with `--count --file=FILE'. - -Modifiers: --f, --file= File to accumulate counts over several runs. --R, --no-report Do not generate the coverage report files. - Useful if you want to accumulate over several runs. --C, --coverdir= Directory where the report files. The coverage - report for . is written to file - //.cover. --m, --missing Annotate executable lines that were not executed - with '>>>>>> '. --s, --summary Write a brief summary on stdout for each file. - (Can only be used with --count or --report.) --g, --timing Prefix each line with the time since the program started. - Only used while tracing. - -Filters, may be repeated multiple times: ---ignore-module= Ignore the given module(s) and its submodules - (if it is a package). Accepts comma separated - list of module names ---ignore-dir= Ignore files in the given directory (multiple - directories can be joined by os.pathsep). -""" % sys.argv[0]) - PRAGMA_NOCOVER = "#pragma NO COVER" # Simple rx to find lines with no code. @@ -264,7 +220,13 @@ class CoverageResults: def write_results(self, show_missing=True, summary=False, coverdir=None): """ - @param coverdir + Write the coverage results. + + :param show_missing: Show lines that had no hits. + :param summary: Include coverage summary per module. + :param coverdir: If None, the results of each module are placed in it's + directory, otherwise it is included in the directory + specified. """ if self.calledfuncs: print() @@ -646,168 +608,135 @@ class Trace: calledfuncs=self._calledfuncs, callers=self._callers) -def _err_exit(msg): - sys.stderr.write("%s: %s\n" % (sys.argv[0], msg)) - sys.exit(1) - -def main(argv=None): - import getopt - - if argv is None: - argv = sys.argv +def main(): + + parser = argparse.ArgumentParser() + parser.add_argument('--version', action='version', version='trace 2.0') + + grp = parser.add_argument_group('Main options', + 'One of these (or --report) must be given') + + grp.add_argument('-c', '--count', action='store_true', + help='Count the number of times each line is executed and write ' + 'the counts to .cover for each module executed, in ' + 'the module\'s directory. See also --coverdir, --file, ' + '--no-report below.') + grp.add_argument('-t', '--trace', action='store_true', + help='Print each line to sys.stdout before it is executed') + grp.add_argument('-l', '--listfuncs', action='store_true', + help='Keep track of which functions are executed at least once ' + 'and write the results to sys.stdout after the program exits. ' + 'Cannot be specified alongside --trace or --count.') + grp.add_argument('-T', '--trackcalls', action='store_true', + help='Keep track of caller/called pairs and write the results to ' + 'sys.stdout after the program exits.') + + grp = parser.add_argument_group('Modifiers') + + _grp = grp.add_mutually_exclusive_group() + _grp.add_argument('-r', '--report', action='store_true', + help='Generate a report from a counts file; does not execute any ' + 'code. --file must specify the results file to read, which ' + 'must have been created in a previous run with --count ' + '--file=FILE') + _grp.add_argument('-R', '--no-report', action='store_true', + help='Do not generate the coverage report files. ' + 'Useful if you want to accumulate over several runs.') + + grp.add_argument('-f', '--file', + help='File to accumulate counts over several runs') + grp.add_argument('-C', '--coverdir', + help='Directory where the report files go. The coverage report ' + 'for . will be written to file ' + '//.cover') + grp.add_argument('-m', '--missing', action='store_true', + help='Annotate executable lines that were not executed with ' + '">>>>>> "') + grp.add_argument('-s', '--summary', action='store_true', + help='Write a brief summary for each file to sys.stdout. ' + 'Can only be used with --count or --report') + grp.add_argument('-g', '--timing', action='store_true', + help='Prefix each line with the time since the program started. ' + 'Only used while tracing') + + grp = parser.add_argument_group('Filters', + 'Can be specified multiple times') + grp.add_argument('--ignore-module', action='append', default=[], + help='Ignore the given module(s) and its submodules' + '(if it is a package). Accepts comma separated list of ' + 'module names.') + grp.add_argument('--ignore-dir', action='append', default=[], + help='Ignore files in the given directory ' + '(multiple directories can be joined by os.pathsep).') + + parser.add_argument('filename', nargs='?', + help='file to run as main program') + parser.add_argument('arguments', nargs=argparse.REMAINDER, + help='arguments to the program') + + opts = parser.parse_args() + + if opts.ignore_dir: + rel_path = 'lib', 'python{0.major}.{0.minor}'.format(sys.version_info) + _prefix = os.path.join(sys.base_prefix, *rel_path) + _exec_prefix = os.path.join(sys.base_exec_prefix, *rel_path) + + def parse_ignore_dir(s): + s = os.path.expanduser(os.path.expandvars(s)) + s = s.replace('$prefix', _prefix).replace('$exec_prefix', _exec_prefix) + return os.path.normpath(s) + + opts.ignore_module = [mod.strip() + for i in opts.ignore_module for mod in i.split(',')] + opts.ignore_dir = [parse_ignore_dir(s) + for i in opts.ignore_dir for s in i.split(os.pathsep)] + + if opts.report: + if not opts.file: + parser.error('-r/--report requires -f/--file') + results = CoverageResults(infile=opts.file, outfile=opts.file) + return results.write_results(opts.missing, opts.summary, opts.coverdir) + + if not any([opts.trace, opts.count, opts.listfuncs, opts.trackcalls]): + parser.error('must specify one of --trace, --count, --report, ' + '--listfuncs, or --trackcalls') + + if opts.listfuncs and (opts.count or opts.trace): + parser.error('cannot specify both --listfuncs and (--trace or --count)') + + if opts.summary and not opts.count: + parser.error('--summary can only be used with --count or --report') + + if opts.filename is None: + parser.error('filename is missing: required with the main options') + + sys.argv = opts.filename, *opts.arguments + sys.path[0] = os.path.dirname(opts.filename) + + t = Trace(opts.count, opts.trace, countfuncs=opts.listfuncs, + countcallers=opts.trackcalls, ignoremods=opts.ignore_module, + ignoredirs=opts.ignore_dir, infile=opts.file, + outfile=opts.file, timing=opts.timing) try: - opts, prog_argv = getopt.getopt(argv[1:], "tcrRf:d:msC:lTg", - ["help", "version", "trace", "count", - "report", "no-report", "summary", - "file=", "missing", - "ignore-module=", "ignore-dir=", - "coverdir=", "listfuncs", - "trackcalls", "timing"]) - - except getopt.error as msg: - sys.stderr.write("%s: %s\n" % (sys.argv[0], msg)) - sys.stderr.write("Try `%s --help' for more information\n" - % sys.argv[0]) - sys.exit(1) - - trace = 0 - count = 0 - report = 0 - no_report = 0 - counts_file = None - missing = 0 - ignore_modules = [] - ignore_dirs = [] - coverdir = None - summary = 0 - listfuncs = False - countcallers = False - timing = False - - for opt, val in opts: - if opt == "--help": - _usage(sys.stdout) - sys.exit(0) - - if opt == "--version": - sys.stdout.write("trace 2.0\n") - sys.exit(0) - - if opt == "-T" or opt == "--trackcalls": - countcallers = True - continue - - if opt == "-l" or opt == "--listfuncs": - listfuncs = True - continue - - if opt == "-g" or opt == "--timing": - timing = True - continue - - if opt == "-t" or opt == "--trace": - trace = 1 - continue - - if opt == "-c" or opt == "--count": - count = 1 - continue - - if opt == "-r" or opt == "--report": - report = 1 - continue - - if opt == "-R" or opt == "--no-report": - no_report = 1 - continue - - if opt == "-f" or opt == "--file": - counts_file = val - continue - - if opt == "-m" or opt == "--missing": - missing = 1 - continue - - if opt == "-C" or opt == "--coverdir": - coverdir = val - continue - - if opt == "-s" or opt == "--summary": - summary = 1 - continue - - if opt == "--ignore-module": - for mod in val.split(","): - ignore_modules.append(mod.strip()) - continue - - if opt == "--ignore-dir": - for s in val.split(os.pathsep): - s = os.path.expandvars(s) - # should I also call expanduser? (after all, could use $HOME) - - s = s.replace("$prefix", - os.path.join(sys.base_prefix, "lib", - "python" + sys.version[:3])) - s = s.replace("$exec_prefix", - os.path.join(sys.base_exec_prefix, "lib", - "python" + sys.version[:3])) - s = os.path.normpath(s) - ignore_dirs.append(s) - continue - - assert 0, "Should never get here" - - if listfuncs and (count or trace): - _err_exit("cannot specify both --listfuncs and (--trace or --count)") - - if not (count or trace or report or listfuncs or countcallers): - _err_exit("must specify one of --trace, --count, --report, " - "--listfuncs, or --trackcalls") - - if report and no_report: - _err_exit("cannot specify both --report and --no-report") - - if report and not counts_file: - _err_exit("--report requires a --file") - - if no_report and len(prog_argv) == 0: - _err_exit("missing name of file to run") - - # everything is ready - if report: - results = CoverageResults(infile=counts_file, outfile=counts_file) - results.write_results(missing, summary=summary, coverdir=coverdir) - else: - sys.argv = prog_argv - progname = prog_argv[0] - sys.path[0] = os.path.split(progname)[0] - - t = Trace(count, trace, countfuncs=listfuncs, - countcallers=countcallers, ignoremods=ignore_modules, - ignoredirs=ignore_dirs, infile=counts_file, - outfile=counts_file, timing=timing) - try: - with open(progname) as fp: - code = compile(fp.read(), progname, 'exec') - # try to emulate __main__ namespace as much as possible - globs = { - '__file__': progname, - '__name__': '__main__', - '__package__': None, - '__cached__': None, - } - t.runctx(code, globs, globs) - except OSError as err: - _err_exit("Cannot run file %r because: %s" % (sys.argv[0], err)) - except SystemExit: - pass + with open(opts.filename) as fp: + code = compile(fp.read(), opts.filename, 'exec') + # try to emulate __main__ namespace as much as possible + globs = { + '__file__': opts.filename, + '__name__': '__main__', + '__package__': None, + '__cached__': None, + } + t.runctx(code, globs, globs) + except OSError as err: + sys.exit("Cannot run file %r because: %s" % (sys.argv[0], err)) + except SystemExit: + pass - results = t.results() + results = t.results() - if not no_report: - results.write_results(missing, summary=summary, coverdir=coverdir) + if not opts.no_report: + results.write_results(opts.missing, opts.summary, opts.coverdir) if __name__=='__main__': main() -- cgit v1.2.1 From 24c0813fd0e6aca249e3500f4a35d04bfec3f38f Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Thu, 14 Jan 2016 00:11:39 -0800 Subject: Issue #25822: Add docstrings to the fields of urllib.parse results. Patch contributed by Swati Jaiswal. --- Lib/urllib/parse.py | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 5e2155ccaf..cecd0b9f5f 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -224,8 +224,71 @@ class _NetlocResultMixinBytes(_NetlocResultMixinBase, _ResultMixinBytes): from collections import namedtuple _DefragResultBase = namedtuple('DefragResult', 'url fragment') -_SplitResultBase = namedtuple('SplitResult', 'scheme netloc path query fragment') -_ParseResultBase = namedtuple('ParseResult', 'scheme netloc path params query fragment') +_SplitResultBase = namedtuple( + 'SplitResult', 'scheme netloc path query fragment') +_ParseResultBase = namedtuple( + 'ParseResult', 'scheme netloc path params query fragment') + +_DefragResultBase.__doc__ = """ +DefragResult(url, fragment) + +A 2-tuple that contains the url without fragment identifier and the fragment +identifier as a separate argument. +""" + +_DefragResultBase.url.__doc__ = """The URL with no fragment identifier.""" + +_DefragResultBase.fragment.__doc__ = """ +Fragment identifier separated from URL, that allows indirect identification of a +secondary resource by reference to a primary resource and additional identifying +information. +""" + +_SplitResultBase.__doc__ = """ +SplitResult(scheme, netloc, path, query, fragment) + +A 5-tuple that contains the different components of a URL. Similar to +ParseResult, but does not split params. +""" + +_SplitResultBase.scheme.__doc__ = """Specifies URL scheme for the request.""" + +_SplitResultBase.netloc.__doc__ = """ +Network location where the request is made to. +""" + +_SplitResultBase.path.__doc__ = """ +The hierarchical path, such as the path to a file to download. +""" + +_SplitResultBase.query.__doc__ = """ +The query component, that contains non-hierarchical data, that along with data +in path component, identifies a resource in the scope of URI's scheme and +network location. +""" + +_SplitResultBase.fragment.__doc__ = """ +Fragment identifier, that allows indirect identification of a secondary resource +by reference to a primary resource and additional identifying information. +""" + +_ParseResultBase.__doc__ = """ +ParseResult(scheme, netloc, path, params, query, fragment) + +A 6-tuple that contains components of a parsed URL. +""" + +_ParseResultBase.scheme.__doc__ = _SplitResultBase.scheme.__doc__ +_ParseResultBase.netloc.__doc__ = _SplitResultBase.netloc.__doc__ +_ParseResultBase.path.__doc__ = _SplitResultBase.path.__doc__ +_ParseResultBase.params.__doc__ = """ +Parameters for last path element used to dereference the URI in order to provide +access to perform some operation on the resource. +""" + +_ParseResultBase.query.__doc__ = _SplitResultBase.query.__doc__ +_ParseResultBase.fragment.__doc__ = _SplitResultBase.fragment.__doc__ + # For backwards compatibility, alias _NetlocResultMixinStr # ResultBase is no longer part of the documented API, but it is -- cgit v1.2.1 From 3c0aaa30943458918669a922f60ccdf72b0d0c2e Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Thu, 14 Jan 2016 13:26:43 +0000 Subject: Issue #25940: test_ssl is working again --- Lib/test/test_ssl.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index b3aee62c7e..f95cfbacd2 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1343,7 +1343,6 @@ class MemoryBIOTests(unittest.TestCase): self.assertRaises(TypeError, bio.write, 1) -@unittest.skipIf(True, "temporarily disabled: see #25940") class NetworkedTests(unittest.TestCase): def test_connect(self): @@ -1712,7 +1711,6 @@ class NetworkedBIOTests(unittest.TestCase): % (count, func.__name__)) return ret - @unittest.skipIf(True, "temporarily disabled: see #25940") def test_handshake(self): with support.transient_internet(REMOTE_HOST): sock = socket.socket(socket.AF_INET) -- cgit v1.2.1 From 8df7769f53ac69a471cc182ee79d9eae4df79960 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 15 Jan 2016 13:33:03 -0800 Subject: Issue #25791: Raise an ImportWarning when __spec__ or __package__ are not defined for a relative import. This is the start of work to try and clean up import semantics to rely more on a module's spec than on the myriad attributes that get set on a module. Thanks to Rose Ames for the patch. --- Lib/importlib/_bootstrap.py | 6 ++++ .../test_importlib/import_/test___package__.py | 40 ++++++++++++++-------- 2 files changed, 32 insertions(+), 14 deletions(-) (limited to 'Lib') diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 6f62bb35fe..9adcf7b889 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1032,8 +1032,14 @@ def _calc___package__(globals): to represent that its proper value is unknown. """ + spec = globals.get('__spec__') + if spec is not None: + return spec.parent package = globals.get('__package__') if package is None: + _warnings.warn("can't resolve package from __spec__ or __package__, " + "falling back on __name__ and __path__", + ImportWarning, stacklevel=3) package = globals['__name__'] if '__path__' not in globals: package = package.rpartition('.')[0] diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index c7d3a2a204..08e41c9ccc 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -33,31 +33,39 @@ class Using__package__: """ - def test_using___package__(self): - # [__package__] + def import_module(self, globals_): with self.mock_modules('pkg.__init__', 'pkg.fake') as importer: with util.import_state(meta_path=[importer]): self.__import__('pkg.fake') module = self.__import__('', - globals={'__package__': 'pkg.fake'}, + globals=globals_, fromlist=['attr'], level=2) + return module + + def test_using___package__(self): + # [__package__] + module = self.import_module({'__package__': 'pkg.fake'}) self.assertEqual(module.__name__, 'pkg') - def test_using___name__(self, package_as_None=False): + def test_using___name__(self): # [__name__] - globals_ = {'__name__': 'pkg.fake', '__path__': []} - if package_as_None: - globals_['__package__'] = None - with self.mock_modules('pkg.__init__', 'pkg.fake') as importer: - with util.import_state(meta_path=[importer]): - self.__import__('pkg.fake') - module = self.__import__('', globals= globals_, - fromlist=['attr'], level=2) - self.assertEqual(module.__name__, 'pkg') + module = self.import_module({'__name__': 'pkg.fake', '__path__': []}) + self.assertEqual(module.__name__, 'pkg') + + def test_warn_when_using___name__(self): + with self.assertWarns(ImportWarning): + self.import_module({'__name__': 'pkg.fake', '__path__': []}) def test_None_as___package__(self): # [None] - self.test_using___name__(package_as_None=True) + module = self.import_module({ + '__name__': 'pkg.fake', '__path__': [], '__package__': None }) + self.assertEqual(module.__name__, 'pkg') + + def test_prefers___spec__(self): + globals = {'__spec__': FakeSpec()} + with self.assertRaises(SystemError): + self.__import__('', globals, {}, ['relimport'], 1) def test_bad__package__(self): globals = {'__package__': ''} @@ -70,6 +78,10 @@ class Using__package__: self.__import__('', globals, {}, ['relimport'], 1) +class FakeSpec: + parent = '' + + class Using__package__PEP302(Using__package__): mock_modules = util.mock_modules -- cgit v1.2.1 From 2cbeccf8a76a3d881885b4704b54caa6a101e72a Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Fri, 15 Jan 2016 15:01:33 -0800 Subject: revert change 87a9dff5106c: pure Enum members again evaluate to True; update Finer Points section of docs to cover boolean evaluation; add more tests for pure and mixed boolean evaluation --- Lib/enum.py | 3 --- Lib/test/test_enum.py | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) (limited to 'Lib') diff --git a/Lib/enum.py b/Lib/enum.py index 35a9c77935..ac89d6ba26 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -482,9 +482,6 @@ class Enum(metaclass=EnumMeta): def __str__(self): return "%s.%s" % (self.__class__.__name__, self._name_) - def __bool__(self): - return bool(self._value_) - def __dir__(self): added_behavior = [ m diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index e4e6c2b51a..7985948041 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -272,11 +272,26 @@ class TestEnum(unittest.TestCase): _any_name_ = 9 def test_bool(self): + # plain Enum members are always True class Logic(Enum): true = True false = False self.assertTrue(Logic.true) - self.assertFalse(Logic.false) + self.assertTrue(Logic.false) + # unless overridden + class RealLogic(Enum): + true = True + false = False + def __bool__(self): + return bool(self._value_) + self.assertTrue(RealLogic.true) + self.assertFalse(RealLogic.false) + # mixed Enums depend on mixed-in type + class IntLogic(int, Enum): + true = 1 + false = 0 + self.assertTrue(IntLogic.true) + self.assertFalse(IntLogic.false) def test_contains(self): Season = self.Season -- cgit v1.2.1 From fb66bde91a50572b03b4246045c212e11d8ab05d Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Jan 2016 06:26:54 +0000 Subject: Issue #23883: Missing fileinput.__all__ APIs; patch by Mauro SM Rodrigues --- Lib/fileinput.py | 3 ++- Lib/test/test_fileinput.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/fileinput.py b/Lib/fileinput.py index 021e39f83a..3543653f26 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -82,7 +82,8 @@ XXX Possible additions: import sys, os __all__ = ["input", "close", "nextfile", "filename", "lineno", "filelineno", - "isfirstline", "isstdin", "FileInput"] + "fileno", "isfirstline", "isstdin", "FileInput", "hook_compressed", + "hook_encoded"] _state = None diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index 91c11668bd..ad8130468b 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -24,6 +24,7 @@ from fileinput import FileInput, hook_encoded from test.support import verbose, TESTFN, run_unittest, check_warnings from test.support import unlink as safe_unlink +from test import support from unittest import mock @@ -913,5 +914,12 @@ class Test_hook_encoded(unittest.TestCase): check('rb', ['A\n', 'B\r\n', 'C\r', 'D\u20ac']) +class MiscTest(unittest.TestCase): + + def test_all(self): + blacklist = {'DEFAULT_BUFSIZE'} + support.check__all__(self, fileinput, blacklist=blacklist) + + if __name__ == "__main__": unittest.main() -- cgit v1.2.1 From 878f77b8572d53a325847b177aaafedb74416eca Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Jan 2016 06:49:30 +0000 Subject: Issue #23883: Add missing APIs to calendar.__all__ Patch by Joel Taddei and Jacek Ko?odziej. --- Lib/calendar.py | 4 +++- Lib/test/test_calendar.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/calendar.py b/Lib/calendar.py index 196a0754fc..85baf2ee4a 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -12,7 +12,9 @@ import locale as _locale __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday", "firstweekday", "isleap", "leapdays", "weekday", "monthrange", "monthcalendar", "prmonth", "month", "prcal", "calendar", - "timegm", "month_name", "month_abbr", "day_name", "day_abbr"] + "timegm", "month_name", "month_abbr", "day_name", "day_abbr", + "Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", + "LocaleHTMLCalendar", "weekheader"] # Exception raised for bad input (with string parameter for details) error = ValueError diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index d9d3128ea8..d95ad04eb0 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -815,5 +815,14 @@ class CommandLineTestCase(unittest.TestCase): b'href="custom.css" />', stdout) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'error', 'mdays', 'January', 'February', 'EPOCH', + 'MONDAY', 'TUESDAY', 'WEDNESDAY', 'THURSDAY', 'FRIDAY', + 'SATURDAY', 'SUNDAY', 'different_locale', 'c', + 'prweek', 'week', 'format', 'formatstring', 'main'} + support.check__all__(self, calendar, blacklist=blacklist) + + if __name__ == "__main__": unittest.main() -- cgit v1.2.1 From 3e900563e00cac407672194e27987bf68a475076 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Jan 2016 06:59:13 +0000 Subject: Issue #23883: Add missing APIs to tarfile.__all__ Patch by Joel Taddei and Jacek Ko?odziej. --- Lib/tarfile.py | 5 ++++- Lib/test/test_tarfile.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 80b6e35d42..7d43b1ecc0 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -64,7 +64,10 @@ except NameError: pass # from tarfile import * -__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"] +__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError", "ReadError", + "CompressionError", "StreamError", "ExtractError", "HeaderError", + "ENCODING", "USTAR_FORMAT", "GNU_FORMAT", "PAX_FORMAT", + "DEFAULT_FORMAT", "open"] #--------------------------------------------------------- # tar constants diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 1412cae111..fee1a6c2ef 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1997,6 +1997,24 @@ class MiscTest(unittest.TestCase): with self.assertRaises(ValueError): tarfile.itn(0x10000000000, 6, tarfile.GNU_FORMAT) + def test__all__(self): + blacklist = {'version', 'bltn_open', 'symlink_exception', + 'NUL', 'BLOCKSIZE', 'RECORDSIZE', 'GNU_MAGIC', + 'POSIX_MAGIC', 'LENGTH_NAME', 'LENGTH_LINK', + 'LENGTH_PREFIX', 'REGTYPE', 'AREGTYPE', 'LNKTYPE', + 'SYMTYPE', 'CHRTYPE', 'BLKTYPE', 'DIRTYPE', 'FIFOTYPE', + 'CONTTYPE', 'GNUTYPE_LONGNAME', 'GNUTYPE_LONGLINK', + 'GNUTYPE_SPARSE', 'XHDTYPE', 'XGLTYPE', 'SOLARIS_XHDTYPE', + 'SUPPORTED_TYPES', 'REGULAR_TYPES', 'GNU_TYPES', + 'PAX_FIELDS', 'PAX_NAME_FIELDS', 'PAX_NUMBER_FIELDS', + 'stn', 'nts', 'nti', 'itn', 'calc_chksums', 'copyfileobj', + 'filemode', + 'EmptyHeaderError', 'TruncatedHeaderError', + 'EOFHeaderError', 'InvalidHeaderError', + 'SubsequentHeaderError', 'ExFileObject', 'TarIter', + 'main'} + support.check__all__(self, tarfile, blacklist=blacklist) + class CommandLineTest(unittest.TestCase): -- cgit v1.2.1 From 33631c9f386f7d55d3c2dde68a3764ed5844c942 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 16 Jan 2016 11:05:11 +0200 Subject: Issue #23883: Removed redundant names from blacklists. --- Lib/test/test_calendar.py | 2 +- Lib/test/test_tarfile.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index d95ad04eb0..6dad058e77 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -817,7 +817,7 @@ class CommandLineTestCase(unittest.TestCase): class MiscTestCase(unittest.TestCase): def test__all__(self): - blacklist = {'error', 'mdays', 'January', 'February', 'EPOCH', + blacklist = {'mdays', 'January', 'February', 'EPOCH', 'MONDAY', 'TUESDAY', 'WEDNESDAY', 'THURSDAY', 'FRIDAY', 'SATURDAY', 'SUNDAY', 'different_locale', 'c', 'prweek', 'week', 'format', 'formatstring', 'main'} diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index fee1a6c2ef..9ffa2b5cf4 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1998,7 +1998,7 @@ class MiscTest(unittest.TestCase): tarfile.itn(0x10000000000, 6, tarfile.GNU_FORMAT) def test__all__(self): - blacklist = {'version', 'bltn_open', 'symlink_exception', + blacklist = {'version', 'symlink_exception', 'NUL', 'BLOCKSIZE', 'RECORDSIZE', 'GNU_MAGIC', 'POSIX_MAGIC', 'LENGTH_NAME', 'LENGTH_LINK', 'LENGTH_PREFIX', 'REGTYPE', 'AREGTYPE', 'LNKTYPE', @@ -2011,7 +2011,7 @@ class MiscTest(unittest.TestCase): 'filemode', 'EmptyHeaderError', 'TruncatedHeaderError', 'EOFHeaderError', 'InvalidHeaderError', - 'SubsequentHeaderError', 'ExFileObject', 'TarIter', + 'SubsequentHeaderError', 'ExFileObject', 'main'} support.check__all__(self, tarfile, blacklist=blacklist) -- cgit v1.2.1 From 884c0718c06be6f0b81a7ae70cd2c0d587f03264 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Jan 2016 11:01:14 +0000 Subject: Issue #23883: grp and pwd are None on Windows --- Lib/test/test_tarfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 9ffa2b5cf4..03875abe84 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1998,7 +1998,7 @@ class MiscTest(unittest.TestCase): tarfile.itn(0x10000000000, 6, tarfile.GNU_FORMAT) def test__all__(self): - blacklist = {'version', 'symlink_exception', + blacklist = {'version', 'grp', 'pwd', 'symlink_exception', 'NUL', 'BLOCKSIZE', 'RECORDSIZE', 'GNU_MAGIC', 'POSIX_MAGIC', 'LENGTH_NAME', 'LENGTH_LINK', 'LENGTH_PREFIX', 'REGTYPE', 'AREGTYPE', 'LNKTYPE', -- cgit v1.2.1 From 443c383258fe9954a696724f98d448e555f03173 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Jan 2016 11:25:50 +0100 Subject: Fix test_compilepath() of test_compileall Issue #26101: Exclude Lib/test/ from sys.path in test_compilepath(). The directory contains invalid Python files like Lib/test/badsyntax_pep3120.py, whereas the test ensures that all files can be compiled. --- Lib/test/test_compileall.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 439c125edb..6d8a86352f 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -103,6 +103,18 @@ class CompileallTests(unittest.TestCase): force=False, quiet=2)) def test_compile_path(self): + # Exclude Lib/test/ which contains invalid Python files like + # Lib/test/badsyntax_pep3120.py + testdir = os.path.realpath(os.path.dirname(__file__)) + if testdir in sys.path: + self.addCleanup(setattr, sys, 'path', sys.path) + + sys.path = list(sys.path) + try: + sys.path.remove(testdir) + except ValueError: + pass + self.assertTrue(compileall.compile_path(quiet=2)) with test.test_importlib.util.import_state(path=[self.directory]): -- cgit v1.2.1 From f10f99298e48e3e471801ff5454a0b2af85f7c50 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Jan 2016 12:15:08 +0100 Subject: subprocess._optim_args_from_interpreter_flags() Issue #26100: * Add subprocess._optim_args_from_interpreter_flags() * Add test.support.optim_args_from_interpreter_flags() * Use new functions in distutils, test_cmd_line_script, test_compileall and test_inspect The change enables test_details() test of test_inspect when -O or -OO command line option is used. --- Lib/distutils/util.py | 15 +++++++++------ Lib/subprocess.py | 14 ++++++++++++-- Lib/test/support/__init__.py | 5 +++++ Lib/test/test_cmd_line_script.py | 5 ++--- Lib/test/test_compileall.py | 7 +++---- Lib/test/test_inspect.py | 7 ++++--- 6 files changed, 35 insertions(+), 18 deletions(-) (limited to 'Lib') diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py index e423325de4..fdcf6fabae 100644 --- a/Lib/distutils/util.py +++ b/Lib/distutils/util.py @@ -7,8 +7,8 @@ one of the other *util.py modules. import os import re import importlib.util -import sys import string +import sys from distutils.errors import DistutilsPlatformError from distutils.dep_util import newer from distutils.spawn import spawn @@ -350,6 +350,11 @@ def byte_compile (py_files, generated in indirect mode; unless you know what you're doing, leave it set to None. """ + + # Late import to fix a bootstrap issue: _posixsubprocess is built by + # setup.py, but setup.py uses distutils. + import subprocess + # nothing is done if sys.dont_write_bytecode is True if sys.dont_write_bytecode: raise DistutilsByteCompileError('byte-compiling is disabled.') @@ -412,11 +417,9 @@ byte_compile(files, optimize=%r, force=%r, script.close() - cmd = [sys.executable, script_name] - if optimize == 1: - cmd.insert(1, "-O") - elif optimize == 2: - cmd.insert(1, "-OO") + cmd = [sys.executable] + cmd.extend(subprocess._optim_args_from_interpreter_flags()) + cmd.append(script_name) spawn(cmd, dry_run=dry_run) execute(os.remove, (script_name,), "removing %s" % script_name, dry_run=dry_run) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index d1324b8aed..640519d8db 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -520,6 +520,16 @@ DEVNULL = -3 # but it's here so that it can be imported when Python is compiled without # threads. +def _optim_args_from_interpreter_flags(): + """Return a list of command-line arguments reproducing the current + optimization settings in sys.flags.""" + args = [] + value = sys.flags.optimize + if value > 0: + args.append('-' + 'O' * value) + return args + + def _args_from_interpreter_flags(): """Return a list of command-line arguments reproducing the current settings in sys.flags and sys.warnoptions.""" @@ -527,7 +537,6 @@ def _args_from_interpreter_flags(): 'debug': 'd', # 'inspect': 'i', # 'interactive': 'i', - 'optimize': 'O', 'dont_write_bytecode': 'B', 'no_user_site': 's', 'no_site': 'S', @@ -535,8 +544,9 @@ def _args_from_interpreter_flags(): 'verbose': 'v', 'bytes_warning': 'b', 'quiet': 'q', + # -O is handled in _optim_args_from_interpreter_flags() } - args = [] + args = _optim_args_from_interpreter_flags() for flag, opt in flag_opt_map.items(): v = getattr(sys.flags, flag) if v > 0: diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2969b36acd..58dcc8a7a7 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2053,6 +2053,11 @@ def args_from_interpreter_flags(): settings in sys.flags and sys.warnoptions.""" return subprocess._args_from_interpreter_flags() +def optim_args_from_interpreter_flags(): + """Return a list of command-line arguments reproducing the current + optimization settings in sys.flags.""" + return subprocess._optim_args_from_interpreter_flags() + #============================================================ # Support for assertions about logging. #============================================================ diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index afac62aac5..9398040ee0 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -138,9 +138,8 @@ class CmdLineTest(unittest.TestCase): expected_argv0, expected_path0, expected_package, expected_loader, *cmd_line_switches): - if not __debug__: - cmd_line_switches += ('-' + 'O' * sys.flags.optimize,) - run_args = cmd_line_switches + (script_name,) + tuple(example_args) + run_args = [*support.optim_args_from_interpreter_flags(), + *cmd_line_switches, script_name, *example_args] rc, out, err = assert_python_ok(*run_args, __isolated=False) self._check_output(script_name, rc, out + err, expected_file, expected_argv0, expected_path0, diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 6d8a86352f..9b424a7250 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -231,10 +231,9 @@ class CommandLineTests(unittest.TestCase): raise unittest.SkipTest('not all entries on sys.path are writable') def _get_run_args(self, args): - interp_args = ['-S'] - if sys.flags.optimize: - interp_args.append({1 : '-O', 2 : '-OO'}[sys.flags.optimize]) - return interp_args + ['-m', 'compileall'] + list(args) + return [*support.optim_args_from_interpreter_flags(), + '-S', '-m', 'compileall', + *args] def assertRunOK(self, *args, **env_vars): rc, out, err = script_helper.assert_python_ok( diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 283c92268c..422a3d60f2 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -30,6 +30,7 @@ from test.support import MISSING_C_DOCSTRINGS, cpython_only from test.support.script_helper import assert_python_ok, assert_python_failure from test import inspect_fodder as mod from test import inspect_fodder2 as mod2 +from test import support from test.test_import import _ready_to_import @@ -3536,14 +3537,14 @@ class TestMain(unittest.TestCase): def test_details(self): module = importlib.import_module('unittest') - rc, out, err = assert_python_ok('-m', 'inspect', + args = support.optim_args_from_interpreter_flags() + rc, out, err = assert_python_ok(*args, '-m', 'inspect', 'unittest', '--details') output = out.decode() # Just a quick sanity check on the output self.assertIn(module.__name__, output) self.assertIn(module.__file__, output) - if not sys.flags.optimize: - self.assertIn(module.__cached__, output) + self.assertIn(module.__cached__, output) self.assertEqual(err, b'') -- cgit v1.2.1 From 098644441e6e0571b01102f5f9ae6775353206ac Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 18 Jan 2016 18:49:57 +0200 Subject: Issue #26129: Deprecated accepting non-integers in grp.getgrgid(). --- Lib/test/test_grp.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_grp.py b/Lib/test/test_grp.py index 272b08615d..69095a3fb9 100644 --- a/Lib/test/test_grp.py +++ b/Lib/test/test_grp.py @@ -92,5 +92,15 @@ class GroupDatabaseTestCase(unittest.TestCase): self.assertRaises(KeyError, grp.getgrgid, fakegid) + def test_noninteger_gid(self): + entries = grp.getgrall() + if not entries: + self.skipTest('no groups') + # Choose an existent gid. + gid = entries[0][2] + self.assertWarns(DeprecationWarning, grp.getgrgid, float(gid)) + self.assertWarns(DeprecationWarning, grp.getgrgid, str(gid)) + + if __name__ == "__main__": unittest.main() -- cgit v1.2.1 From d12188e25b201cb7adf634f0ec9d93ca6921ad72 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 19 Jan 2016 14:09:33 +0200 Subject: Issue #16620: Got rid of using undocumented function glob.glob1(). --- Lib/msilib/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/msilib/__init__.py b/Lib/msilib/__init__.py index 6d0a28f876..823644a006 100644 --- a/Lib/msilib/__init__.py +++ b/Lib/msilib/__init__.py @@ -1,7 +1,7 @@ # Copyright (C) 2005 Martin v. Löwis # Licensed to PSF under a Contributor Agreement. from _msi import * -import glob +import fnmatch import os import re import string @@ -379,7 +379,13 @@ class Directory: def glob(self, pattern, exclude = None): """Add a list of files to the current component as specified in the glob pattern. Individual files can be excluded in the exclude list.""" - files = glob.glob1(self.absolute, pattern) + try: + files = os.listdir(self.absolute) + except OSError: + return [] + if pattern[:1] != '.': + files = (f for f in files if f[0] != '.') + files = fnmatch.filter(files, pattern) for f in files: if exclude and f in exclude: continue self.add_file(f) -- cgit v1.2.1 From fd429314d44df33056e2b10999b9d383a1967ee2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 20 Jan 2016 12:16:21 +0100 Subject: co_lnotab supports negative line number delta Issue #26107: The format of the co_lnotab attribute of code objects changes to support negative line number delta. Changes: * assemble_lnotab(): if line number delta is less than -128 or greater than 127, emit multiple (offset_delta, lineno_delta) in co_lnotab * update functions decoding co_lnotab to use signed 8-bit integers - dis.findlinestarts() - PyCode_Addr2Line() - _PyCode_CheckLineNumber() - frame_setlineno() * update lnotab_notes.txt * increase importlib MAGIC_NUMBER to 3361 * document the change in What's New in Python 3.6 * cleanup also PyCode_Optimize() to use better variable names --- Lib/dis.py | 7 +++++-- Lib/importlib/_bootstrap_external.py | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'Lib') diff --git a/Lib/dis.py b/Lib/dis.py index 3540b4714d..2e80e178f0 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -397,8 +397,8 @@ def findlinestarts(code): Generate pairs (offset, lineno) as described in Python/compile.c. """ - byte_increments = list(code.co_lnotab[0::2]) - line_increments = list(code.co_lnotab[1::2]) + byte_increments = code.co_lnotab[0::2] + line_increments = code.co_lnotab[1::2] lastlineno = None lineno = code.co_firstlineno @@ -409,6 +409,9 @@ def findlinestarts(code): yield (addr, lineno) lastlineno = lineno addr += byte_incr + if line_incr >= 0x80: + # line_increments is an array of 8-bit signed integers + line_incr -= 0x100 lineno += line_incr if lineno != lastlineno: yield (addr, lineno) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index f27450153e..71098f1085 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -223,13 +223,14 @@ _code_type = type(_write_atomic.__code__) # Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations) # Python 3.5b2 3340 (fix dictionary display evaluation order #11205) # Python 3.5b2 3350 (add GET_YIELD_FROM_ITER opcode #24400) -# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) +# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483 +# Python 3.6a0 3361 (lineno delta of code.co_lnotab becomes signed) # # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually # due to the addition of new opcodes). -MAGIC_NUMBER = (3360).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3361).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' -- cgit v1.2.1 From b0b17823dbfe4e1c0b6cccadd2c3412da069981e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Jan 2016 12:22:07 +0100 Subject: site: error on sitecustomize import error Issue #26099: The site module now writes an error into stderr if sitecustomize module can be imported but executing the module raise an ImportError. Same change for usercustomize. --- Lib/site.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) (limited to 'Lib') diff --git a/Lib/site.py b/Lib/site.py index 3a8d1c3728..13ec8fa43f 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -504,9 +504,13 @@ def venv(known_paths): def execsitecustomize(): """Run custom site specific code, if available.""" try: - import sitecustomize - except ImportError: - pass + try: + import sitecustomize + except ImportError as exc: + if exc.name == 'sitecustomize': + pass + else: + raise except Exception as err: if os.environ.get("PYTHONVERBOSE"): sys.excepthook(*sys.exc_info()) @@ -520,9 +524,13 @@ def execsitecustomize(): def execusercustomize(): """Run custom user specific code, if available.""" try: - import usercustomize - except ImportError: - pass + try: + import usercustomize + except ImportError as exc: + if exc.name == 'usercustomize': + pass + else: + raise except Exception as err: if os.environ.get("PYTHONVERBOSE"): sys.excepthook(*sys.exc_info()) -- cgit v1.2.1 From 402b06649e29d500412af7a1ffe52ea762b31efd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Jan 2016 12:33:12 +0100 Subject: code_richcompare() now uses the constants types Issue #25843: When compiling code, don't merge constants if they are equal but have a different types. For example, "f1, f2 = lambda: 1, lambda: 1.0" is now correctly compiled to two different functions: f1() returns 1 (int) and f2() returns 1.0 (int), even if 1 and 1.0 are equal. Add a new _PyCode_ConstantKey() private function. --- Lib/test/test_compile.py | 82 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 8935c651cc..e0fdee3349 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -572,6 +572,88 @@ if 1: exec(memoryview(b"ax = 123")[1:-1], namespace) self.assertEqual(namespace['x'], 12) + def check_constant(self, func, expected): + for const in func.__code__.co_consts: + if repr(const) == repr(expected): + break + else: + self.fail("unable to find constant %r in %r" + % (expected, func.__code__.co_consts)) + + # Merging equal constants is not a strict requirement for the Python + # semantics, it's a more an implementation detail. + @support.cpython_only + def test_merge_constants(self): + # Issue #25843: compile() must merge constants which are equal + # and have the same type. + + def check_same_constant(const): + ns = {} + code = "f1, f2 = lambda: %r, lambda: %r" % (const, const) + exec(code, ns) + f1 = ns['f1'] + f2 = ns['f2'] + self.assertIs(f1.__code__, f2.__code__) + self.check_constant(f1, const) + self.assertEqual(repr(f1()), repr(const)) + + check_same_constant(None) + check_same_constant(0) + check_same_constant(0.0) + check_same_constant(b'abc') + check_same_constant('abc') + + # Note: "lambda: ..." emits "LOAD_CONST Ellipsis", + # whereas "lambda: Ellipsis" emits "LOAD_GLOBAL Ellipsis" + f1, f2 = lambda: ..., lambda: ... + self.assertIs(f1.__code__, f2.__code__) + self.check_constant(f1, Ellipsis) + self.assertEqual(repr(f1()), repr(Ellipsis)) + + # {0} is converted to a constant frozenset({0}) by the peephole + # optimizer + f1, f2 = lambda x: x in {0}, lambda x: x in {0} + self.assertIs(f1.__code__, f2.__code__) + self.check_constant(f1, frozenset({0})) + self.assertTrue(f1(0)) + + def test_dont_merge_constants(self): + # Issue #25843: compile() must not merge constants which are equal + # but have a different type. + + def check_different_constants(const1, const2): + ns = {} + exec("f1, f2 = lambda: %r, lambda: %r" % (const1, const2), ns) + f1 = ns['f1'] + f2 = ns['f2'] + self.assertIsNot(f1.__code__, f2.__code__) + self.check_constant(f1, const1) + self.check_constant(f2, const2) + self.assertEqual(repr(f1()), repr(const1)) + self.assertEqual(repr(f2()), repr(const2)) + + check_different_constants(0, 0.0) + check_different_constants(+0.0, -0.0) + check_different_constants((0,), (0.0,)) + + # check_different_constants() cannot be used because repr(-0j) is + # '(-0-0j)', but when '(-0-0j)' is evaluated to 0j: we loose the sign. + f1, f2 = lambda: +0.0j, lambda: -0.0j + self.assertIsNot(f1.__code__, f2.__code__) + self.check_constant(f1, +0.0j) + self.check_constant(f2, -0.0j) + self.assertEqual(repr(f1()), repr(+0.0j)) + self.assertEqual(repr(f2()), repr(-0.0j)) + + # {0} is converted to a constant frozenset({0}) by the peephole + # optimizer + f1, f2 = lambda x: x in {0}, lambda x: x in {0.0} + self.assertIsNot(f1.__code__, f2.__code__) + self.check_constant(f1, frozenset({0})) + self.check_constant(f2, frozenset({0.0})) + self.assertTrue(f1(0)) + self.assertTrue(f2(0.0)) + class TestStackSize(unittest.TestCase): # These tests check that the computed stack size for a code object -- cgit v1.2.1 From 5d45ccf6ecf4779ac897eb440a8773a59e07bc93 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Jan 2016 14:16:47 +0100 Subject: Issue #25876: test_gdb: use subprocess._args_from_interpreter_flags() to test Python with more options. --- Lib/test/test_gdb.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 78fc55c2f2..cc5aab1d1e 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -176,6 +176,7 @@ class DebuggerTests(unittest.TestCase): args = ['--eval-command=%s' % cmd for cmd in commands] args += ["--args", sys.executable] + args.extend(subprocess._args_from_interpreter_flags()) if not import_site: # -S suppresses the default 'import site' @@ -301,7 +302,9 @@ class PrettyPrintTests(DebuggerTests): 'Verify the pretty-printing of dictionaries' self.assertGdbRepr({}) self.assertGdbRepr({'foo': 'bar'}, "{'foo': 'bar'}") - self.assertGdbRepr({'foo': 'bar', 'douglas': 42}, "{'douglas': 42, 'foo': 'bar'}") + # PYTHONHASHSEED is need to get the exact item order + if not sys.flags.ignore_environment: + self.assertGdbRepr({'foo': 'bar', 'douglas': 42}, "{'douglas': 42, 'foo': 'bar'}") def test_lists(self): 'Verify the pretty-printing of lists' @@ -379,9 +382,12 @@ id(s)''') 'Verify the pretty-printing of frozensets' if (gdb_major_version, gdb_minor_version) < (7, 3): self.skipTest("pretty-printing of frozensets needs gdb 7.3 or later") - self.assertGdbRepr(frozenset(), 'frozenset()') - self.assertGdbRepr(frozenset(['a', 'b']), "frozenset({'a', 'b'})") - self.assertGdbRepr(frozenset([4, 5, 6]), "frozenset({4, 5, 6})") + self.assertGdbRepr(frozenset(), "frozenset()") + self.assertGdbRepr(frozenset(['a']), "frozenset({'a'})") + # PYTHONHASHSEED is need to get the exact frozenset item order + if not sys.flags.ignore_environment: + self.assertGdbRepr(frozenset(['a', 'b']), "frozenset({'a', 'b'})") + self.assertGdbRepr(frozenset([4, 5, 6]), "frozenset({4, 5, 6})") def test_exceptions(self): # Test a RuntimeError @@ -510,6 +516,10 @@ id(foo)''') def test_builtins_help(self): 'Ensure that the new-style class _Helper in site.py can be handled' + + if sys.flags.no_site: + self.skipTest("need site module, but -S option was used") + # (this was the issue causing tracebacks in # http://bugs.python.org/issue8032#msg100537 ) gdb_repr, gdb_output = self.get_gdb_repr('id(__builtins__.help)', import_site=True) -- cgit v1.2.1 From d0a622d9c9edc309ca05f35f62e2e95d1e38ad71 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Jan 2016 15:04:27 +0100 Subject: Issue #25876: Fix also test_set() of test_gdb when -E command line is used --- Lib/test/test_gdb.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index cc5aab1d1e..3fe15e4507 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -367,9 +367,12 @@ class PrettyPrintTests(DebuggerTests): 'Verify the pretty-printing of sets' if (gdb_major_version, gdb_minor_version) < (7, 3): self.skipTest("pretty-printing of sets needs gdb 7.3 or later") - self.assertGdbRepr(set(), 'set()') - self.assertGdbRepr(set(['a', 'b']), "{'a', 'b'}") - self.assertGdbRepr(set([4, 5, 6]), "{4, 5, 6}") + self.assertGdbRepr(set(), "set()") + self.assertGdbRepr(set(['a']), "{'a'}") + # PYTHONHASHSEED is need to get the exact frozenset item order + if not sys.flags.ignore_environment: + self.assertGdbRepr(set(['a', 'b']), "{'a', 'b'}") + self.assertGdbRepr(set([4, 5, 6]), "{4, 5, 6}") # Ensure that we handle sets containing the "dummy" key value, # which happens on deletion: -- cgit v1.2.1 From 8089b917b39ed072cbc1d03a4028cd39f0f25ecf Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 22 Jan 2016 15:25:50 -0800 Subject: Issue #25791: Warn when __package__ != __spec__.parent. In a previous change, __spec__.parent was prioritized over __package__. That is a backwards-compatibility break, but we do eventually want __spec__ to be the ground truth for module details. So this change reverts the change in semantics and instead raises an ImportWarning when __package__ != __spec__.parent to give people time to adjust to using spec objects. --- Lib/importlib/_bootstrap.py | 12 ++++++-- .../test_importlib/import_/test___package__.py | 33 +++++++++++++++------- .../import_/test_relative_imports.py | 16 +++++++---- 3 files changed, 43 insertions(+), 18 deletions(-) (limited to 'Lib') diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 9adcf7b889..f0a4e1cd62 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1032,11 +1032,17 @@ def _calc___package__(globals): to represent that its proper value is unknown. """ + package = globals.get('__package__') spec = globals.get('__spec__') - if spec is not None: + if package is not None: + if spec is not None and package != spec.parent: + _warnings.warn("__package__ != __spec__.parent " + f"({package!r} != {spec.parent!r})", + ImportWarning, stacklevel=3) + return package + elif spec is not None: return spec.parent - package = globals.get('__package__') - if package is None: + else: _warnings.warn("can't resolve package from __spec__ or __package__, " "falling back on __name__ and __path__", ImportWarning, stacklevel=3) diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index 08e41c9ccc..ddeeacafab 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -5,6 +5,7 @@ of using the typical __path__/__name__ test). """ import unittest +import warnings from .. import util @@ -38,8 +39,8 @@ class Using__package__: with util.import_state(meta_path=[importer]): self.__import__('pkg.fake') module = self.__import__('', - globals=globals_, - fromlist=['attr'], level=2) + globals=globals_, + fromlist=['attr'], level=2) return module def test_using___package__(self): @@ -49,7 +50,10 @@ class Using__package__: def test_using___name__(self): # [__name__] - module = self.import_module({'__name__': 'pkg.fake', '__path__': []}) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + module = self.import_module({'__name__': 'pkg.fake', + '__path__': []}) self.assertEqual(module.__name__, 'pkg') def test_warn_when_using___name__(self): @@ -58,14 +62,22 @@ class Using__package__: def test_None_as___package__(self): # [None] - module = self.import_module({ - '__name__': 'pkg.fake', '__path__': [], '__package__': None }) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + module = self.import_module({ + '__name__': 'pkg.fake', '__path__': [], '__package__': None }) self.assertEqual(module.__name__, 'pkg') - def test_prefers___spec__(self): - globals = {'__spec__': FakeSpec()} - with self.assertRaises(SystemError): - self.__import__('', globals, {}, ['relimport'], 1) + def test_spec_fallback(self): + # If __package__ isn't defined, fall back on __spec__.parent. + module = self.import_module({'__spec__': FakeSpec('pkg.fake')}) + self.assertEqual(module.__name__, 'pkg') + + def test_warn_when_package_and_spec_disagree(self): + # Raise an ImportWarning if __package__ != __spec__.parent. + with self.assertWarns(ImportWarning): + self.import_module({'__package__': 'pkg.fake', + '__spec__': FakeSpec('pkg.fakefake')}) def test_bad__package__(self): globals = {'__package__': ''} @@ -79,7 +91,8 @@ class Using__package__: class FakeSpec: - parent = '' + def __init__(self, parent): + self.parent = parent class Using__package__PEP302(Using__package__): diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index 28bb6f7c0c..0409f22b4d 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -2,6 +2,8 @@ from .. import util import sys import unittest +import warnings + class RelativeImports: @@ -65,9 +67,11 @@ class RelativeImports: uncache_names.append(name[:-len('.__init__')]) with util.mock_spec(*create) as importer: with util.import_state(meta_path=[importer]): - for global_ in globals_: - with util.uncache(*uncache_names): - callback(global_) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + for global_ in globals_: + with util.uncache(*uncache_names): + callback(global_) def test_module_from_module(self): @@ -204,8 +208,10 @@ class RelativeImports: def test_relative_import_no_globals(self): # No globals for a relative import is an error. - with self.assertRaises(KeyError): - self.__import__('sys', level=1) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with self.assertRaises(KeyError): + self.__import__('sys', level=1) (Frozen_RelativeImports, -- cgit v1.2.1 From b0a1802cd2b90637c94dd63064dc71bf5a3e8d53 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 22 Jan 2016 15:26:56 -0800 Subject: whitespace cleanup --- Lib/test/test_importlib/import_/test___package__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index ddeeacafab..7f64548748 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -76,8 +76,8 @@ class Using__package__: def test_warn_when_package_and_spec_disagree(self): # Raise an ImportWarning if __package__ != __spec__.parent. with self.assertWarns(ImportWarning): - self.import_module({'__package__': 'pkg.fake', - '__spec__': FakeSpec('pkg.fakefake')}) + self.import_module({'__package__': 'pkg.fake', + '__spec__': FakeSpec('pkg.fakefake')}) def test_bad__package__(self): globals = {'__package__': ''} -- cgit v1.2.1 From 7ab798cbbd346b6292a33a7ba62debaec8c17545 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 22 Jan 2016 15:55:56 -0800 Subject: Issue #25234: Skip test_eintr.test_open() under OS X to avoid hanging --- Lib/test/eintrdata/eintr_tester.py | 1 + 1 file changed, 1 insertion(+) (limited to 'Lib') diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index e1ed3da828..1cb86e5be0 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -341,6 +341,7 @@ class SocketEINTRTest(EINTRBaseTest): self._test_open("fp = open(path, 'r')\nfp.close()", self.python_open) + @unittest.skipIf(sys.platform == 'darwin', "hangs under OS X; see issue #25234") def os_open(self, path): fd = os.open(path, os.O_WRONLY) os.close(fd) -- cgit v1.2.1 From 0ee2c5b8d80a46dcbb0194a1443331f41f8d1d8f Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 22 Jan 2016 16:39:02 -0800 Subject: Issue #18018: Raise an ImportError if a relative import is attempted with no known parent package. Previously SystemError was raised if the parent package didn't exist (e.g., __package__ was set to ''). Thanks to Florent Xicluna and Yongzhi Pan for reporting the issue. --- Lib/test/test_importlib/import_/test_relative_imports.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index 0409f22b4d..1cad6b360a 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -213,6 +213,11 @@ class RelativeImports: with self.assertRaises(KeyError): self.__import__('sys', level=1) + def test_relative_import_no_package(self): + with self.assertRaises(ImportError): + self.__import__('a', {'__package__': '', '__spec__': None}, + level=1) + (Frozen_RelativeImports, Source_RelativeImports -- cgit v1.2.1 From 48920e0d00c1d7bd03ef2eebccb250b99c8a0b8b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 23 Jan 2016 13:29:02 +0100 Subject: test_gc: remove unused imports --- Lib/test/test_gc.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 1f0867d379..872ca62b5e 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -682,7 +682,6 @@ class GCTests(unittest.TestCase): # Create a reference cycle through the __main__ module and check # it gets collected at interpreter shutdown. code = """if 1: - import weakref class C: def __del__(self): print('__del__ called') @@ -696,7 +695,6 @@ class GCTests(unittest.TestCase): # Same as above, but with a non-__main__ module. with temp_dir() as script_dir: module = """if 1: - import weakref class C: def __del__(self): print('__del__ called') -- cgit v1.2.1 From e07fab7215d2763de7bf6cd73aacf3fedcbf5464 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 23 Jan 2016 13:52:05 +0100 Subject: Cleanup test_dict * Write one import per line * Sort imports by name * Add an empty line: 2 empty lines between code blocks at the module level (PEP 8) --- Lib/test/test_dict.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 3b42414225..1049be5023 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1,10 +1,12 @@ -import unittest -from test import support - -import collections, random, string +import collections import collections.abc -import gc, weakref +import gc import pickle +import random +import string +import unittest +import weakref +from test import support class DictTest(unittest.TestCase): @@ -963,5 +965,6 @@ class Dict(dict): class SubclassMappingTests(mapping_tests.BasicTestMappingProtocol): type2test = Dict + if __name__ == "__main__": unittest.main() -- cgit v1.2.1 From bfc057fcb43946cd16d923e044e79185fb82d74c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 23 Jan 2016 14:15:48 +0100 Subject: Issue #26146: marshal.loads() now uses the empty frozenset singleton --- Lib/test/test_marshal.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index c7def9a599..b378ffea00 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -135,6 +135,13 @@ class ContainerTestCase(unittest.TestCase, HelperMixin): for constructor in (set, frozenset): self.helper(constructor(self.d.keys())) + @support.cpython_only + def test_empty_frozenset_singleton(self): + # marshal.loads() must reuse the empty frozenset singleton + obj = frozenset() + obj2 = marshal.loads(marshal.dumps(obj)) + self.assertIs(obj2, obj) + class BufferTestCase(unittest.TestCase, HelperMixin): -- cgit v1.2.1 From 05b0626e91b9f17c0f0be156917a0d087d5724ae Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 26 Jan 2016 00:40:57 +0100 Subject: Add ast.Constant Issue #26146: Add a new kind of AST node: ast.Constant. It can be used by external AST optimizers, but the compiler does not emit directly such node. An optimizer can replace the following AST nodes with ast.Constant: * ast.NameConstant: None, False, True * ast.Num: int, float, complex * ast.Str: str * ast.Bytes: bytes * ast.Tuple if items are constants too: tuple * frozenset Update code to accept ast.Constant instead of ast.Num and/or ast.Str: * compiler * docstrings * ast.literal_eval() * Tools/parser/unparse.py --- Lib/ast.py | 52 ++++++++++++---------- Lib/test/test_ast.py | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 149 insertions(+), 23 deletions(-) (limited to 'Lib') diff --git a/Lib/ast.py b/Lib/ast.py index 017047275b..156a1f2773 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -35,6 +35,8 @@ def parse(source, filename='', mode='exec'): return compile(source, filename, mode, PyCF_ONLY_AST) +_NUM_TYPES = (int, float, complex) + def literal_eval(node_or_string): """ Safely evaluate an expression node or a string containing a Python @@ -47,7 +49,9 @@ def literal_eval(node_or_string): if isinstance(node_or_string, Expression): node_or_string = node_or_string.body def _convert(node): - if isinstance(node, (Str, Bytes)): + if isinstance(node, Constant): + return node.value + elif isinstance(node, (Str, Bytes)): return node.s elif isinstance(node, Num): return node.n @@ -62,24 +66,21 @@ def literal_eval(node_or_string): in zip(node.keys, node.values)) elif isinstance(node, NameConstant): return node.value - elif isinstance(node, UnaryOp) and \ - isinstance(node.op, (UAdd, USub)) and \ - isinstance(node.operand, (Num, UnaryOp, BinOp)): + elif isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)): operand = _convert(node.operand) - if isinstance(node.op, UAdd): - return + operand - else: - return - operand - elif isinstance(node, BinOp) and \ - isinstance(node.op, (Add, Sub)) and \ - isinstance(node.right, (Num, UnaryOp, BinOp)) and \ - isinstance(node.left, (Num, UnaryOp, BinOp)): + if isinstance(operand, _NUM_TYPES): + if isinstance(node.op, UAdd): + return + operand + else: + return - operand + elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)): left = _convert(node.left) right = _convert(node.right) - if isinstance(node.op, Add): - return left + right - else: - return left - right + if isinstance(left, _NUM_TYPES) and isinstance(right, _NUM_TYPES): + if isinstance(node.op, Add): + return left + right + else: + return left - right raise ValueError('malformed node or string: ' + repr(node)) return _convert(node_or_string) @@ -196,12 +197,19 @@ def get_docstring(node, clean=True): """ if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)): raise TypeError("%r can't have docstrings" % node.__class__.__name__) - if node.body and isinstance(node.body[0], Expr) and \ - isinstance(node.body[0].value, Str): - if clean: - import inspect - return inspect.cleandoc(node.body[0].value.s) - return node.body[0].value.s + if not(node.body and isinstance(node.body[0], Expr)): + return + node = node.body[0].value + if isinstance(node, Str): + text = node.s + elif isinstance(node, Constant) and isinstance(node.value, str): + text = node.value + else: + return + if clean: + import inspect + text = inspect.cleandoc(text) + return text def walk(node): diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index d3e6d35943..6d6c9bd26f 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1,7 +1,8 @@ +import ast +import dis import os import sys import unittest -import ast import weakref from test import support @@ -933,6 +934,123 @@ class ASTValidatorTests(unittest.TestCase): compile(mod, fn, "exec") +class ConstantTests(unittest.TestCase): + """Tests on the ast.Constant node type.""" + + def compile_constant(self, value): + tree = ast.parse("x = 123") + + node = tree.body[0].value + new_node = ast.Constant(value=value) + ast.copy_location(new_node, node) + tree.body[0].value = new_node + + code = compile(tree, "", "exec") + + ns = {} + exec(code, ns) + return ns['x'] + + def test_singletons(self): + for const in (None, False, True, Ellipsis, b'', frozenset()): + with self.subTest(const=const): + value = self.compile_constant(const) + self.assertIs(value, const) + + def test_values(self): + nested_tuple = (1,) + nested_frozenset = frozenset({1}) + for level in range(3): + nested_tuple = (nested_tuple, 2) + nested_frozenset = frozenset({nested_frozenset, 2}) + values = (123, 123.0, 123j, + "unicode", b'bytes', + tuple("tuple"), frozenset("frozenset"), + nested_tuple, nested_frozenset) + for value in values: + with self.subTest(value=value): + result = self.compile_constant(value) + self.assertEqual(result, value) + + def test_assign_to_constant(self): + tree = ast.parse("x = 1") + + target = tree.body[0].targets[0] + new_target = ast.Constant(value=1) + ast.copy_location(new_target, target) + tree.body[0].targets[0] = new_target + + with self.assertRaises(ValueError) as cm: + compile(tree, "string", "exec") + self.assertEqual(str(cm.exception), + "expression which can't be assigned " + "to in Store context") + + def test_get_docstring(self): + tree = ast.parse("'docstring'\nx = 1") + self.assertEqual(ast.get_docstring(tree), 'docstring') + + tree.body[0].value = ast.Constant(value='constant docstring') + self.assertEqual(ast.get_docstring(tree), 'constant docstring') + + def get_load_const(self, tree): + # Compile to bytecode, disassemble and get parameter of LOAD_CONST + # instructions + co = compile(tree, '', 'exec') + consts = [] + for instr in dis.get_instructions(co): + if instr.opname == 'LOAD_CONST': + consts.append(instr.argval) + return consts + + @support.cpython_only + def test_load_const(self): + consts = [None, + True, False, + 124, + 2.0, + 3j, + "unicode", + b'bytes', + (1, 2, 3)] + + code = '\n'.join(map(repr, consts)) + code += '\n...' + + code_consts = [const for const in consts + if (not isinstance(const, (str, int, float, complex)) + or isinstance(const, bool))] + code_consts.append(Ellipsis) + # the compiler adds a final "LOAD_CONST None" + code_consts.append(None) + + tree = ast.parse(code) + self.assertEqual(self.get_load_const(tree), code_consts) + + # Replace expression nodes with constants + for expr_node, const in zip(tree.body, consts): + assert isinstance(expr_node, ast.Expr) + new_node = ast.Constant(value=const) + ast.copy_location(new_node, expr_node.value) + expr_node.value = new_node + + self.assertEqual(self.get_load_const(tree), code_consts) + + def test_literal_eval(self): + tree = ast.parse("1 + 2") + binop = tree.body[0].value + + new_left = ast.Constant(value=10) + ast.copy_location(new_left, binop.left) + binop.left = new_left + + new_right = ast.Constant(value=20) + ast.copy_location(new_right, binop.right) + binop.right = new_right + + self.assertEqual(ast.literal_eval(binop), 30) + + def main(): if __name__ != '__main__': return -- cgit v1.2.1 From 0b8db4ae96b6117b34bb6558fa02bc0c01a04039 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 25 Jan 2016 23:00:21 -0800 Subject: Fix typo --- Lib/test/test_deque.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index c61e80bc2e..f75b3ffc5c 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -289,7 +289,7 @@ class TestBasic(unittest.TestCase): else: self.assertEqual(d.index(element, start, stop), target) - def test_insert_bug_24913(self): + def test_index_bug_24913(self): d = deque('A' * 3) with self.assertRaises(ValueError): i = d.index("Hello world", 0, 4) -- cgit v1.2.1 From 74b4e9d264d388b9d8cb6f7e5da22608a8244647 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Jan 2016 00:39:12 +0100 Subject: Issue #26146: enhance ast.Constant error message Mention the name of the invalid type in error message of AST validation for constants. Suggestion made by Joseph Jevnik on a review. --- Lib/test/test_ast.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 6d6c9bd26f..57060d833c 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -951,6 +951,12 @@ class ConstantTests(unittest.TestCase): exec(code, ns) return ns['x'] + def test_validation(self): + with self.assertRaises(TypeError) as cm: + self.compile_constant([1, 2, 3]) + self.assertEqual(str(cm.exception), + "got an invalid type in Constant: list") + def test_singletons(self): for const in (None, False, True, Ellipsis, b'', frozenset()): with self.subTest(const=const): -- cgit v1.2.1 From ca5483098f9026a131342b8654c95add2b532997 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 30 Jan 2016 12:34:12 +0200 Subject: Simply docstrings of venv module This will hopefully make maintenance of venv documentation easier. For example, see commits a4f0d76af176 and 5764cc02244d. This patch has been reviewed by Vinaj Sajip, the maintainer of venv module. --- Lib/venv/__init__.py | 40 +--------------------------------------- 1 file changed, 1 insertion(+), 39 deletions(-) (limited to 'Lib') diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 74245abbb2..fa3d2a3056 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -3,28 +3,6 @@ Virtual environment (venv) package for Python. Based on PEP 405. Copyright (C) 2011-2014 Vinay Sajip. Licensed to the PSF under a contributor agreement. - -usage: python -m venv [-h] [--system-site-packages] [--symlinks] [--clear] - [--upgrade] - ENV_DIR [ENV_DIR ...] - -Creates virtual Python environments in one or more target directories. - -positional arguments: - ENV_DIR A directory to create the environment in. - -optional arguments: - -h, --help show this help message and exit - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --symlinks Attempt to symlink rather than copy. - --clear Delete the contents of the environment directory if it - already exists, before environment creation. - --upgrade Upgrade the environment directory to use this version - of Python, assuming Python has been upgraded in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) """ import logging import os @@ -349,23 +327,7 @@ class EnvBuilder: def create(env_dir, system_site_packages=False, clear=False, symlinks=False, with_pip=False): - """ - Create a virtual environment in a directory. - - By default, makes the system (global) site-packages dir *un*available to - the created environment, and uses copying rather than symlinking for files - obtained from the source Python installation. - - :param env_dir: The target directory to create an environment in. - :param system_site_packages: If True, the system (global) site-packages - dir is available to the environment. - :param clear: If True, delete the contents of the environment directory if - it already exists, before environment creation. - :param symlinks: If True, attempt to symlink rather than copy files into - virtual environment. - :param with_pip: If True, ensure pip is installed in the virtual - environment - """ + """Create a virtual environment in a directory.""" builder = EnvBuilder(system_site_packages=system_site_packages, clear=clear, symlinks=symlinks, with_pip=with_pip) builder.create(env_dir) -- cgit v1.2.1 From 47714719f2002322524845f8c7d5c347c6e2d7cc Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 1 Feb 2016 21:21:19 -0800 Subject: merge --- Lib/test/test_deque.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 34b1be28f5..e2c6b6c76c 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -304,19 +304,20 @@ class TestBasic(unittest.TestCase): s.insert(i, 'Z') self.assertEqual(list(d), s) - def test_index_bug_26194(self): + def test_insert_bug_26194(self): data = 'ABC' - for i in range(len(data) + 1): - d = deque(data, len(data)) - d.insert(i, None) - s = list(data) - s.insert(i, None) - s.pop() - self.assertEqual(list(d), s) - if i < len(data): - self.assertIsNone(d[i]) + d = deque(data, maxlen=len(data)) + with self.assertRaises(IndexError): + d.insert(2, None) + + elements = 'ABCDEFGHI' + for i in range(-len(elements), len(elements)): + d = deque(elements, maxlen=len(elements)+1) + d.insert(i, 'Z') + if i >= 0: + self.assertEqual(d[i], 'Z') else: - self.assertTrue(None not in d) + self.assertEqual(d[i-1], 'Z') def test_imul(self): for n in (-10, -1, 0, 1, 2, 10, 1000): -- cgit v1.2.1 From 265edce9784c4e47eb1c5533dd958185e124dc49 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Fri, 5 Feb 2016 18:23:08 -0500 Subject: Fix issue 26287: While handling FORMAT_VALUE opcode, the top of stack was being corrupted if an error occurred in PyObject_Format(). --- Lib/test/test_fstring.py | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index d6f781c846..a82dedffe4 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -692,6 +692,17 @@ f'{a * x()}'""" r"f'{a(4]}'", ]) + def test_errors(self): + # see issue 26287 + self.assertAllRaise(TypeError, 'non-empty', + [r"f'{(lambda: 0):x}'", + r"f'{(0,):x}'", + ]) + self.assertAllRaise(ValueError, 'Unknown format code', + [r"f'{1000:j}'", + r"f'{1000:j}'", + ]) + def test_loop(self): for i in range(1000): self.assertEqual(f'i:{i}', 'i:' + str(i)) -- cgit v1.2.1 From 9004e3d5ff3fd639d53581ca232de02e7b8f31a8 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 8 Feb 2016 00:02:25 +0200 Subject: Issue #26039: Added zipfile.ZipInfo.from_file() and zipinfo.ZipInfo.is_dir(). Patch by Thomas Kluyver. --- Lib/test/test_zipfile.py | 15 ++++++++++ Lib/zipfile.py | 75 ++++++++++++++++++++++++++++++------------------ 2 files changed, 62 insertions(+), 28 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 2c10821a0c..8589342a80 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -3,6 +3,7 @@ import io import os import sys import importlib.util +import posixpath import time import struct import zipfile @@ -2071,5 +2072,19 @@ class LzmaUniversalNewlineTests(AbstractUniversalNewlineTests, unittest.TestCase): compression = zipfile.ZIP_LZMA +class ZipInfoTests(unittest.TestCase): + def test_from_file(self): + zi = zipfile.ZipInfo.from_file(__file__) + self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') + self.assertFalse(zi.is_dir()) + + def test_from_dir(self): + dirpath = os.path.dirname(os.path.abspath(__file__)) + zi = zipfile.ZipInfo.from_file(dirpath, 'stdlib_tests') + self.assertEqual(zi.filename, 'stdlib_tests/') + self.assertTrue(zi.is_dir()) + self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) + self.assertEqual(zi.file_size, 0) + if __name__ == "__main__": unittest.main() diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 56a2479fb3..e0598d27ed 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -371,7 +371,7 @@ class ZipInfo (object): result.append(' filemode=%r' % stat.filemode(hi)) if lo: result.append(' external_attr=%#x' % lo) - isdir = self.filename[-1:] == '/' + isdir = self.is_dir() if not isdir or self.file_size: result.append(' file_size=%r' % self.file_size) if ((not isdir or self.compress_size) and @@ -469,6 +469,41 @@ class ZipInfo (object): extra = extra[ln+4:] + @classmethod + def from_file(cls, filename, arcname=None): + """Construct an appropriate ZipInfo for a file on the filesystem. + + filename should be the path to a file or directory on the filesystem. + + arcname is the name which it will have within the archive (by default, + this will be the same as filename, but without a drive letter and with + leading path separators removed). + """ + st = os.stat(filename) + isdir = stat.S_ISDIR(st.st_mode) + mtime = time.localtime(st.st_mtime) + date_time = mtime[0:6] + # Create ZipInfo instance to store file information + if arcname is None: + arcname = filename + arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) + while arcname[0] in (os.sep, os.altsep): + arcname = arcname[1:] + if isdir: + arcname += '/' + zinfo = cls(arcname, date_time) + zinfo.external_attr = (st.st_mode & 0xFFFF) << 16 # Unix attributes + if isdir: + zinfo.file_size = 0 + zinfo.external_attr |= 0x10 # MS-DOS directory flag + else: + zinfo.file_size = st.st_size + + return zinfo + + def is_dir(self): + return self.filename[-1] == '/' + class _ZipDecrypter: """Class to handle decryption of files stored within a ZIP archive. @@ -1389,7 +1424,7 @@ class ZipFile: if upperdirs and not os.path.exists(upperdirs): os.makedirs(upperdirs) - if member.filename[-1] == '/': + if member.is_dir(): if not os.path.isdir(targetpath): os.mkdir(targetpath) return targetpath @@ -1430,29 +1465,17 @@ class ZipFile: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") - st = os.stat(filename) - isdir = stat.S_ISDIR(st.st_mode) - mtime = time.localtime(st.st_mtime) - date_time = mtime[0:6] - # Create ZipInfo instance to store file information - if arcname is None: - arcname = filename - arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) - while arcname[0] in (os.sep, os.altsep): - arcname = arcname[1:] - if isdir: - arcname += '/' - zinfo = ZipInfo(arcname, date_time) - zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes - if isdir: - zinfo.compress_type = ZIP_STORED - elif compress_type is None: - zinfo.compress_type = self.compression + zinfo = ZipInfo.from_file(filename, arcname) + + if zinfo.is_dir(): + zinfo.compress_size = 0 + zinfo.CRC = 0 else: - zinfo.compress_type = compress_type + if compress_type is not None: + zinfo.compress_type = compress_type + else: + zinfo.compress_type = self.compression - zinfo.file_size = st.st_size - zinfo.flag_bits = 0x00 with self._lock: if self._seekable: self.fp.seek(self.start_dir) @@ -1464,11 +1487,7 @@ class ZipFile: self._writecheck(zinfo) self._didModify = True - if isdir: - zinfo.file_size = 0 - zinfo.compress_size = 0 - zinfo.CRC = 0 - zinfo.external_attr |= 0x10 # MS-DOS directory flag + if zinfo.is_dir(): self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo self.fp.write(zinfo.FileHeader(False)) -- cgit v1.2.1 From a72fd9dd58ffd4d390ef02b329c5ebc0c1f4a5d2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 8 Feb 2016 01:22:47 +0200 Subject: Issue #26198: ValueError is now raised instead of TypeError on buffer overflow in parsing "es#" and "et#" format units. SystemError is now raised instead of TypeError on programmical error in parsing format string. --- Lib/test/test_capi.py | 4 ++-- Lib/test/test_getargs2.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 96666d1d16..1a743fdf2e 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -488,10 +488,10 @@ class SkipitemTest(unittest.TestCase): _testcapi.parse_tuple_and_keywords(tuple_1, dict_b, format.encode("ascii"), keywords) when_not_skipped = False - except TypeError as e: + except SystemError as e: s = "argument 1 (impossible)" when_not_skipped = (str(e) == s) - except RuntimeError as e: + except (TypeError, RuntimeError): when_not_skipped = False # test the format unit when skipped diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_getargs2.py index 64911355ce..be777af6b4 100644 --- a/Lib/test/test_getargs2.py +++ b/Lib/test/test_getargs2.py @@ -636,10 +636,10 @@ class String_TestCase(unittest.TestCase): self.assertEqual(getargs_es_hash('abc\xe9', 'latin1', buf), b'abc\xe9') self.assertEqual(buf, bytearray(b'abc\xe9\x00')) buf = bytearray(b'x'*4) - self.assertRaises(TypeError, getargs_es_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_es_hash, 'abc\xe9', 'latin1', buf) self.assertEqual(buf, bytearray(b'x'*4)) buf = bytearray() - self.assertRaises(TypeError, getargs_es_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_es_hash, 'abc\xe9', 'latin1', buf) def test_et_hash(self): from _testcapi import getargs_et_hash @@ -662,10 +662,10 @@ class String_TestCase(unittest.TestCase): self.assertEqual(getargs_et_hash('abc\xe9', 'latin1', buf), b'abc\xe9') self.assertEqual(buf, bytearray(b'abc\xe9\x00')) buf = bytearray(b'x'*4) - self.assertRaises(TypeError, getargs_et_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) self.assertEqual(buf, bytearray(b'x'*4)) buf = bytearray() - self.assertRaises(TypeError, getargs_et_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) def test_u(self): from _testcapi import getargs_u -- cgit v1.2.1 From 9bdc22bd2da949f1246ca6d504891c618635d8e2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 8 Feb 2016 16:39:05 +0200 Subject: Issue #25949: __dict__ for an OrderedDict instance is now created only when needed. --- Lib/test/test_ordered_dict.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'Lib') diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 633e90919b..e1564ba1c7 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -298,9 +298,11 @@ class OrderedDictTests: # do not save instance dictionary if not needed pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] od = OrderedDict(pairs) + self.assertIsInstance(od.__dict__, dict) self.assertIsNone(od.__reduce__()[2]) od.x = 10 - self.assertIsNotNone(od.__reduce__()[2]) + self.assertEqual(od.__dict__['x'], 10) + self.assertEqual(od.__reduce__()[2], {'x': 10}) def test_pickle_recursive(self): OrderedDict = self.OrderedDict -- cgit v1.2.1 From 76b91c2dd1380f5a703badca363a670ec3575f4a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Feb 2016 17:15:21 +0100 Subject: Simplify main() of test_ast * Use ast.parse() to get the AST for a statement * Use str%args syntax for format a line Issue #26204. --- Lib/test/test_ast.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 57060d833c..f5c4adeb6f 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1064,8 +1064,9 @@ def main(): for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), (eval_tests, "eval")): print(kind+"_results = [") - for s in statements: - print(repr(to_tuple(compile(s, "?", kind, 0x400)))+",") + for statement in statements: + tree = ast.parse(statement, "?", kind) + print("%r," % (to_tuple(tree),)) print("]") print("main()") raise SystemExit -- cgit v1.2.1 From ba4e49debad3abb913eeb2b3a4864680f64df8ac Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Feb 2016 17:57:02 +0100 Subject: Replace noop constant statement with expression * Constant statements will be ignored and the compiler will emit a SyntaxWarning. * Replace constant statement (ex: "1") with an expression statement (ex: "x=1"). * test_traceback: use context manager on the file. Issue #26204. --- Lib/test/test_inspect.py | 2 +- Lib/test/test_peepholer.py | 20 +++++++++++--------- Lib/test/test_support.py | 3 ++- Lib/test/test_sys_settrace.py | 4 ++-- Lib/test/test_traceback.py | 14 +++++++------- 5 files changed, 23 insertions(+), 20 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 422a3d60f2..98d596e5e4 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -401,7 +401,7 @@ class TestRetrievingSourceCode(GetSourceBase): self.assertEqual(normcase(inspect.getsourcefile(mod.spam)), modfile) self.assertEqual(normcase(inspect.getsourcefile(git.abuse)), modfile) fn = "_non_existing_filename_used_for_sourcefile_test.py" - co = compile("None", fn, "exec") + co = compile("x=1", fn, "exec") self.assertEqual(inspect.getsourcefile(co), None) linecache.cache[co.co_filename] = (1, None, "None", co.co_filename) try: diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 41e5091fab..b0336407c3 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1,9 +1,8 @@ import dis import re import sys -from io import StringIO +import textwrap import unittest -from math import copysign from test.bytecode_helper import BytecodeTestCase @@ -30,22 +29,25 @@ class TestTranforms(BytecodeTestCase): def test_global_as_constant(self): # LOAD_GLOBAL None/True/False --> LOAD_CONST None/True/False - def f(x): - None - None + def f(): + x = None + x = None return x - def g(x): - True + def g(): + x = True return x - def h(x): - False + def h(): + x = False return x + for func, elem in ((f, None), (g, True), (h, False)): self.assertNotInBytecode(func, 'LOAD_GLOBAL') self.assertInBytecode(func, 'LOAD_CONST', elem) + def f(): 'Adding a docstring made this test fail in Py2.5.0' return None + self.assertNotInBytecode(f, 'LOAD_GLOBAL') self.assertInBytecode(f, 'LOAD_CONST', None) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index f86ea918e1..269d9bf2b6 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -230,7 +230,8 @@ class TestSupport(unittest.TestCase): def test_check_syntax_error(self): support.check_syntax_error(self, "def class") - self.assertRaises(AssertionError, support.check_syntax_error, self, "1") + with self.assertRaises(AssertionError): + support.check_syntax_error(self, "x=1") def test_CleanImport(self): import importlib diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index ae8f845ee1..bb836233e1 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -338,8 +338,8 @@ class TraceTestCase(unittest.TestCase): def test_14_onliner_if(self): def onliners(): - if True: False - else: True + if True: x=False + else: x=True return 0 self.run_and_compare( onliners, diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index b7695d6eb4..2f739ba703 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -129,12 +129,12 @@ class SyntaxTracebackCases(unittest.TestCase): def do_test(firstlines, message, charset, lineno): # Raise the message in a subprocess, and catch the output try: - output = open(TESTFN, "w", encoding=charset) - output.write("""{0}if 1: - import traceback; - raise RuntimeError('{1}') - """.format(firstlines, message)) - output.close() + with open(TESTFN, "w", encoding=charset) as output: + output.write("""{0}if 1: + import traceback; + raise RuntimeError('{1}') + """.format(firstlines, message)) + process = subprocess.Popen([sys.executable, TESTFN], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout, stderr = process.communicate() @@ -176,7 +176,7 @@ class SyntaxTracebackCases(unittest.TestCase): do_test(" \t\f\n# coding: {0}\n".format(charset), text, charset, 5) # Issue #18960: coding spec should has no effect - do_test("0\n# coding: GBK\n", "h\xe9 ho", 'utf-8', 5) + do_test("x=0\n# coding: GBK\n", "h\xe9 ho", 'utf-8', 5) def test_print_traceback_at_exit(self): # Issue #22599: Ensure that it is possible to use the traceback module -- cgit v1.2.1 From 6887064d43421164e67d68d401562206cf5756c7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Feb 2016 18:17:58 +0100 Subject: compiler now ignores constant statements The compile ignores constant statements and emit a SyntaxWarning warning. Don't emit the warning for string statement because triple quoted string is a common syntax for multiline comments. Don't emit the warning on ellipis neither: 'def f(): ...' is a legit syntax for abstract functions. Changes: * test_ast: ignore SyntaxWarning when compiling test statements. Modify test_load_const() to use assignment expressions rather than constant expression. * test_code: add more kinds of constant statements, ignore SyntaxWarning when testing that the compiler removes constant statements. * test_grammar: ignore SyntaxWarning on the statement "1" --- Lib/test/test_ast.py | 33 +++++++++++++-------------- Lib/test/test_code.py | 58 +++++++++++++++++++++++++++++++++--------------- Lib/test/test_grammar.py | 6 ++++- 3 files changed, 61 insertions(+), 36 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index f5c4adeb6f..dbcd9f74ff 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -3,6 +3,7 @@ import dis import os import sys import unittest +import warnings import weakref from test import support @@ -239,8 +240,10 @@ class AST_Tests(unittest.TestCase): ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) self.assertEqual(to_tuple(ast_tree), o) self._assertTrueorder(ast_tree, (0, 0)) - with self.subTest(action="compiling", input=i): - compile(ast_tree, "?", kind) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=SyntaxWarning) + with self.subTest(action="compiling", input=i, kind=kind): + compile(ast_tree, "?", kind) def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice @@ -1020,27 +1023,23 @@ class ConstantTests(unittest.TestCase): b'bytes', (1, 2, 3)] - code = '\n'.join(map(repr, consts)) - code += '\n...' - - code_consts = [const for const in consts - if (not isinstance(const, (str, int, float, complex)) - or isinstance(const, bool))] - code_consts.append(Ellipsis) - # the compiler adds a final "LOAD_CONST None" - code_consts.append(None) + code = '\n'.join(['x={!r}'.format(const) for const in consts]) + code += '\nx = ...' + consts.extend((Ellipsis, None)) tree = ast.parse(code) - self.assertEqual(self.get_load_const(tree), code_consts) + self.assertEqual(self.get_load_const(tree), + consts) # Replace expression nodes with constants - for expr_node, const in zip(tree.body, consts): - assert isinstance(expr_node, ast.Expr) + for assign, const in zip(tree.body, consts): + assert isinstance(assign, ast.Assign), ast.dump(assign) new_node = ast.Constant(value=const) - ast.copy_location(new_node, expr_node.value) - expr_node.value = new_node + ast.copy_location(new_node, assign.value) + assign.value = new_node - self.assertEqual(self.get_load_const(tree), code_consts) + self.assertEqual(self.get_load_const(tree), + consts) def test_literal_eval(self): tree = ast.parse("1 + 2") diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 21b12a56e4..2cf088ce3c 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -66,24 +66,6 @@ nlocals: 1 flags: 67 consts: ('None',) ->>> def optimize_away(): -... 'doc string' -... 'not a docstring' -... 53 -... 0x53 - ->>> dump(optimize_away.__code__) -name: optimize_away -argcount: 0 -kwonlyargcount: 0 -names: () -varnames: () -cellvars: () -freevars: () -nlocals: 0 -flags: 67 -consts: ("'doc string'", 'None') - >>> def keywordonly_args(a,b,*,k1): ... return a,b,k1 ... @@ -102,8 +84,10 @@ consts: ('None',) """ +import textwrap import unittest import weakref +import warnings from test.support import run_doctest, run_unittest, cpython_only @@ -134,6 +118,44 @@ class CodeTest(unittest.TestCase): self.assertEqual(co.co_name, "funcname") self.assertEqual(co.co_firstlineno, 15) + def dump(self, co): + dump = {} + for attr in ["name", "argcount", "kwonlyargcount", "names", "varnames", + "cellvars", "freevars", "nlocals", "flags"]: + dump[attr] = getattr(co, "co_" + attr) + dump['consts'] = tuple(consts(co.co_consts)) + return dump + + def test_optimize_away(self): + ns = {} + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=SyntaxWarning) + exec(textwrap.dedent(''' + def optimize_away(): + 'doc string' + 'not a docstring' + 53 + 0x53 + b'bytes' + 1.0 + True + False + None + ... + '''), ns) + + self.assertEqual(self.dump(ns['optimize_away'].__code__), + {'name': 'optimize_away', + 'argcount': 0, + 'kwonlyargcount': 0, + 'names': (), + 'varnames': (), + 'cellvars': (), + 'freevars': (), + 'nlocals': 0, + 'flags': 67, + 'consts': ("'doc string'", 'None')}) + class CodeWeakRefTest(unittest.TestCase): diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 8f8d71ce85..2d6f5edf05 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -7,6 +7,7 @@ import unittest import sys # testing import * from sys import * +from test import support class TokenTests(unittest.TestCase): @@ -424,8 +425,11 @@ class GrammarTests(unittest.TestCase): # Tested below def test_expr_stmt(self): + msg = 'ignore constant statement' + with support.check_warnings((msg, SyntaxWarning)): + exec("1") + # (exprlist '=')* exprlist - 1 1, 2, 3 x = 1 x = 1, 2, 3 -- cgit v1.2.1 From 5ceba03e1af31e79d9dc3492bf109538f804f12b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Feb 2016 22:45:06 +0100 Subject: compiler: don't emit SyntaxWarning on const stmt Issue #26204: the compiler doesn't emit SyntaxWarning warnings anymore when constant statements are ignored. --- Lib/test/test_ast.py | 7 ++---- Lib/test/test_code.py | 58 +++++++++++++++--------------------------------- Lib/test/test_grammar.py | 6 +---- 3 files changed, 21 insertions(+), 50 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index dbcd9f74ff..a025c20006 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -3,7 +3,6 @@ import dis import os import sys import unittest -import warnings import weakref from test import support @@ -240,10 +239,8 @@ class AST_Tests(unittest.TestCase): ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) self.assertEqual(to_tuple(ast_tree), o) self._assertTrueorder(ast_tree, (0, 0)) - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=SyntaxWarning) - with self.subTest(action="compiling", input=i, kind=kind): - compile(ast_tree, "?", kind) + with self.subTest(action="compiling", input=i, kind=kind): + compile(ast_tree, "?", kind) def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 2cf088ce3c..21b12a56e4 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -66,6 +66,24 @@ nlocals: 1 flags: 67 consts: ('None',) +>>> def optimize_away(): +... 'doc string' +... 'not a docstring' +... 53 +... 0x53 + +>>> dump(optimize_away.__code__) +name: optimize_away +argcount: 0 +kwonlyargcount: 0 +names: () +varnames: () +cellvars: () +freevars: () +nlocals: 0 +flags: 67 +consts: ("'doc string'", 'None') + >>> def keywordonly_args(a,b,*,k1): ... return a,b,k1 ... @@ -84,10 +102,8 @@ consts: ('None',) """ -import textwrap import unittest import weakref -import warnings from test.support import run_doctest, run_unittest, cpython_only @@ -118,44 +134,6 @@ class CodeTest(unittest.TestCase): self.assertEqual(co.co_name, "funcname") self.assertEqual(co.co_firstlineno, 15) - def dump(self, co): - dump = {} - for attr in ["name", "argcount", "kwonlyargcount", "names", "varnames", - "cellvars", "freevars", "nlocals", "flags"]: - dump[attr] = getattr(co, "co_" + attr) - dump['consts'] = tuple(consts(co.co_consts)) - return dump - - def test_optimize_away(self): - ns = {} - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=SyntaxWarning) - exec(textwrap.dedent(''' - def optimize_away(): - 'doc string' - 'not a docstring' - 53 - 0x53 - b'bytes' - 1.0 - True - False - None - ... - '''), ns) - - self.assertEqual(self.dump(ns['optimize_away'].__code__), - {'name': 'optimize_away', - 'argcount': 0, - 'kwonlyargcount': 0, - 'names': (), - 'varnames': (), - 'cellvars': (), - 'freevars': (), - 'nlocals': 0, - 'flags': 67, - 'consts': ("'doc string'", 'None')}) - class CodeWeakRefTest(unittest.TestCase): diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 2d6f5edf05..8f8d71ce85 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -7,7 +7,6 @@ import unittest import sys # testing import * from sys import * -from test import support class TokenTests(unittest.TestCase): @@ -425,11 +424,8 @@ class GrammarTests(unittest.TestCase): # Tested below def test_expr_stmt(self): - msg = 'ignore constant statement' - with support.check_warnings((msg, SyntaxWarning)): - exec("1") - # (exprlist '=')* exprlist + 1 1, 2, 3 x = 1 x = 1, 2, 3 -- cgit v1.2.1 From fce7cdddcc3c287e649b3d1f555e5ef38de38c8c Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 10 Feb 2016 10:06:36 +0000 Subject: Issue #26243: zlib.compress() keyword argument support by Aviv Palivoda --- Lib/test/test_zlib.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'Lib') diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index ecdb5a7de8..ca30116f08 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -162,6 +162,10 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): x = zlib.compress(HAMLET_SCENE) self.assertEqual(zlib.decompress(x), HAMLET_SCENE) + def test_keywords(self): + x = zlib.compress(data=HAMLET_SCENE, level=3) + self.assertEqual(zlib.decompress(x), HAMLET_SCENE) + def test_speech128(self): # compress more data data = HAMLET_SCENE * 128 -- cgit v1.2.1 From cc3ba16af422a2ba6fd25d83e8c21e327dbf7a0d Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 10 Feb 2016 04:40:48 +0000 Subject: Issue #26136: Upgrade the generator_stop warning to DeprecationWarning Patch by Anish Shah. --- Lib/test/test_contextlib.py | 2 +- Lib/test/test_generators.py | 6 +++--- Lib/test/test_with.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 30a6377bed..04fc875e5f 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -89,7 +89,7 @@ class ContextManagerTestCase(unittest.TestCase): def woohoo(): yield try: - with self.assertWarnsRegex(PendingDeprecationWarning, + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): with woohoo(): raise stop_exc diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index b92d5ceeb9..0389b48f4e 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -245,11 +245,11 @@ class ExceptionTest(unittest.TestCase): yield with self.assertRaises(StopIteration), \ - self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + self.assertWarnsRegex(DeprecationWarning, "StopIteration"): next(gen()) - with self.assertRaisesRegex(PendingDeprecationWarning, + with self.assertRaisesRegex(DeprecationWarning, "generator .* raised StopIteration"), \ warnings.catch_warnings(): @@ -268,7 +268,7 @@ class ExceptionTest(unittest.TestCase): g = f() self.assertEqual(next(g), 1) - with self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): with self.assertRaises(StopIteration): next(g) diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index e8d789bc2c..3815062a58 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -454,7 +454,7 @@ class ExceptionalTestCase(ContextmanagerAssertionMixin, unittest.TestCase): with cm(): raise StopIteration("from with") - with self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): self.assertRaises(StopIteration, shouldThrow) def testRaisedStopIteration2(self): @@ -482,7 +482,7 @@ class ExceptionalTestCase(ContextmanagerAssertionMixin, unittest.TestCase): with cm(): raise next(iter([])) - with self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): self.assertRaises(StopIteration, shouldThrow) def testRaisedGeneratorExit1(self): -- cgit v1.2.1 From c3af758882dc81b4f1309a7c614b8ef427586a8e Mon Sep 17 00:00:00 2001 From: Charles-Fran?ois Natali Date: Wed, 10 Feb 2016 22:58:18 +0000 Subject: Issue #23992: multiprocessing: make MapResult not fail-fast upon exception. --- Lib/multiprocessing/pool.py | 20 ++++++++++++-------- Lib/test/_test_multiprocessing.py | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 8 deletions(-) (limited to 'Lib') diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 6d25469e16..ffdf42614d 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -638,22 +638,26 @@ class MapResult(ApplyResult): self._number_left = length//chunksize + bool(length % chunksize) def _set(self, i, success_result): + self._number_left -= 1 success, result = success_result - if success: + if success and self._success: self._value[i*self._chunksize:(i+1)*self._chunksize] = result - self._number_left -= 1 if self._number_left == 0: if self._callback: self._callback(self._value) del self._cache[self._job] self._event.set() else: - self._success = False - self._value = result - if self._error_callback: - self._error_callback(self._value) - del self._cache[self._job] - self._event.set() + if not success and self._success: + # only store first exception + self._success = False + self._value = result + if self._number_left == 0: + # only consider the result ready once all jobs are done + if self._error_callback: + self._error_callback(self._value) + del self._cache[self._job] + self._event.set() # # Class whose instances are returned by `Pool.imap()` diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index e9120abe5d..a59d2ba669 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1660,6 +1660,10 @@ def sqr(x, wait=0.0): def mul(x, y): return x*y +def raise_large_valuerror(wait): + time.sleep(wait) + raise ValueError("x" * 1024**2) + class SayWhenError(ValueError): pass def exception_throwing_generator(total, when): @@ -1895,6 +1899,26 @@ class _TestPool(BaseTestCase): with self.assertRaises(RuntimeError): p.apply(self._test_wrapped_exception) + def test_map_no_failfast(self): + # Issue #23992: the fail-fast behaviour when an exception is raised + # during map() would make Pool.join() deadlock, because a worker + # process would fill the result queue (after the result handler thread + # terminated, hence not draining it anymore). + + t_start = time.time() + + with self.assertRaises(ValueError): + with self.Pool(2) as p: + try: + p.map(raise_large_valuerror, [0, 1]) + finally: + time.sleep(0.5) + p.close() + p.join() + + # check that we indeed waited for all jobs + self.assertGreater(time.time() - t_start, 0.9) + def raising(): raise KeyError("key") -- cgit v1.2.1 From 8ada0a1cca05c25fc8fb09847f52efa7a0810e31 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 11 Feb 2016 12:41:40 +0200 Subject: Issue #26312: SystemError is now raised in all programming bugs with using PyArg_ParseTupleAndKeywords(). RuntimeError did raised before in some programming bugs. --- Lib/test/test_capi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Lib') diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 1a743fdf2e..74ec6c5155 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -491,7 +491,7 @@ class SkipitemTest(unittest.TestCase): except SystemError as e: s = "argument 1 (impossible)" when_not_skipped = (str(e) == s) - except (TypeError, RuntimeError): + except TypeError: when_not_skipped = False # test the format unit when skipped @@ -500,7 +500,7 @@ class SkipitemTest(unittest.TestCase): _testcapi.parse_tuple_and_keywords(empty_tuple, dict_b, optional_format.encode("ascii"), keywords) when_skipped = False - except RuntimeError as e: + except SystemError as e: s = "impossible: '{}'".format(format) when_skipped = (str(e) == s) -- cgit v1.2.1 From 2b442f7d5b61dcefc5a7a18a6e7af9de5983790c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 11 Feb 2016 13:10:36 +0200 Subject: Issue #25985: sys.version_info is now used instead of sys.version to format short Python version. --- Lib/distutils/command/bdist_msi.py | 2 +- Lib/distutils/command/bdist_wininst.py | 2 +- Lib/distutils/command/build.py | 4 ++-- Lib/distutils/command/install.py | 4 ++-- Lib/distutils/command/install_egg_info.py | 4 ++-- Lib/distutils/sysconfig.py | 2 +- Lib/distutils/tests/test_build.py | 5 +++-- Lib/importlib/_bootstrap_external.py | 2 +- Lib/pydoc.py | 6 +++--- Lib/site.py | 4 ++-- Lib/sysconfig.py | 8 ++++---- Lib/test/test_importlib/test_windows.py | 2 +- Lib/test/test_site.py | 5 +++-- Lib/test/test_venv.py | 2 +- Lib/urllib/request.py | 2 +- Lib/xmlrpc/client.py | 2 +- 16 files changed, 29 insertions(+), 27 deletions(-) (limited to 'Lib') diff --git a/Lib/distutils/command/bdist_msi.py b/Lib/distutils/command/bdist_msi.py index b3cfe9ceff..f6c21aee44 100644 --- a/Lib/distutils/command/bdist_msi.py +++ b/Lib/distutils/command/bdist_msi.py @@ -199,7 +199,7 @@ class bdist_msi(Command): target_version = self.target_version if not target_version: assert self.skip_build, "Should have already checked this" - target_version = sys.version[0:3] + target_version = '%d.%d' % sys.version_info[:2] plat_specifier = ".%s-%s" % (self.plat_name, target_version) build = self.get_finalized_command('build') build.build_lib = os.path.join(build.build_base, diff --git a/Lib/distutils/command/bdist_wininst.py b/Lib/distutils/command/bdist_wininst.py index 0c0e2c1a26..d3e1d3af22 100644 --- a/Lib/distutils/command/bdist_wininst.py +++ b/Lib/distutils/command/bdist_wininst.py @@ -141,7 +141,7 @@ class bdist_wininst(Command): target_version = self.target_version if not target_version: assert self.skip_build, "Should have already checked this" - target_version = sys.version[0:3] + target_version = '%d.%d' % sys.version_info[:2] plat_specifier = ".%s-%s" % (self.plat_name, target_version) build = self.get_finalized_command('build') build.build_lib = os.path.join(build.build_base, diff --git a/Lib/distutils/command/build.py b/Lib/distutils/command/build.py index 337dd0bfc1..c6f52e61e1 100644 --- a/Lib/distutils/command/build.py +++ b/Lib/distutils/command/build.py @@ -81,7 +81,7 @@ class build(Command): "--plat-name only supported on Windows (try " "using './configure --help' on your platform)") - plat_specifier = ".%s-%s" % (self.plat_name, sys.version[0:3]) + plat_specifier = ".%s-%d.%d" % (self.plat_name, *sys.version_info[:2]) # Make it so Python 2.x and Python 2.x with --with-pydebug don't # share the same build directories. Doing so confuses the build @@ -114,7 +114,7 @@ class build(Command): 'temp' + plat_specifier) if self.build_scripts is None: self.build_scripts = os.path.join(self.build_base, - 'scripts-' + sys.version[0:3]) + 'scripts-%d.%d' % sys.version_info[:2]) if self.executable is None: self.executable = os.path.normpath(sys.executable) diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py index 67db007a02..9474e9c599 100644 --- a/Lib/distutils/command/install.py +++ b/Lib/distutils/command/install.py @@ -290,8 +290,8 @@ class install(Command): 'dist_version': self.distribution.get_version(), 'dist_fullname': self.distribution.get_fullname(), 'py_version': py_version, - 'py_version_short': py_version[0:3], - 'py_version_nodot': py_version[0] + py_version[2], + 'py_version_short': '%d.%d' % sys.version_info[:2], + 'py_version_nodot': '%d%d' % sys.version_info[:2], 'sys_prefix': prefix, 'prefix': prefix, 'sys_exec_prefix': exec_prefix, diff --git a/Lib/distutils/command/install_egg_info.py b/Lib/distutils/command/install_egg_info.py index c2a7d649c0..0ddc7367cc 100644 --- a/Lib/distutils/command/install_egg_info.py +++ b/Lib/distutils/command/install_egg_info.py @@ -21,10 +21,10 @@ class install_egg_info(Command): def finalize_options(self): self.set_undefined_options('install_lib',('install_dir','install_dir')) - basename = "%s-%s-py%s.egg-info" % ( + basename = "%s-%s-py%d.%d.egg-info" % ( to_filename(safe_name(self.distribution.get_name())), to_filename(safe_version(self.distribution.get_version())), - sys.version[:3] + *sys.version_info[:2] ) self.target = os.path.join(self.install_dir, basename) self.outputs = [self.target] diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 573724ddd7..d203f8e42b 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -70,7 +70,7 @@ def get_python_version(): leaving off the patchlevel. Sample return values could be '1.5' or '2.2'. """ - return sys.version[:3] + return '%d.%d' % sys.version_info[:2] def get_python_inc(plat_specific=0, prefix=None): diff --git a/Lib/distutils/tests/test_build.py b/Lib/distutils/tests/test_build.py index 3391f36d4b..b020a5ba35 100644 --- a/Lib/distutils/tests/test_build.py +++ b/Lib/distutils/tests/test_build.py @@ -27,7 +27,7 @@ class BuildTestCase(support.TempdirManager, # build_platlib is 'build/lib.platform-x.x[-pydebug]' # examples: # build/lib.macosx-10.3-i386-2.7 - plat_spec = '.%s-%s' % (cmd.plat_name, sys.version[0:3]) + plat_spec = '.%s-%d.%d' % (cmd.plat_name, *sys.version_info[:2]) if hasattr(sys, 'gettotalrefcount'): self.assertTrue(cmd.build_platlib.endswith('-pydebug')) plat_spec += '-pydebug' @@ -42,7 +42,8 @@ class BuildTestCase(support.TempdirManager, self.assertEqual(cmd.build_temp, wanted) # build_scripts is build/scripts-x.x - wanted = os.path.join(cmd.build_base, 'scripts-' + sys.version[0:3]) + wanted = os.path.join(cmd.build_base, + 'scripts-%d.%d' % sys.version_info[:2]) self.assertEqual(cmd.build_scripts, wanted) # executable is os.path.normpath(sys.executable) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 71098f1085..ddb2456e9f 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -593,7 +593,7 @@ class WindowsRegistryFinder: else: registry_key = cls.REGISTRY_KEY key = registry_key.format(fullname=fullname, - sys_version=sys.version[:3]) + sys_version='%d.%d' % sys.version_info[:2]) try: with cls._open_registry(key) as hkey: filepath = _winreg.QueryValue(hkey, '') diff --git a/Lib/pydoc.py b/Lib/pydoc.py index a73298d715..5e5a8aeaa5 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1911,10 +1911,10 @@ has the same effect as typing a particular string at the help> prompt. def intro(self): self.output.write(''' -Welcome to Python %s's help utility! +Welcome to Python {0}'s help utility! If this is your first time using Python, you should definitely check out -the tutorial on the Internet at http://docs.python.org/%s/tutorial/. +the tutorial on the Internet at http://docs.python.org/{0}/tutorial/. Enter the name of any module, keyword, or topic to get help on writing Python programs and using Python modules. To quit this help utility and @@ -1924,7 +1924,7 @@ To get a list of available modules, keywords, symbols, or topics, type "modules", "keywords", "symbols", or "topics". Each module also comes with a one-line summary of what it does; to list the modules whose name or summary contain a given string such as "spam", type "modules spam". -''' % tuple([sys.version[:3]]*2)) +'''.format('%d.%d' % sys.version_info[:2])) def list(self, items, columns=4, width=80): items = list(sorted(items)) diff --git a/Lib/site.py b/Lib/site.py index 13ec8fa43f..56ba709157 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -304,7 +304,7 @@ def getsitepackages(prefixes=None): if os.sep == '/': sitepackages.append(os.path.join(prefix, "lib", - "python" + sys.version[:3], + "python%d.%d" % sys.version_info[:2], "site-packages")) else: sitepackages.append(prefix) @@ -317,7 +317,7 @@ def getsitepackages(prefixes=None): if framework: sitepackages.append( os.path.join("/Library", framework, - sys.version[:3], "site-packages")) + '%d.%d' % sys.version_info[:2], "site-packages")) return sitepackages def addsitepackages(known_paths, prefixes=None): diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 9c34be0a07..f18b1bc958 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -86,8 +86,8 @@ _SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include', # FIXME don't rely on sys.version here, its format is an implementation detail # of CPython, use sys.version_info or sys.hexversion _PY_VERSION = sys.version.split()[0] -_PY_VERSION_SHORT = sys.version[:3] -_PY_VERSION_SHORT_NO_DOT = _PY_VERSION[0] + _PY_VERSION[2] +_PY_VERSION_SHORT = '%d.%d' % sys.version_info[:2] +_PY_VERSION_SHORT_NO_DOT = '%d%d' % sys.version_info[:2] _PREFIX = os.path.normpath(sys.prefix) _BASE_PREFIX = os.path.normpath(sys.base_prefix) _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) @@ -386,7 +386,7 @@ def _generate_posix_vars(): module.build_time_vars = vars sys.modules[name] = module - pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version[:3]) + pybuilddir = 'build/lib.%s-%s' % (get_platform(), _PY_VERSION_SHORT) if hasattr(sys, "gettotalrefcount"): pybuilddir += '-pydebug' os.makedirs(pybuilddir, exist_ok=True) @@ -518,7 +518,7 @@ def get_config_vars(*args): _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX _CONFIG_VARS['py_version'] = _PY_VERSION _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT - _CONFIG_VARS['py_version_nodot'] = _PY_VERSION[0] + _PY_VERSION[2] + _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT _CONFIG_VARS['installed_base'] = _BASE_PREFIX _CONFIG_VARS['base'] = _PREFIX _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index c893bcf565..005b685cc0 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -40,7 +40,7 @@ def setup_module(machinery, name, path=None): else: root = machinery.WindowsRegistryFinder.REGISTRY_KEY key = root.format(fullname=name, - sys_version=sys.version[:3]) + sys_version='%d.%d' % sys.version_info[:2]) try: with temp_module(name, "a = 1") as location: subkey = CreateKey(HKEY_CURRENT_USER, key) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 6615080c46..574e4969ff 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -238,13 +238,14 @@ class HelperFunctionsTests(unittest.TestCase): self.assertEqual(len(dirs), 2) wanted = os.path.join('/Library', sysconfig.get_config_var("PYTHONFRAMEWORK"), - sys.version[:3], + '%d.%d' % sys.version_info[:2], 'site-packages') self.assertEqual(dirs[1], wanted) elif os.sep == '/': # OS X non-framwework builds, Linux, FreeBSD, etc self.assertEqual(len(dirs), 1) - wanted = os.path.join('xoxo', 'lib', 'python' + sys.version[:3], + wanted = os.path.join('xoxo', 'lib', + 'python%d.%d' % sys.version_info[:2], 'site-packages') self.assertEqual(dirs[0], wanted) else: diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 28b0f6c3e3..f38f52df4d 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -51,7 +51,7 @@ class BaseTest(unittest.TestCase): self.include = 'Include' else: self.bindir = 'bin' - self.lib = ('lib', 'python%s' % sys.version[:3]) + self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) self.include = 'include' if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in os.environ: executable = os.environ['__PYVENV_LAUNCHER__'] diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 4c2b9fe0e2..e3eed16131 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -133,7 +133,7 @@ __all__ = [ ] # used in User-Agent header sent -__version__ = sys.version[:3] +__version__ = '%d.%d' % sys.version_info[:2] _opener = None def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py index bf42835843..07a4f03006 100644 --- a/Lib/xmlrpc/client.py +++ b/Lib/xmlrpc/client.py @@ -151,7 +151,7 @@ def escape(s): return s.replace(">", ">",) # used in User-Agent header sent -__version__ = sys.version[:3] +__version__ = '%d.%d' % sys.version_info[:2] # xmlrpc integer limits MAXINT = 2**31-1 -- cgit v1.2.1