diff options
author | scoder <stefan_ml@behnel.de> | 2017-08-25 19:23:32 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-08-25 19:23:32 +0200 |
commit | de3436dabc3fdbda6d1a05f5ff57efcf5a9d28de (patch) | |
tree | 2f18f89312f0091cc4dfb6678f4a79aa63df3441 | |
parent | 18237a403a441403ca5a46b84e86968cfa58938d (diff) | |
parent | b43b156c5e8909315ece92fc434123d3f93b5a61 (diff) | |
download | cython-de3436dabc3fdbda6d1a05f5ff57efcf5a9d28de.tar.gz |
Merge pull request #1832 from scoder/gen_exc_handling
Repair some issues with coroutine exception handling
-rw-r--r-- | Cython/Compiler/Code.pxd | 1 | ||||
-rw-r--r-- | Cython/Compiler/Code.py | 8 | ||||
-rw-r--r-- | Cython/Compiler/ExprNodes.py | 11 | ||||
-rw-r--r-- | Cython/Compiler/Nodes.py | 32 | ||||
-rw-r--r-- | Cython/Compiler/ParseTreeTransforms.pxd | 1 | ||||
-rw-r--r-- | Cython/Compiler/ParseTreeTransforms.py | 10 | ||||
-rw-r--r-- | Cython/Utility/Coroutine.c | 95 | ||||
-rw-r--r-- | Cython/Utility/Exceptions.c | 2 | ||||
-rw-r--r-- | Tools/cevaltrace.py | 149 | ||||
-rwxr-xr-x | runtests.py | 3 | ||||
-rw-r--r-- | tests/run/generator_frame_cycle.py | 42 | ||||
-rw-r--r-- | tests/run/generators_GH1731.pyx | 70 | ||||
-rw-r--r-- | tests/run/generators_py.py | 42 |
13 files changed, 395 insertions, 71 deletions
diff --git a/Cython/Compiler/Code.pxd b/Cython/Compiler/Code.pxd index 37f5b7ee0..f7908b6f5 100644 --- a/Cython/Compiler/Code.pxd +++ b/Cython/Compiler/Code.pxd @@ -33,6 +33,7 @@ cdef class FunctionState: cdef public object return_from_error_cleanup_label # not used in __init__ ? cdef public object exc_vars + cdef public object current_except cdef public bint in_try_finally cdef public bint can_trace cdef public bint gil_owned diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py index 7d30ab9b8..fe4495143 100644 --- a/Cython/Compiler/Code.py +++ b/Cython/Compiler/Code.py @@ -6,10 +6,11 @@ from __future__ import absolute_import import cython -cython.declare(os=object, re=object, operator=object, - Naming=object, Options=object, StringEncoding=object, +cython.declare(os=object, re=object, operator=object, textwrap=object, + Template=object, Naming=object, Options=object, StringEncoding=object, Utils=object, SourceDescriptor=object, StringIOTree=object, - DebugFlags=object, basestring=object) + DebugFlags=object, basestring=object, defaultdict=object, + closing=object, partial=object) import os import re @@ -602,6 +603,7 @@ class FunctionState(object): self.in_try_finally = 0 self.exc_vars = None + self.current_except = None self.can_trace = False self.gil_owned = True diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index 95dff68f5..1ca50cb42 100644 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -9477,6 +9477,13 @@ class YieldExprNode(ExprNode): nogil=not code.funcstate.gil_owned) code.put_finish_refcount_context() + if code.funcstate.current_except is not None: + # inside of an except block => save away currently handled exception + code.putln("__Pyx_Coroutine_SwapException(%s);" % Naming.generator_cname) + else: + # no exceptions being handled => restore exception state of caller + code.putln("__Pyx_Coroutine_ResetAndClearException(%s);" % Naming.generator_cname) + code.putln("/* return from %sgenerator, %sing value */" % ( 'async ' if self.in_async_gen else '', 'await' if self.is_await else 'yield')) @@ -9540,7 +9547,7 @@ class _YieldDelegationExprNode(YieldExprNode): code.put_gotref(self.result()) def handle_iteration_exception(self, code): - code.putln("PyObject* exc_type = PyErr_Occurred();") + code.putln("PyObject* exc_type = __Pyx_PyErr_Occurred();") code.putln("if (exc_type) {") code.putln("if (likely(exc_type == PyExc_StopIteration || (exc_type != PyExc_GeneratorExit &&" " __Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)))) PyErr_Clear();") @@ -9590,7 +9597,7 @@ class AwaitIterNextExprNode(AwaitExprNode): def _generate_break(self, code): code.globalstate.use_utility_code(UtilityCode.load_cached("StopAsyncIteration", "Coroutine.c")) - code.putln("PyObject* exc_type = PyErr_Occurred();") + code.putln("PyObject* exc_type = __Pyx_PyErr_Occurred();") code.putln("if (unlikely(exc_type && (exc_type == __Pyx_PyExc_StopAsyncIteration || (" " exc_type != PyExc_StopIteration && exc_type != PyExc_GeneratorExit &&" " __Pyx_PyErr_GivenExceptionMatches(exc_type, __Pyx_PyExc_StopAsyncIteration))))) {") diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py index dc3307486..01cd67c2e 100644 --- a/Cython/Compiler/Nodes.py +++ b/Cython/Compiler/Nodes.py @@ -4054,9 +4054,10 @@ class GeneratorBodyDefNode(DefNode): self.declare_generator_body(env) def generate_function_header(self, code, proto=False): - header = "static PyObject *%s(__pyx_CoroutineObject *%s, PyObject *%s)" % ( + header = "static PyObject *%s(__pyx_CoroutineObject *%s, CYTHON_UNUSED PyThreadState *%s, PyObject *%s)" % ( self.entry.func_cname, Naming.generator_cname, + Naming.local_tstate_cname, Naming.sent_value_cname) if proto: code.putln('%s; /* proto */' % header) @@ -4157,6 +4158,7 @@ class GeneratorBodyDefNode(DefNode): code.put_xgiveref(Naming.retval_cname) else: code.put_xdecref_clear(Naming.retval_cname, py_object_type) + code.putln("__Pyx_Coroutine_ResetAndClearException(%s);" % Naming.generator_cname) code.putln('%s->resume_label = -1;' % Naming.generator_cname) # clean up as early as possible to help breaking any reference cycles code.putln('__Pyx_Coroutine_clear((PyObject*)%s);' % Naming.generator_cname) @@ -6696,6 +6698,7 @@ class TryExceptStatNode(StatNode): # else_clause StatNode or None child_attrs = ["body", "except_clauses", "else_clause"] + in_generator = False def analyse_declarations(self, env): self.body.analyse_declarations(env) @@ -6755,8 +6758,9 @@ class TryExceptStatNode(StatNode): if can_raise: # inject code before the try block to save away the exception state code.globalstate.use_utility_code(reset_exception_utility_code) - save_exc.putln("__Pyx_PyThreadState_declare") - save_exc.putln("__Pyx_PyThreadState_assign") + if not self.in_generator: + save_exc.putln("__Pyx_PyThreadState_declare") + save_exc.putln("__Pyx_PyThreadState_assign") save_exc.putln("__Pyx_ExceptionSave(%s);" % ( ', '.join(['&%s' % var for var in exc_save_vars]))) for var in exc_save_vars: @@ -6794,11 +6798,16 @@ class TryExceptStatNode(StatNode): code.put_xdecref_clear(var, py_object_type) code.put_goto(try_end_label) code.put_label(our_error_label) - code.putln("__Pyx_PyThreadState_assign") # re-assign in case a generator yielded for temp_name, temp_type in temps_to_clean_up: code.put_xdecref_clear(temp_name, temp_type) + + outer_except = code.funcstate.current_except + # Currently points to self, but the ExceptClauseNode would also be ok. Change if needed. + code.funcstate.current_except = self for except_clause in self.except_clauses: except_clause.generate_handling_code(code, except_end_label) + code.funcstate.current_except = outer_except + if not self.has_default_clause: code.put_goto(except_error_label) @@ -6813,7 +6822,6 @@ class TryExceptStatNode(StatNode): code.put_label(exit_label) code.mark_pos(self.pos, trace=False) if can_raise: - code.putln("__Pyx_PyThreadState_assign") # re-assign in case a generator yielded restore_saved_exception() code.put_goto(old_label) @@ -6822,7 +6830,6 @@ class TryExceptStatNode(StatNode): code.put_goto(try_end_label) code.put_label(except_end_label) if can_raise: - code.putln("__Pyx_PyThreadState_assign") # re-assign in case a generator yielded restore_saved_exception() if code.label_used(try_end_label): code.put_label(try_end_label) @@ -6939,8 +6946,8 @@ class ExceptClauseNode(Node): exc_args = "&%s, &%s, &%s" % tuple(exc_vars) code.putln("if (__Pyx_GetException(%s) < 0) %s" % ( exc_args, code.error_goto(self.pos))) - for x in exc_vars: - code.put_gotref(x) + for var in exc_vars: + code.put_gotref(var) if self.target: self.exc_value.set_var(exc_vars[1]) self.exc_value.generate_evaluation_code(code) @@ -6957,6 +6964,7 @@ class ExceptClauseNode(Node): code.funcstate.exc_vars = exc_vars self.body.generate_execution_code(code) code.funcstate.exc_vars = old_exc_vars + if not self.body.is_terminator: for var in exc_vars: code.put_decref_clear(var, py_object_type) @@ -7086,7 +7094,8 @@ class TryFinallyStatNode(StatNode): if preserve_error: code.putln('/*exception exit:*/{') - code.putln("__Pyx_PyThreadState_declare") + if not self.in_generator: + code.putln("__Pyx_PyThreadState_declare") if self.is_try_finally_in_nogil: code.declare_gilstate() if needs_success_cleanup: @@ -7148,7 +7157,6 @@ class TryFinallyStatNode(StatNode): if old_label == return_label: # return actually raises an (uncatchable) exception in generators that we must preserve if self.in_generator: - code.putln("__Pyx_PyThreadState_declare") exc_vars = tuple([ code.funcstate.allocate_temp(py_object_type, manage_ref=False) for _ in range(6)]) @@ -7229,8 +7237,6 @@ class TryFinallyStatNode(StatNode): if self.is_try_finally_in_nogil: code.put_ensure_gil(declare_gilstate=False) - if self.in_generator: - code.putln("__Pyx_PyThreadState_assign") # re-assign in case a generator yielded # not using preprocessor here to avoid warnings about # unused utility functions and/or temps @@ -7257,8 +7263,6 @@ class TryFinallyStatNode(StatNode): code.globalstate.use_utility_code(reset_exception_utility_code) if self.is_try_finally_in_nogil: code.put_ensure_gil(declare_gilstate=False) - if self.in_generator: - code.putln("__Pyx_PyThreadState_assign") # re-assign in case a generator yielded # not using preprocessor here to avoid warnings about # unused utility functions and/or temps diff --git a/Cython/Compiler/ParseTreeTransforms.pxd b/Cython/Compiler/ParseTreeTransforms.pxd index cfffccec7..8e7862708 100644 --- a/Cython/Compiler/ParseTreeTransforms.pxd +++ b/Cython/Compiler/ParseTreeTransforms.pxd @@ -54,6 +54,7 @@ cdef class YieldNodeCollector(TreeVisitor): cdef public list yields cdef public list returns cdef public list finallys + cdef public list excepts cdef public bint has_return_value cdef public bint has_yield cdef public bint has_await diff --git a/Cython/Compiler/ParseTreeTransforms.py b/Cython/Compiler/ParseTreeTransforms.py index 9e720c941..dc079e071 100644 --- a/Cython/Compiler/ParseTreeTransforms.py +++ b/Cython/Compiler/ParseTreeTransforms.py @@ -2475,6 +2475,7 @@ class YieldNodeCollector(TreeVisitor): self.yields = [] self.returns = [] self.finallys = [] + self.excepts = [] self.has_return_value = False self.has_yield = False self.has_await = False @@ -2502,6 +2503,10 @@ class YieldNodeCollector(TreeVisitor): self.visitchildren(node) self.finallys.append(node) + def visit_TryExceptStatNode(self, node): + self.visitchildren(node) + self.excepts.append(node) + def visit_ClassDefNode(self, node): pass @@ -2552,7 +2557,7 @@ class MarkClosureVisitor(CythonTransform): for i, yield_expr in enumerate(collector.yields, 1): yield_expr.label_num = i - for retnode in collector.returns + collector.finallys: + for retnode in collector.returns + collector.finallys + collector.excepts: retnode.in_generator = True gbody = Nodes.GeneratorBodyDefNode( @@ -2665,6 +2670,9 @@ class CreateClosureClasses(CythonTransform): class_scope = entry.type.scope class_scope.is_internal = True class_scope.is_closure_class_scope = True + if node.is_async_def or node.is_generator: + # Generators need their closure intact during cleanup as they resume to handle GeneratorExit + class_scope.directives['no_gc_clear'] = True if Options.closure_freelist_size: class_scope.directives['freelist'] = Options.closure_freelist_size diff --git a/Cython/Utility/Coroutine.c b/Cython/Utility/Coroutine.c index e8d25b63d..6f9c6af36 100644 --- a/Cython/Utility/Coroutine.c +++ b/Cython/Utility/Coroutine.c @@ -355,8 +355,9 @@ static void __Pyx_Generator_Replace_StopIteration(CYTHON_UNUSED int in_async_gen //////////////////// CoroutineBase.proto //////////////////// +//@substitute: naming -typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyObject *); +typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyThreadState *, PyObject *); typedef struct { PyObject_HEAD @@ -389,11 +390,25 @@ static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); /*proto* static PyObject *__Pyx_Coroutine_Close(PyObject *self); /*proto*/ static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); /*proto*/ -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue); /*proto*/ +// macros for exception state swapping instead of inline functions to make use of the local thread state context +#define __Pyx_Coroutine_SwapException(self) { \ + __Pyx_ExceptionSwap(&(self)->exc_type, &(self)->exc_value, &(self)->exc_traceback); \ + __Pyx_Coroutine_ResetFrameBackpointer(self); \ + } +#define __Pyx_Coroutine_ResetAndClearException(self) { \ + __Pyx_ExceptionReset((self)->exc_type, (self)->exc_value, (self)->exc_traceback); \ + (self)->exc_type = (self)->exc_value = (self)->exc_traceback = NULL; \ + } + +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyGen_FetchStopIterationValue(pvalue) \ + __Pyx_PyGen__FetchStopIterationValue($local_tstate_cname, pvalue) #else -#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue) +#define __Pyx_PyGen_FetchStopIterationValue(pvalue) \ + __Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, pvalue) #endif +static int __Pyx_PyGen__FetchStopIterationValue(PyThreadState *tstate, PyObject **pvalue); /*proto*/ +static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__pyx_CoroutineObject *self); /*proto*/ //////////////////// Coroutine.proto //////////////////// @@ -443,6 +458,7 @@ static int __pyx_Generator_init(void); /*proto*/ //@requires: Exceptions.c::PyThreadStateGet //@requires: Exceptions.c::SwapException //@requires: Exceptions.c::RaiseException +//@requires: Exceptions.c::SaveResetException //@requires: ObjectHandling.c::PyObjectCallMethod1 //@requires: ObjectHandling.c::PyObjectGetAttrStr //@requires: CommonStructures.c::FetchCommonType @@ -458,12 +474,9 @@ static int __pyx_Generator_init(void); /*proto*/ // Returns 0 if no exception or StopIteration is set. // If any other exception is set, returns -1 and leaves // pvalue unchanged. -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { +static int __Pyx_PyGen__FetchStopIterationValue(CYTHON_UNUSED PyThreadState *$local_tstate_cname, PyObject **pvalue) { PyObject *et, *ev, *tb; PyObject *value = NULL; - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign __Pyx_ErrFetch(&et, &ev, &tb); @@ -550,7 +563,6 @@ static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { *pvalue = value; return 0; } -#endif static CYTHON_INLINE void __Pyx_Coroutine_ExceptionClear(__pyx_CoroutineObject *self) { @@ -627,8 +639,9 @@ static void __Pyx__Coroutine_AlreadyTerminatedError(CYTHON_UNUSED PyObject *gen, static PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, int closing) { - PyObject *retval; __Pyx_PyThreadState_declare + PyThreadState *tstate; + PyObject *retval; assert(!self->is_running); @@ -642,8 +655,21 @@ PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, i return __Pyx_Coroutine_AlreadyTerminatedError((PyObject*)self, value, closing); } +#if CYTHON_FAST_THREAD_STATE __Pyx_PyThreadState_assign - if (value) { + tstate = $local_tstate_cname; +#else + tstate = __Pyx_PyThreadState_Current; +#endif + + // Traceback/Frame rules: + // - on entry, save external exception state in self->exc_*, restore it on exit + // - on exit, keep internally generated exceptions in self->exc_*, clear everything else + // - on entry, set "f_back" pointer of internal exception traceback to (current) outer call frame + // - on exit, clear "f_back" of internal exception traceback + // - do not touch external frames and tracebacks + + if (self->exc_type) { #if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON // FIXME: what to do in PyPy? #else @@ -653,41 +679,42 @@ PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, i PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; PyFrameObject *f = tb->tb_frame; - Py_XINCREF($local_tstate_cname->frame); + Py_XINCREF(tstate->frame); assert(f->f_back == NULL); - f->f_back = $local_tstate_cname->frame; + f->f_back = tstate->frame; } #endif + // We were in an except handler when we left, + // restore the exception state which was put aside. __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback); + // self->exc_* now holds the exception state of the caller } else { + // save away the exception state of the caller __Pyx_Coroutine_ExceptionClear(self); + __Pyx_ExceptionSave(&self->exc_type, &self->exc_value, &self->exc_traceback); } self->is_running = 1; - retval = self->body((PyObject *) self, value); + retval = self->body((PyObject *) self, tstate, value); self->is_running = 0; - if (retval) { - __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, - &self->exc_traceback); + return retval; +} + +static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__pyx_CoroutineObject *self) { + // Don't keep the reference to f_back any longer than necessary. It + // may keep a chain of frames alive or it could create a reference + // cycle. + if (likely(self->exc_traceback)) { #if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON - // FIXME: what to do in PyPy? + // FIXME: what to do in PyPy? #else - // Don't keep the reference to f_back any longer than necessary. It - // may keep a chain of frames alive or it could create a reference - // cycle. - if (self->exc_traceback) { - PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; - PyFrameObject *f = tb->tb_frame; - Py_CLEAR(f->f_back); - } + PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; + PyFrameObject *f = tb->tb_frame; + Py_CLEAR(f->f_back); #endif - } else { - __Pyx_Coroutine_ExceptionClear(self); } - - return retval; } static CYTHON_INLINE @@ -709,7 +736,7 @@ PyObject *__Pyx_Coroutine_FinishDelegation(__pyx_CoroutineObject *gen) { PyObject *ret; PyObject *val = NULL; __Pyx_Coroutine_Undelegate(gen); - __Pyx_PyGen_FetchStopIterationValue(&val); + __Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, &val); // val == NULL on failure => pass on exception ret = __Pyx_Coroutine_SendEx(gen, val, 0); Py_XDECREF(val); @@ -876,10 +903,10 @@ static PyObject *__Pyx_Coroutine_Close(PyObject *self) { if (err == 0) PyErr_SetNone(PyExc_GeneratorExit); retval = __Pyx_Coroutine_SendEx(gen, NULL, 1); - if (retval) { + if (unlikely(retval)) { const char *msg; Py_DECREF(retval); - if (0) { + if ((0)) { #ifdef __Pyx_Coroutine_USED } else if (__Pyx_Coroutine_CheckExact(self)) { msg = "coroutine ignored GeneratorExit"; @@ -899,7 +926,7 @@ static PyObject *__Pyx_Coroutine_Close(PyObject *self) { return NULL; } raised_exception = PyErr_Occurred(); - if (!raised_exception || __Pyx_PyErr_GivenExceptionMatches2(raised_exception, PyExc_GeneratorExit, PyExc_StopIteration)) { + if (likely(!raised_exception || __Pyx_PyErr_GivenExceptionMatches2(raised_exception, PyExc_GeneratorExit, PyExc_StopIteration))) { // ignore these errors if (raised_exception) PyErr_Clear(); Py_INCREF(Py_None); diff --git a/Cython/Utility/Exceptions.c b/Cython/Utility/Exceptions.c index 52aa100f9..0095ad161 100644 --- a/Cython/Utility/Exceptions.c +++ b/Cython/Utility/Exceptions.c @@ -11,6 +11,7 @@ #if CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_declare PyThreadState *$local_tstate_cname; +#define __Pyx_PyErr_Occurred() $local_tstate_cname->curexc_type #if PY_VERSION_HEX >= 0x03050000 #define __Pyx_PyThreadState_assign $local_tstate_cname = _PyThreadState_UncheckedGet(); #elif PY_VERSION_HEX >= 0x03000000 @@ -23,6 +24,7 @@ #else #define __Pyx_PyThreadState_declare #define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() PyErr_Occurred() #endif diff --git a/Tools/cevaltrace.py b/Tools/cevaltrace.py new file mode 100644 index 000000000..e2a8f6da1 --- /dev/null +++ b/Tools/cevaltrace.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +""" +Translate the byte code of a Python function into the corresponding +sequences of C code in CPython's "ceval.c". +""" + +from __future__ import print_function, absolute_import + +import re +import os.path + +from dis import get_instructions # requires Python 3.4+ + +# collapse some really boring byte codes +_COLLAPSE = {'NOP', 'LOAD_CONST', 'POP_TOP', 'JUMP_FORWARD'} +#_COLLAPSE.clear() + +_is_start = re.compile(r"\s* switch \s* \( opcode \)", re.VERBOSE).match +# Py3: TARGET(XX), Py2: case XX +_match_target = re.compile(r"\s* (?: TARGET \s* \( | case \s* ) \s* (\w+) \s* [:)]", re.VERBOSE).match +_ignored = re.compile(r"\s* PREDICTED[A-Z_]*\(", re.VERBOSE).match +_is_end = re.compile(r"\s* } \s* /\* \s* switch \s* \*/", re.VERBOSE).match + +_find_pyversion = re.compile(r'\#define \s+ PY_VERSION \s+ "([^"]+)"', re.VERBOSE).findall + +class ParseError(Exception): + def __init__(self, message="Failed to parse ceval.c"): + super(ParseError, self).__init__(message) + + +def parse_ceval(file_path): + snippets = {} + with open(file_path) as f: + lines = iter(f) + + for line in lines: + if _is_start(line): + break + else: + raise ParseError() + + targets = [] + code_lines = [] + for line in lines: + target_match = _match_target(line) + if target_match: + if code_lines: + code = ''.join(code_lines).rstrip() + for target in targets: + snippets[target] = code + del code_lines[:], targets[:] + targets.append(target_match.group(1)) + elif _ignored(line): + pass + elif _is_end(line): + break + else: + code_lines.append(line) + else: + if not snippets: + raise ParseError() + return snippets + + +def translate(func, ceval_snippets): + start_offset = 0 + code_obj = getattr(func, '__code__', None) + if code_obj and os.path.exists(code_obj.co_filename): + start_offset = code_obj.co_firstlineno + with open(code_obj.co_filename) as f: + code_line_at = { + i: line.strip() + for i, line in enumerate(f, 1) + if line.strip() + }.get + else: + code_line_at = lambda _: None + + for instr in get_instructions(func): + code_line = code_line_at(instr.starts_line) + line_no = (instr.starts_line or start_offset) - start_offset + yield line_no, code_line, instr, ceval_snippets.get(instr.opname) + + +def main(): + import sys + import importlib.util + + if len(sys.argv) < 3: + print("Usage: %s path/to/Python/ceval.c script.py ..." % sys.argv[0], file=sys.stderr) + return + + ceval_source_file = sys.argv[1] + version_header = os.path.join(os.path.dirname(ceval_source_file), '..', 'Include', 'patchlevel.h') + if os.path.exists(version_header): + with open(version_header) as f: + py_version = _find_pyversion(f.read()) + if py_version: + py_version = py_version[0] + if not sys.version.startswith(py_version + ' '): + print("Warning: disassembling with Python %s, but ceval.c has version %s" % ( + sys.version.split(None, 1)[0], + py_version, + ), file=sys.stderr) + + snippets = parse_ceval(ceval_source_file) + + for code in _COLLAPSE: + if code in snippets: + snippets[code] = '' + + for file_path in sys.argv[2:]: + module_name = os.path.basename(file_path) + print("/*######## MODULE %s ########*/" % module_name) + print('') + + spec = importlib.util.spec_from_file_location(module_name, file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + for func_name, item in sorted(vars(module).items()): + if not callable(item): + continue + print("/* FUNCTION %s */" % func_name) + print("static void") # assuming that it highlights in editors + print("%s() {" % func_name) + + last_line = None + for line_no, code_line, instr, snippet in translate(item, snippets): + if last_line != line_no: + if code_line: + print('') + print('/*# %3d %s */' % (line_no, code_line)) + print('') + last_line = line_no + + print(" %s:%s {%s" % ( + instr.opname, + ' /* %s */' % instr.argrepr if instr.arg is not None else '', + ' /* ??? */' if snippet is None else ' /* ... */ }' if snippet == '' else '', + )) + print(snippet or '') + + print("} /* FUNCTION %s */" % func_name) + + +if __name__ == '__main__': + main() diff --git a/runtests.py b/runtests.py index 4d6012654..aa16d7bc5 100755 --- a/runtests.py +++ b/runtests.py @@ -2164,6 +2164,9 @@ def runtests(options, cmd_args, coverage=None): pyximport.install(pyimport=True, build_dir=os.path.join(WORKDIR, '_pyximport'), load_py_module_on_import_failure=True, inplace=True) + import gc + gc.set_debug(gc.DEBUG_UNCOLLECTABLE) + result = test_runner.run(test_suite) if common_utility_dir and options.shard_num < 0 and options.cleanup_workdir: diff --git a/tests/run/generator_frame_cycle.py b/tests/run/generator_frame_cycle.py index 9647306cf..03a50f86c 100644 --- a/tests/run/generator_frame_cycle.py +++ b/tests/run/generator_frame_cycle.py @@ -1,13 +1,9 @@ # mode: run # tag: generator +import cython import sys -def _next(it): - if sys.version_info[0] >= 3: - return next(it) - else: - return it.next() def test_generator_frame_cycle(): """ @@ -23,8 +19,42 @@ def test_generator_frame_cycle(): finally: testit.append("I'm done") g = whoo() - _next(g) + next(g) + # Frame object cycle eval('g.throw(ValueError)', {'g': g}) del g + + return tuple(testit) + + +def test_generator_frame_cycle_with_outer_exc(): + """ + >>> test_generator_frame_cycle_with_outer_exc() + ("I'm done",) + """ + testit = [] + def whoo(): + try: + yield + except: + yield + finally: + testit.append("I'm done") + g = whoo() + next(g) + + try: + raise ValueError() + except ValueError as exc: + assert sys.exc_info()[1] is exc, sys.exc_info() + # Frame object cycle + eval('g.throw(ValueError)', {'g': g}) + # CPython 3.3 handles this incorrectly itself :) + if cython.compiled or sys.version_info[:2] not in [(3, 2), (3, 3)]: + assert sys.exc_info()[1] is exc, sys.exc_info() + del g + if cython.compiled or sys.version_info[:2] not in [(3, 2), (3, 3)]: + assert sys.exc_info()[1] is exc, sys.exc_info() + return tuple(testit) diff --git a/tests/run/generators_GH1731.pyx b/tests/run/generators_GH1731.pyx new file mode 100644 index 000000000..99cae90ca --- /dev/null +++ b/tests/run/generators_GH1731.pyx @@ -0,0 +1,70 @@ +# mode: run +# ticket: gh1731 + + +def cygen(): + yield 1 + + +def test_from_cython(g): + """ + >>> def pygen(): yield 1 + >>> test_from_cython(pygen) + Traceback (most recent call last): + ZeroDivisionError: integer division or modulo by zero + + >>> test_from_cython(cygen) + Traceback (most recent call last): + ZeroDivisionError: integer division or modulo by zero + """ + try: + 1 / 0 + except: + for _ in g(): + pass + raise + + +def test_from_python(): + """ + >>> def test(g): + ... try: + ... 1 / 0 + ... except: + ... for _ in g(): + ... pass + ... raise + + >>> def pygen(): + ... yield 1 + >>> test(pygen) # doctest: +ELLIPSIS + Traceback (most recent call last): + ZeroDivisionError: ...division ...by zero + + >>> test(cygen) # doctest: +ELLIPSIS + Traceback (most recent call last): + ZeroDivisionError: ...division ...by zero + """ + + +def test_from_console(): + """ + >>> def pygen(): yield 1 + >>> try: # doctest: +ELLIPSIS + ... 1 / 0 + ... except: + ... for _ in pygen(): + ... pass + ... raise + Traceback (most recent call last): + ZeroDivisionError: ...division ...by zero + + >>> try: # doctest: +ELLIPSIS + ... 1 / 0 + ... except: + ... for _ in cygen(): + ... pass + ... raise + Traceback (most recent call last): + ZeroDivisionError: ...division ...by zero + """ diff --git a/tests/run/generators_py.py b/tests/run/generators_py.py index 152e06a39..db4ffd1a5 100644 --- a/tests/run/generators_py.py +++ b/tests/run/generators_py.py @@ -1,6 +1,7 @@ # mode: run # tag: generators +import sys import cython @@ -147,25 +148,39 @@ def check_throw(): except ValueError: pass + def check_yield_in_except(): """ - >>> import sys - >>> orig_exc = sys.exc_info()[0] - >>> g = check_yield_in_except() - >>> next(g) - >>> next(g) - >>> orig_exc is sys.exc_info()[0] or sys.exc_info()[0] + >>> if sys.version_info[0] == 2: sys.exc_clear() + >>> try: + ... raise TypeError("RAISED !") + ... except TypeError as orig_exc: + ... assert isinstance(orig_exc, TypeError), orig_exc + ... g = check_yield_in_except() + ... print(orig_exc is sys.exc_info()[1] or sys.exc_info()) + ... next(g) + ... print(orig_exc is sys.exc_info()[1] or sys.exc_info()) + ... next(g) + ... print(orig_exc is sys.exc_info()[1] or sys.exc_info()) True + True + True + >>> next(g) + Traceback (most recent call last): + StopIteration """ try: yield raise ValueError - except ValueError: + except ValueError as exc: + assert sys.exc_info()[1] is exc, sys.exc_info() yield + if cython.compiled or sys.version_info[0] > 2: + assert sys.exc_info()[1] is exc, sys.exc_info() + def yield_in_except_throw_exc_type(): """ - >>> import sys >>> g = yield_in_except_throw_exc_type() >>> next(g) >>> g.throw(TypeError) @@ -177,12 +192,14 @@ def yield_in_except_throw_exc_type(): """ try: raise ValueError - except ValueError: + except ValueError as exc: + assert sys.exc_info()[1] is exc, sys.exc_info() yield + assert sys.exc_info()[1] is exc, sys.exc_info() + def yield_in_except_throw_instance(): """ - >>> import sys >>> g = yield_in_except_throw_instance() >>> next(g) >>> g.throw(TypeError()) @@ -194,8 +211,11 @@ def yield_in_except_throw_instance(): """ try: raise ValueError - except ValueError: + except ValueError as exc: + assert sys.exc_info()[1] is exc, sys.exc_info() yield + assert sys.exc_info()[1] is exc, sys.exc_info() + def test_swap_assignment(): """ |