From 5486e94c1ad7342d7a892631cc8e7ff33a022ae8 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 12 Jun 2016 00:39:41 +0300 Subject: Issue #27140: Added BUILD_CONST_KEY_MAP opcode. --- Python/compile.c | 185 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 156 insertions(+), 29 deletions(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index ffde903d65..2d59d38037 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -980,6 +980,8 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return 1 - (oparg & 0xFF); case BUILD_MAP: return 1 - 2*oparg; + case BUILD_CONST_KEY_MAP: + return -oparg; case LOAD_ATTR: return 0; case COMPARE_OP: @@ -1234,6 +1236,15 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute) return 0; \ } +/* Same as ADDOP_O, but steals a reference. */ +#define ADDOP_N(C, OP, O, TYPE) { \ + if (!compiler_addop_o((C), (OP), (C)->u->u_ ## TYPE, (O))) { \ + Py_DECREF((O)); \ + return 0; \ + } \ + Py_DECREF((O)); \ +} + #define ADDOP_NAME(C, OP, O, TYPE) { \ if (!compiler_addop_name((C), (OP), (C)->u->u_ ## TYPE, (O))) \ return 0; \ @@ -1309,6 +1320,44 @@ compiler_isdocstring(stmt_ty s) return 0; } +static int +is_const(expr_ty e) +{ + switch (e->kind) { + case Constant_kind: + case Num_kind: + case Str_kind: + case Bytes_kind: + case Ellipsis_kind: + case NameConstant_kind: + return 1; + default: + return 0; + } +} + +static PyObject * +get_const_value(expr_ty e) +{ + switch (e->kind) { + case Constant_kind: + return e->v.Constant.value; + case Num_kind: + return e->v.Num.n; + case Str_kind: + return e->v.Str.s; + case Bytes_kind: + return e->v.Bytes.s; + case Ellipsis_kind: + return Py_Ellipsis; + case NameConstant_kind: + return e->v.NameConstant.value; + default: + assert(!is_const(e)); + return NULL; + } +} + /* Compile a sequence of statements, checking for a docstring. */ static int @@ -2604,19 +2653,9 @@ compiler_visit_stmt_expr(struct compiler *c, expr_ty value) return 1; } - switch (value->kind) - { - case Str_kind: - case Num_kind: - case Ellipsis_kind: - case Bytes_kind: - case NameConstant_kind: - case Constant_kind: + if (is_const(value)) { /* ignore constant statement */ return 1; - - default: - break; } VISIT(c, expr, value); @@ -3095,6 +3134,49 @@ compiler_set(struct compiler *c, expr_ty e) BUILD_SET, BUILD_SET_UNPACK); } +static int +are_all_items_const(asdl_seq *seq, Py_ssize_t begin, Py_ssize_t end) +{ + Py_ssize_t i; + for (i = begin; i < end; i++) { + expr_ty key = (expr_ty)asdl_seq_GET(seq, i); + if (key == NULL || !is_const(key)) + return 0; + } + return 1; +} + +static int +compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end) +{ + Py_ssize_t i, n = end - begin; + PyObject *keys, *key; + if (n > 1 && are_all_items_const(e->v.Dict.keys, begin, end)) { + for (i = begin; i < end; i++) { + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); + } + keys = PyTuple_New(n); + if (keys == NULL) { + return 0; + } + for (i = begin; i < end; i++) { + key = get_const_value((expr_ty)asdl_seq_GET(e->v.Dict.keys, i)); + Py_INCREF(key); + PyTuple_SET_ITEM(keys, i - begin, key); + } + ADDOP_N(c, LOAD_CONST, keys, consts); + ADDOP_I(c, BUILD_CONST_KEY_MAP, n); + } + else { + for (i = begin; i < end; i++) { + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i)); + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); + } + ADDOP_I(c, BUILD_MAP, n); + } + return 1; +} + static int compiler_dict(struct compiler *c, expr_ty e) { @@ -3107,7 +3189,8 @@ compiler_dict(struct compiler *c, expr_ty e) for (i = 0; i < n; i++) { is_unpacking = (expr_ty)asdl_seq_GET(e->v.Dict.keys, i) == NULL; if (elements == 0xFFFF || (elements && is_unpacking)) { - ADDOP_I(c, BUILD_MAP, elements); + if (!compiler_subdict(c, e, i - elements, i)) + return 0; containers++; elements = 0; } @@ -3116,13 +3199,12 @@ compiler_dict(struct compiler *c, expr_ty e) containers++; } else { - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i)); - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); elements++; } } if (elements || containers == 0) { - ADDOP_I(c, BUILD_MAP, elements); + if (!compiler_subdict(c, e, n - elements, n)) + return 0; containers++; } /* If there is more than one dict, they need to be merged into a new @@ -3266,6 +3348,42 @@ compiler_formatted_value(struct compiler *c, expr_ty e) return 1; } +static int +compiler_subkwargs(struct compiler *c, asdl_seq *keywords, Py_ssize_t begin, Py_ssize_t end) +{ + Py_ssize_t i, n = end - begin; + keyword_ty kw; + PyObject *keys, *key; + assert(n > 0); + if (n > 1) { + for (i = begin; i < end; i++) { + kw = asdl_seq_GET(keywords, i); + VISIT(c, expr, kw->value); + } + keys = PyTuple_New(n); + if (keys == NULL) { + return 0; + } + for (i = begin; i < end; i++) { + key = ((keyword_ty) asdl_seq_GET(keywords, i))->arg; + Py_INCREF(key); + PyTuple_SET_ITEM(keys, i - begin, key); + } + ADDOP_N(c, LOAD_CONST, keys, consts); + ADDOP_I(c, BUILD_CONST_KEY_MAP, n); + } + else { + /* a for loop only executes once */ + for (i = begin; i < end; i++) { + kw = asdl_seq_GET(keywords, i); + ADDOP_O(c, LOAD_CONST, kw->arg, consts); + VISIT(c, expr, kw->value); + } + ADDOP_I(c, BUILD_MAP, n); + } + return 1; +} + /* shared code between compiler_call and compiler_class */ static int compiler_call_helper(struct compiler *c, @@ -3332,29 +3450,38 @@ compiler_call_helper(struct compiler *c, if (kw->arg == NULL) { /* A keyword argument unpacking. */ if (nseen) { - ADDOP_I(c, BUILD_MAP, nseen); + if (nsubkwargs) { + if (!compiler_subkwargs(c, keywords, i - nseen, i)) + return 0; + nsubkwargs++; + } + else { + Py_ssize_t j; + for (j = 0; j < nseen; j++) { + VISIT(c, keyword, asdl_seq_GET(keywords, j)); + } + nkw = nseen; + } nseen = 0; - nsubkwargs++; } VISIT(c, expr, kw->value); nsubkwargs++; } - else if (nsubkwargs) { - /* A keyword argument and we already have a dict. */ - ADDOP_O(c, LOAD_CONST, kw->arg, consts); - VISIT(c, expr, kw->value); - nseen++; - } else { - /* keyword argument */ - VISIT(c, keyword, kw) - nkw++; + nseen++; } } if (nseen) { - /* Pack up any trailing keyword arguments. */ - ADDOP_I(c, BUILD_MAP, nseen); - nsubkwargs++; + if (nsubkwargs) { + /* Pack up any trailing keyword arguments. */ + if (!compiler_subkwargs(c, keywords, nelts - nseen, nelts)) + return 0; + nsubkwargs++; + } + else { + VISIT_SEQ(c, keyword, keywords); + nkw = nseen; + } } if (nsubkwargs) { code |= 2; -- cgit v1.2.1 From fa60de800763bc0f7b70d67ca749fce15d2d53de Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 12 Jun 2016 17:36:24 +0300 Subject: Issue #27095: Simplified MAKE_FUNCTION and removed MAKE_CLOSURE opcodes. Patch by Demur Rumed. --- Python/compile.c | 247 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 136 insertions(+), 111 deletions(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index 485fdd7a28..efb6c7ee3e 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1030,11 +1030,10 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return -NARGS(oparg)-1; case CALL_FUNCTION_VAR_KW: return -NARGS(oparg)-2; - case MAKE_FUNCTION: - return -1 -NARGS(oparg) - ((oparg >> 16) & 0xffff); - case MAKE_CLOSURE: - return -2 - NARGS(oparg) - ((oparg >> 16) & 0xffff); #undef NARGS + case MAKE_FUNCTION: + return -1 - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0) - + ((oparg & 0x04) != 0) - ((oparg & 0x08) != 0); case BUILD_SLICE: if (oparg == 3) return -2; @@ -1472,53 +1471,50 @@ compiler_lookup_arg(PyObject *dict, PyObject *name) } static int -compiler_make_closure(struct compiler *c, PyCodeObject *co, Py_ssize_t args, PyObject *qualname) +compiler_make_closure(struct compiler *c, PyCodeObject *co, Py_ssize_t flags, PyObject *qualname) { Py_ssize_t i, free = PyCode_GetNumFree(co); if (qualname == NULL) qualname = co->co_name; - if (free == 0) { - ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts); - ADDOP_O(c, LOAD_CONST, qualname, consts); - ADDOP_I(c, MAKE_FUNCTION, args); - return 1; - } - for (i = 0; i < free; ++i) { - /* Bypass com_addop_varname because it will generate - LOAD_DEREF but LOAD_CLOSURE is needed. - */ - PyObject *name = PyTuple_GET_ITEM(co->co_freevars, i); - int arg, reftype; - - /* Special case: If a class contains a method with a - free variable that has the same name as a method, - the name will be considered free *and* local in the - class. It should be handled by the closure, as - well as by the normal name loookup logic. - */ - reftype = get_ref_type(c, name); - if (reftype == CELL) - arg = compiler_lookup_arg(c->u->u_cellvars, name); - else /* (reftype == FREE) */ - arg = compiler_lookup_arg(c->u->u_freevars, name); - if (arg == -1) { - fprintf(stderr, - "lookup %s in %s %d %d\n" - "freevars of %s: %s\n", - PyUnicode_AsUTF8(PyObject_Repr(name)), - PyUnicode_AsUTF8(c->u->u_name), - reftype, arg, - PyUnicode_AsUTF8(co->co_name), - PyUnicode_AsUTF8(PyObject_Repr(co->co_freevars))); - Py_FatalError("compiler_make_closure()"); + if (free) { + for (i = 0; i < free; ++i) { + /* Bypass com_addop_varname because it will generate + LOAD_DEREF but LOAD_CLOSURE is needed. + */ + PyObject *name = PyTuple_GET_ITEM(co->co_freevars, i); + int arg, reftype; + + /* Special case: If a class contains a method with a + free variable that has the same name as a method, + the name will be considered free *and* local in the + class. It should be handled by the closure, as + well as by the normal name loookup logic. + */ + reftype = get_ref_type(c, name); + if (reftype == CELL) + arg = compiler_lookup_arg(c->u->u_cellvars, name); + else /* (reftype == FREE) */ + arg = compiler_lookup_arg(c->u->u_freevars, name); + if (arg == -1) { + fprintf(stderr, + "lookup %s in %s %d %d\n" + "freevars of %s: %s\n", + PyUnicode_AsUTF8(PyObject_Repr(name)), + PyUnicode_AsUTF8(c->u->u_name), + reftype, arg, + PyUnicode_AsUTF8(co->co_name), + PyUnicode_AsUTF8(PyObject_Repr(co->co_freevars))); + Py_FatalError("compiler_make_closure()"); + } + ADDOP_I(c, LOAD_CLOSURE, arg); } - ADDOP_I(c, LOAD_CLOSURE, arg); + flags |= 0x08; + ADDOP_I(c, BUILD_TUPLE, free); } - ADDOP_I(c, BUILD_TUPLE, free); ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts); ADDOP_O(c, LOAD_CONST, qualname, consts); - ADDOP_I(c, MAKE_CLOSURE, args); + ADDOP_I(c, MAKE_FUNCTION, flags); return 1; } @@ -1536,27 +1532,59 @@ compiler_decorators(struct compiler *c, asdl_seq* decos) return 1; } -static int +static Py_ssize_t compiler_visit_kwonlydefaults(struct compiler *c, asdl_seq *kwonlyargs, asdl_seq *kw_defaults) { - int i, default_count = 0; + int i; + PyObject *keys = NULL; + for (i = 0; i < asdl_seq_LEN(kwonlyargs); i++) { arg_ty arg = asdl_seq_GET(kwonlyargs, i); expr_ty default_ = asdl_seq_GET(kw_defaults, i); if (default_) { PyObject *mangled = _Py_Mangle(c->u->u_private, arg->arg); - if (!mangled) - return -1; - ADDOP_O(c, LOAD_CONST, mangled, consts); - Py_DECREF(mangled); + if (!mangled) { + goto error; + } + if (keys == NULL) { + keys = PyList_New(1); + if (keys == NULL) { + Py_DECREF(mangled); + return -1; + } + PyList_SET_ITEM(keys, 0, mangled); + } + else { + int res = PyList_Append(keys, mangled); + Py_DECREF(mangled); + if (res == -1) { + goto error; + } + } if (!compiler_visit_expr(c, default_)) { - return -1; + goto error; } - default_count++; } } - return default_count; + if (keys != NULL) { + Py_ssize_t default_count = PyList_GET_SIZE(keys); + PyObject *keys_tuple = PyList_AsTuple(keys); + Py_DECREF(keys); + if (keys_tuple == NULL) { + return -1; + } + ADDOP_N(c, LOAD_CONST, keys_tuple, consts); + ADDOP_I(c, BUILD_CONST_KEY_MAP, default_count); + return default_count; + } + else { + return 0; + } + +error: + Py_XDECREF(keys); + return -1; } static int @@ -1595,15 +1623,14 @@ compiler_visit_argannotations(struct compiler *c, asdl_seq* args, return 1; } -static int +static Py_ssize_t compiler_visit_annotations(struct compiler *c, arguments_ty args, expr_ty returns) { - /* Push arg annotations and a list of the argument names. Return the # - of items pushed. The expressions are evaluated out-of-order wrt the - source code. + /* Push arg annotation dict. Return # of items pushed. + The expressions are evaluated out-of-order wrt the source code. - More than 2^16-1 annotations is a SyntaxError. Returns -1 on error. + Returns -1 on error. */ static identifier return_str; PyObject *names; @@ -1635,38 +1662,47 @@ compiler_visit_annotations(struct compiler *c, arguments_ty args, } len = PyList_GET_SIZE(names); - if (len > 65534) { - /* len must fit in 16 bits, and len is incremented below */ - PyErr_SetString(PyExc_SyntaxError, - "too many annotations"); - goto error; - } if (len) { - /* convert names to a tuple and place on stack */ - PyObject *elt; - Py_ssize_t i; - PyObject *s = PyTuple_New(len); - if (!s) - goto error; - for (i = 0; i < len; i++) { - elt = PyList_GET_ITEM(names, i); - Py_INCREF(elt); - PyTuple_SET_ITEM(s, i, elt); + PyObject *keytuple = PyList_AsTuple(names); + Py_DECREF(names); + if (keytuple == NULL) { + return -1; } - ADDOP_O(c, LOAD_CONST, s, consts); - Py_DECREF(s); - len++; /* include the just-pushed tuple */ + ADDOP_N(c, LOAD_CONST, keytuple, consts); + ADDOP_I(c, BUILD_CONST_KEY_MAP, len); } - Py_DECREF(names); - - /* We just checked that len <= 65535, see above */ - return Py_SAFE_DOWNCAST(len, Py_ssize_t, int); + else { + Py_DECREF(names); + } + return len; error: Py_DECREF(names); return -1; } +static Py_ssize_t +compiler_default_arguments(struct compiler *c, arguments_ty args) +{ + Py_ssize_t funcflags = 0; + if (args->defaults && asdl_seq_LEN(args->defaults) > 0) { + VISIT_SEQ(c, expr, args->defaults); + ADDOP_I(c, BUILD_TUPLE, asdl_seq_LEN(args->defaults)); + funcflags |= 0x01; + } + if (args->kwonlyargs) { + Py_ssize_t res = compiler_visit_kwonlydefaults(c, args->kwonlyargs, + args->kw_defaults); + if (res < 0) { + return -1; + } + else if (res > 0) { + funcflags |= 0x02; + } + } + return funcflags; +} + static int compiler_function(struct compiler *c, stmt_ty s, int is_async) { @@ -1678,12 +1714,11 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) asdl_seq* decos; asdl_seq *body; stmt_ty st; - Py_ssize_t i, n, arglength; - int docstring, kw_default_count = 0; + Py_ssize_t i, n, funcflags; + int docstring; int num_annotations; int scope_type; - if (is_async) { assert(s->kind == AsyncFunctionDef_kind); @@ -1708,24 +1743,23 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (!compiler_decorators(c, decos)) return 0; - if (args->defaults) - VISIT_SEQ(c, expr, args->defaults); - if (args->kwonlyargs) { - int res = compiler_visit_kwonlydefaults(c, args->kwonlyargs, - args->kw_defaults); - if (res < 0) - return 0; - kw_default_count = res; + + funcflags = compiler_default_arguments(c, args); + if (funcflags == -1) { + return 0; } + num_annotations = compiler_visit_annotations(c, args, returns); - if (num_annotations < 0) + if (num_annotations < 0) { return 0; - assert((num_annotations & 0xFFFF) == num_annotations); + } + else if (num_annotations > 0) { + funcflags |= 0x04; + } - if (!compiler_enter_scope(c, name, - scope_type, (void *)s, - s->lineno)) + if (!compiler_enter_scope(c, name, scope_type, (void *)s, s->lineno)) { return 0; + } st = (stmt_ty)asdl_seq_GET(body, 0); docstring = compiler_isdocstring(st); @@ -1758,12 +1792,9 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) return 0; } - arglength = asdl_seq_LEN(args->defaults); - arglength |= kw_default_count << 8; - arglength |= num_annotations << 16; if (is_async) co->co_flags |= CO_COROUTINE; - compiler_make_closure(c, co, arglength, qualname); + compiler_make_closure(c, co, funcflags, qualname); Py_DECREF(qualname); Py_DECREF(co); @@ -1923,8 +1954,7 @@ compiler_lambda(struct compiler *c, expr_ty e) PyCodeObject *co; PyObject *qualname; static identifier name; - int kw_default_count = 0; - Py_ssize_t arglength; + Py_ssize_t funcflags; arguments_ty args = e->v.Lambda.args; assert(e->kind == Lambda_kind); @@ -1934,14 +1964,11 @@ compiler_lambda(struct compiler *c, expr_ty e) return 0; } - if (args->defaults) - VISIT_SEQ(c, expr, args->defaults); - if (args->kwonlyargs) { - int res = compiler_visit_kwonlydefaults(c, args->kwonlyargs, - args->kw_defaults); - if (res < 0) return 0; - kw_default_count = res; + funcflags = compiler_default_arguments(c, args); + if (funcflags == -1) { + return 0; } + if (!compiler_enter_scope(c, name, COMPILER_SCOPE_LAMBDA, (void *)e, e->lineno)) return 0; @@ -1967,9 +1994,7 @@ compiler_lambda(struct compiler *c, expr_ty e) if (co == NULL) return 0; - arglength = asdl_seq_LEN(args->defaults); - arglength |= kw_default_count << 8; - compiler_make_closure(c, co, arglength, qualname); + compiler_make_closure(c, co, funcflags, qualname); Py_DECREF(qualname); Py_DECREF(co); -- cgit v1.2.1 From 72559773cff88f5f727bcb742b3dfc769b033387 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Wed, 17 Aug 2016 17:18:33 -0400 Subject: Issue #27594: Prevent assertion error when running test_ast with coverage enabled: ensure code object has a valid first line number. Patch suggested by Ivan Levkivskyi. --- Python/compile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index e46676c0b0..fb80f51fc0 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4963,7 +4963,7 @@ assemble(struct compiler *c, int addNone) /* Set firstlineno if it wasn't explicitly set. */ if (!c->u->u_firstlineno) { - if (entryblock && entryblock->b_instr) + if (entryblock && entryblock->b_instr && entryblock->b_instr->i_lineno) c->u->u_firstlineno = entryblock->b_instr->i_lineno; else c->u->u_firstlineno = 1; -- cgit v1.2.1 From 0c3c0375111bde5f7586e6c951f08a0035039a39 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 6 Sep 2016 22:07:53 +0300 Subject: Issue #27078: Added BUILD_STRING opcode. Optimized f-strings evaluation. --- Python/compile.c | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index fb80f51fc0..b5629895c1 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -970,6 +970,7 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) case BUILD_TUPLE: case BUILD_LIST: case BUILD_SET: + case BUILD_STRING: return 1-oparg; case BUILD_LIST_UNPACK: case BUILD_TUPLE_UNPACK: @@ -3315,31 +3316,8 @@ compiler_call(struct compiler *c, expr_ty e) static int compiler_joined_str(struct compiler *c, expr_ty e) { - /* Concatenate parts of a string using ''.join(parts). There are - probably better ways of doing this. - - This is used for constructs like "'x=' f'{42}'", which have to - be evaluated at compile time. */ - - static PyObject *empty_string; - static PyObject *join_string; - - if (!empty_string) { - empty_string = PyUnicode_FromString(""); - if (!empty_string) - return 0; - } - if (!join_string) { - join_string = PyUnicode_FromString("join"); - if (!join_string) - return 0; - } - - ADDOP_O(c, LOAD_CONST, empty_string, consts); - ADDOP_NAME(c, LOAD_ATTR, join_string, names); VISIT_SEQ(c, expr, e->v.JoinedStr.values); - ADDOP_I(c, BUILD_LIST, asdl_seq_LEN(e->v.JoinedStr.values)); - ADDOP_I(c, CALL_FUNCTION, 1); + ADDOP_I(c, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values)); return 1; } -- cgit v1.2.1 From 82b75eb31160ec9c025e450ac9fa3300d560a245 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 6 Sep 2016 13:47:26 -0700 Subject: replace Py_(u)intptr_t with the c99 standard types --- Python/compile.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index b5629895c1..6fe5d5fa82 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -476,9 +476,9 @@ compiler_unit_check(struct compiler_unit *u) { basicblock *block; for (block = u->u_blocks; block != NULL; block = block->b_list) { - assert((Py_uintptr_t)block != 0xcbcbcbcbU); - assert((Py_uintptr_t)block != 0xfbfbfbfbU); - assert((Py_uintptr_t)block != 0xdbdbdbdbU); + assert((uintptr_t)block != 0xcbcbcbcbU); + assert((uintptr_t)block != 0xfbfbfbfbU); + assert((uintptr_t)block != 0xdbdbdbdbU); if (block->b_instr != NULL) { assert(block->b_ialloc > 0); assert(block->b_iused > 0); -- cgit v1.2.1 From 58a260158ae807ba57a7f83d13543907c13bc415 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 7 Sep 2016 09:26:18 -0700 Subject: replace PY_SIZE_MAX with SIZE_MAX --- Python/compile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index 6fe5d5fa82..45e4262b40 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -804,7 +804,7 @@ compiler_next_instr(struct compiler *c, basicblock *b) oldsize = b->b_ialloc * sizeof(struct instr); newsize = oldsize << 1; - if (oldsize > (PY_SIZE_MAX >> 1)) { + if (oldsize > (SIZE_MAX >> 1)) { PyErr_NoMemory(); return -1; } @@ -4520,7 +4520,7 @@ assemble_init(struct assembler *a, int nblocks, int firstlineno) a->a_lnotab = PyBytes_FromStringAndSize(NULL, DEFAULT_LNOTAB_SIZE); if (!a->a_lnotab) return 0; - if ((size_t)nblocks > PY_SIZE_MAX / sizeof(basicblock *)) { + if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) { PyErr_NoMemory(); return 0; } -- cgit v1.2.1 From b164476aaf77ceffac36ddbbdc7f4df90fbfebd3 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 8 Sep 2016 20:50:03 -0700 Subject: Issue #27985: Implement PEP 526 -- Syntax for Variable Annotations. Patch by Ivan Levkivskyi. --- Python/compile.c | 215 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 214 insertions(+), 1 deletion(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index 45e4262b40..b46edd4985 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -179,6 +179,7 @@ static int compiler_visit_stmt(struct compiler *, stmt_ty); static int compiler_visit_keyword(struct compiler *, keyword_ty); static int compiler_visit_expr(struct compiler *, expr_ty); static int compiler_augassign(struct compiler *, stmt_ty); +static int compiler_annassign(struct compiler *, stmt_ty); static int compiler_visit_slice(struct compiler *, slice_ty, expr_context_ty); @@ -933,6 +934,8 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return -1; case IMPORT_STAR: return -1; + case SETUP_ANNOTATIONS: + return 0; case YIELD_VALUE: return 0; case YIELD_FROM: @@ -1020,6 +1023,8 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return -1; case DELETE_FAST: return 0; + case STORE_ANNOTATION: + return -1; case RAISE_VARARGS: return -oparg; @@ -1358,7 +1363,65 @@ get_const_value(expr_ty e) } } -/* Compile a sequence of statements, checking for a docstring. */ +/* Search if variable annotations are present statically in a block. */ + +static int +find_ann(asdl_seq *stmts) +{ + int i, j, res = 0; + stmt_ty st; + + for (i = 0; i < asdl_seq_LEN(stmts); i++) { + st = (stmt_ty)asdl_seq_GET(stmts, i); + switch (st->kind) { + case AnnAssign_kind: + return 1; + case For_kind: + res = find_ann(st->v.For.body) || + find_ann(st->v.For.orelse); + break; + case AsyncFor_kind: + res = find_ann(st->v.AsyncFor.body) || + find_ann(st->v.AsyncFor.orelse); + break; + case While_kind: + res = find_ann(st->v.While.body) || + find_ann(st->v.While.orelse); + break; + case If_kind: + res = find_ann(st->v.If.body) || + find_ann(st->v.If.orelse); + break; + case With_kind: + res = find_ann(st->v.With.body); + break; + case AsyncWith_kind: + res = find_ann(st->v.AsyncWith.body); + break; + case Try_kind: + for (j = 0; j < asdl_seq_LEN(st->v.Try.handlers); j++) { + excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( + st->v.Try.handlers, j); + if (find_ann(handler->v.ExceptHandler.body)) { + return 1; + } + } + res = find_ann(st->v.Try.body) || + find_ann(st->v.Try.finalbody) || + find_ann(st->v.Try.orelse); + break; + default: + res = 0; + } + if (res) { + break; + } + } + return res; +} + +/* Compile a sequence of statements, checking for a docstring + and for annotations. */ static int compiler_body(struct compiler *c, asdl_seq *stmts) @@ -1366,6 +1429,19 @@ compiler_body(struct compiler *c, asdl_seq *stmts) int i = 0; stmt_ty st; + /* Set current line number to the line number of first statement. + This way line number for SETUP_ANNOTATIONS will always + coincide with the line number of first "real" statement in module. + If body is empy, then lineno will be set later in assemble. */ + if (c->u->u_scope_type == COMPILER_SCOPE_MODULE && + !c->u->u_lineno && asdl_seq_LEN(stmts)) { + st = (stmt_ty)asdl_seq_GET(stmts, 0); + c->u->u_lineno = st->lineno; + } + /* Every annotated class and module should have __annotations__. */ + if (find_ann(stmts)) { + ADDOP(c, SETUP_ANNOTATIONS); + } if (!asdl_seq_LEN(stmts)) return 1; st = (stmt_ty)asdl_seq_GET(stmts, 0); @@ -1403,6 +1479,9 @@ compiler_mod(struct compiler *c, mod_ty mod) } break; case Interactive_kind: + if (find_ann(mod->v.Interactive.body)) { + ADDOP(c, SETUP_ANNOTATIONS); + } c->c_interactive = 1; VISIT_SEQ_IN_SCOPE(c, stmt, mod->v.Interactive.body); @@ -2743,6 +2822,8 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) break; case AugAssign_kind: return compiler_augassign(c, s); + case AnnAssign_kind: + return compiler_annassign(c, s); case For_kind: return compiler_for(c, s); case While_kind: @@ -4222,6 +4303,138 @@ compiler_augassign(struct compiler *c, stmt_ty s) return 1; } +static int +check_ann_expr(struct compiler *c, expr_ty e) +{ + VISIT(c, expr, e); + ADDOP(c, POP_TOP); + return 1; +} + +static int +check_annotation(struct compiler *c, stmt_ty s) +{ + /* Annotations are only evaluated in a module or class. */ + if (c->u->u_scope_type == COMPILER_SCOPE_MODULE || + c->u->u_scope_type == COMPILER_SCOPE_CLASS) { + return check_ann_expr(c, s->v.AnnAssign.annotation); + } + return 1; +} + +static int +check_ann_slice(struct compiler *c, slice_ty sl) +{ + switch(sl->kind) { + case Index_kind: + return check_ann_expr(c, sl->v.Index.value); + case Slice_kind: + if (sl->v.Slice.lower && !check_ann_expr(c, sl->v.Slice.lower)) { + return 0; + } + if (sl->v.Slice.upper && !check_ann_expr(c, sl->v.Slice.upper)) { + return 0; + } + if (sl->v.Slice.step && !check_ann_expr(c, sl->v.Slice.step)) { + return 0; + } + break; + default: + PyErr_SetString(PyExc_SystemError, + "unexpected slice kind"); + return 0; + } + return 1; +} + +static int +check_ann_subscr(struct compiler *c, slice_ty sl) +{ + /* We check that everything in a subscript is defined at runtime. */ + Py_ssize_t i, n; + + switch (sl->kind) { + case Index_kind: + case Slice_kind: + if (!check_ann_slice(c, sl)) { + return 0; + } + break; + case ExtSlice_kind: + n = asdl_seq_LEN(sl->v.ExtSlice.dims); + for (i = 0; i < n; i++) { + slice_ty subsl = (slice_ty)asdl_seq_GET(sl->v.ExtSlice.dims, i); + switch (subsl->kind) { + case Index_kind: + case Slice_kind: + if (!check_ann_slice(c, subsl)) { + return 0; + } + break; + case ExtSlice_kind: + default: + PyErr_SetString(PyExc_SystemError, + "extended slice invalid in nested slice"); + return 0; + } + } + break; + default: + PyErr_Format(PyExc_SystemError, + "invalid subscript kind %d", sl->kind); + return 0; + } + return 1; +} + +static int +compiler_annassign(struct compiler *c, stmt_ty s) +{ + expr_ty targ = s->v.AnnAssign.target; + + assert(s->kind == AnnAssign_kind); + + /* We perform the actual assignment first. */ + if (s->v.AnnAssign.value) { + VISIT(c, expr, s->v.AnnAssign.value); + VISIT(c, expr, targ); + } + switch (targ->kind) { + case Name_kind: + /* If we have a simple name in a module or class, store annotation. */ + if (s->v.AnnAssign.simple && + (c->u->u_scope_type == COMPILER_SCOPE_MODULE || + c->u->u_scope_type == COMPILER_SCOPE_CLASS)) { + VISIT(c, expr, s->v.AnnAssign.annotation); + ADDOP_O(c, STORE_ANNOTATION, targ->v.Name.id, names) + } + break; + case Attribute_kind: + if (!s->v.AnnAssign.value && + !check_ann_expr(c, targ->v.Attribute.value)) { + return 0; + } + break; + case Subscript_kind: + if (!s->v.AnnAssign.value && + (!check_ann_expr(c, targ->v.Subscript.value) || + !check_ann_subscr(c, targ->v.Subscript.slice))) { + return 0; + } + break; + default: + PyErr_Format(PyExc_SystemError, + "invalid node type (%d) for annotated assignment", + targ->kind); + return 0; + } + /* Annotation is evaluated last. */ + if (!s->v.AnnAssign.simple && !check_annotation(c, s)) { + return 0; + } + return 1; +} + static int compiler_push_fblock(struct compiler *c, enum fblocktype t, basicblock *b) { -- cgit v1.2.1 From 17668cfa5e0e6f50376cc233d9b063b908a19845 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 8 Sep 2016 22:01:51 -0700 Subject: Issue #28003: Implement PEP 525 -- Asynchronous Generators. --- Python/compile.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index b46edd4985..faae4f5828 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1886,8 +1886,6 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) return 0; } - if (is_async) - co->co_flags |= CO_COROUTINE; compiler_make_closure(c, co, funcflags, qualname); Py_DECREF(qualname); Py_DECREF(co); @@ -2801,6 +2799,9 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) if (c->u->u_ste->ste_type != FunctionBlock) return compiler_error(c, "'return' outside function"); if (s->v.Return.value) { + if (c->u->u_ste->ste_coroutine && c->u->u_ste->ste_generator) + return compiler_error( + c, "'return' with value in async generator"); VISIT(c, expr, s->v.Return.value); } else @@ -4115,8 +4116,6 @@ compiler_visit_expr(struct compiler *c, expr_ty e) case Yield_kind: if (c->u->u_ste->ste_type != FunctionBlock) return compiler_error(c, "'yield' outside function"); - if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION) - return compiler_error(c, "'yield' inside async function"); if (e->v.Yield.value) { VISIT(c, expr, e->v.Yield.value); } @@ -4992,8 +4991,12 @@ compute_code_flags(struct compiler *c) flags |= CO_NEWLOCALS | CO_OPTIMIZED; if (ste->ste_nested) flags |= CO_NESTED; - if (ste->ste_generator) + if (ste->ste_generator && !ste->ste_coroutine) flags |= CO_GENERATOR; + if (!ste->ste_generator && ste->ste_coroutine) + flags |= CO_COROUTINE; + if (ste->ste_generator && ste->ste_coroutine) + flags |= CO_ASYNC_GENERATOR; if (ste->ste_varargs) flags |= CO_VARARGS; if (ste->ste_varkeywords) -- cgit v1.2.1 From b26bd4a4bf3de9dbedbbbd1209fea77cd7af7689 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 9 Sep 2016 10:36:01 -0700 Subject: Issue #28008: Implement PEP 530 -- asynchronous comprehensions. --- Python/compile.c | 218 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 202 insertions(+), 16 deletions(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index faae4f5828..20fe4bc5b5 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -202,6 +202,16 @@ static int compiler_call_helper(struct compiler *c, int n, static int compiler_try_except(struct compiler *, stmt_ty); static int compiler_set_qualname(struct compiler *); +static int compiler_sync_comprehension_generator( + struct compiler *c, + asdl_seq *generators, int gen_index, + expr_ty elt, expr_ty val, int type); + +static int compiler_async_comprehension_generator( + struct compiler *c, + asdl_seq *generators, int gen_index, + expr_ty elt, expr_ty val, int type); + static PyCodeObject *assemble(struct compiler *, int addNone); static PyObject *__doc__; @@ -2165,14 +2175,14 @@ compiler_for(struct compiler *c, stmt_ty s) static int compiler_async_for(struct compiler *c, stmt_ty s) { - static PyObject *stopiter_error = NULL; + _Py_IDENTIFIER(StopAsyncIteration); + basicblock *try, *except, *end, *after_try, *try_cleanup, *after_loop, *after_loop_else; - if (stopiter_error == NULL) { - stopiter_error = PyUnicode_InternFromString("StopAsyncIteration"); - if (stopiter_error == NULL) - return 0; + PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration); + if (stop_aiter_error == NULL) { + return 0; } try = compiler_new_block(c); @@ -2214,7 +2224,7 @@ compiler_async_for(struct compiler *c, stmt_ty s) compiler_use_next_block(c, except); ADDOP(c, DUP_TOP); - ADDOP_O(c, LOAD_GLOBAL, stopiter_error, names); + ADDOP_O(c, LOAD_GLOBAL, stop_aiter_error, names); ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH); ADDOP_JABS(c, POP_JUMP_IF_FALSE, try_cleanup); @@ -3627,10 +3637,27 @@ compiler_call_helper(struct compiler *c, - iterate over the generator sequence instead of using recursion */ + static int compiler_comprehension_generator(struct compiler *c, asdl_seq *generators, int gen_index, expr_ty elt, expr_ty val, int type) +{ + comprehension_ty gen; + gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); + if (gen->is_async) { + return compiler_async_comprehension_generator( + c, generators, gen_index, elt, val, type); + } else { + return compiler_sync_comprehension_generator( + c, generators, gen_index, elt, val, type); + } +} + +static int +compiler_sync_comprehension_generator(struct compiler *c, + asdl_seq *generators, int gen_index, + expr_ty elt, expr_ty val, int type) { /* generate code for the iterator, then each of the ifs, and then write to the element */ @@ -3717,21 +3744,168 @@ compiler_comprehension_generator(struct compiler *c, return 1; } +static int +compiler_async_comprehension_generator(struct compiler *c, + asdl_seq *generators, int gen_index, + expr_ty elt, expr_ty val, int type) +{ + _Py_IDENTIFIER(StopAsyncIteration); + + comprehension_ty gen; + basicblock *anchor, *skip, *if_cleanup, *try, + *after_try, *except, *try_cleanup; + Py_ssize_t i, n; + + PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration); + if (stop_aiter_error == NULL) { + return 0; + } + + try = compiler_new_block(c); + after_try = compiler_new_block(c); + try_cleanup = compiler_new_block(c); + except = compiler_new_block(c); + skip = compiler_new_block(c); + if_cleanup = compiler_new_block(c); + anchor = compiler_new_block(c); + + if (skip == NULL || if_cleanup == NULL || anchor == NULL || + try == NULL || after_try == NULL || + except == NULL || after_try == NULL) { + return 0; + } + + gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); + + if (gen_index == 0) { + /* Receive outermost iter as an implicit argument */ + c->u->u_argcount = 1; + ADDOP_I(c, LOAD_FAST, 0); + } + else { + /* Sub-iter - calculate on the fly */ + VISIT(c, expr, gen->iter); + ADDOP(c, GET_AITER); + ADDOP_O(c, LOAD_CONST, Py_None, consts); + ADDOP(c, YIELD_FROM); + } + + compiler_use_next_block(c, try); + + + ADDOP_JREL(c, SETUP_EXCEPT, except); + if (!compiler_push_fblock(c, EXCEPT, try)) + return 0; + + ADDOP(c, GET_ANEXT); + ADDOP_O(c, LOAD_CONST, Py_None, consts); + ADDOP(c, YIELD_FROM); + VISIT(c, expr, gen->target); + ADDOP(c, POP_BLOCK); + compiler_pop_fblock(c, EXCEPT, try); + ADDOP_JREL(c, JUMP_FORWARD, after_try); + + + compiler_use_next_block(c, except); + ADDOP(c, DUP_TOP); + ADDOP_O(c, LOAD_GLOBAL, stop_aiter_error, names); + ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH); + ADDOP_JABS(c, POP_JUMP_IF_FALSE, try_cleanup); + + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */ + ADDOP_JABS(c, JUMP_ABSOLUTE, anchor); + + + compiler_use_next_block(c, try_cleanup); + ADDOP(c, END_FINALLY); + + compiler_use_next_block(c, after_try); + + n = asdl_seq_LEN(gen->ifs); + for (i = 0; i < n; i++) { + expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); + VISIT(c, expr, e); + ADDOP_JABS(c, POP_JUMP_IF_FALSE, if_cleanup); + NEXT_BLOCK(c); + } + + if (++gen_index < asdl_seq_LEN(generators)) + if (!compiler_comprehension_generator(c, + generators, gen_index, + elt, val, type)) + return 0; + + /* only append after the last for generator */ + if (gen_index >= asdl_seq_LEN(generators)) { + /* comprehension specific code */ + switch (type) { + case COMP_GENEXP: + VISIT(c, expr, elt); + ADDOP(c, YIELD_VALUE); + ADDOP(c, POP_TOP); + break; + case COMP_LISTCOMP: + VISIT(c, expr, elt); + ADDOP_I(c, LIST_APPEND, gen_index + 1); + break; + case COMP_SETCOMP: + VISIT(c, expr, elt); + ADDOP_I(c, SET_ADD, gen_index + 1); + break; + case COMP_DICTCOMP: + /* With 'd[k] = v', v is evaluated before k, so we do + the same. */ + VISIT(c, expr, val); + VISIT(c, expr, elt); + ADDOP_I(c, MAP_ADD, gen_index + 1); + break; + default: + return 0; + } + + compiler_use_next_block(c, skip); + } + compiler_use_next_block(c, if_cleanup); + ADDOP_JABS(c, JUMP_ABSOLUTE, try); + compiler_use_next_block(c, anchor); + ADDOP(c, POP_TOP); + + return 1; +} + static int compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name, asdl_seq *generators, expr_ty elt, expr_ty val) { PyCodeObject *co = NULL; - expr_ty outermost_iter; + comprehension_ty outermost; PyObject *qualname = NULL; + int is_async_function = c->u->u_ste->ste_coroutine; + int is_async_generator = 0; - outermost_iter = ((comprehension_ty) - asdl_seq_GET(generators, 0))->iter; + outermost = (comprehension_ty) asdl_seq_GET(generators, 0); if (!compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION, (void *)e, e->lineno)) + { goto error; + } + + is_async_generator = c->u->u_ste->ste_coroutine; + + if (is_async_generator && !is_async_function) { + if (e->lineno > c->u->u_lineno) { + c->u->u_lineno = e->lineno; + c->u->u_lineno_set = 0; + } + compiler_error(c, "asynchronous comprehension outside of " + "an asynchronous function"); + goto error_in_scope; + } if (type != COMP_GENEXP) { int op; @@ -3774,9 +3948,24 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, Py_DECREF(qualname); Py_DECREF(co); - VISIT(c, expr, outermost_iter); - ADDOP(c, GET_ITER); + VISIT(c, expr, outermost->iter); + + if (outermost->is_async) { + ADDOP(c, GET_AITER); + ADDOP_O(c, LOAD_CONST, Py_None, consts); + ADDOP(c, YIELD_FROM); + } else { + ADDOP(c, GET_ITER); + } + ADDOP_I(c, CALL_FUNCTION, 1); + + if (is_async_generator && type != COMP_GENEXP) { + ADDOP(c, GET_AWAITABLE); + ADDOP_O(c, LOAD_CONST, Py_None, consts); + ADDOP(c, YIELD_FROM); + } + return 1; error_in_scope: compiler_exit_scope(c); @@ -4140,11 +4329,8 @@ compiler_visit_expr(struct compiler *c, expr_ty e) if (c->u->u_ste->ste_type != FunctionBlock) return compiler_error(c, "'await' outside function"); - if (c->u->u_scope_type == COMPILER_SCOPE_COMPREHENSION) - return compiler_error( - c, "'await' expressions in comprehensions are not supported"); - - if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) + if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && + c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION) return compiler_error(c, "'await' outside async function"); VISIT(c, expr, e->v.Await.value); -- cgit v1.2.1 From 2b5865ab2c78d3bdaf8ad7047762b5d0d41d8654 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Sep 2016 10:17:08 -0700 Subject: Rework CALL_FUNCTION* opcodes Issue #27213: Rework CALL_FUNCTION* opcodes to produce shorter and more efficient bytecode: * CALL_FUNCTION now only accepts position arguments * CALL_FUNCTION_KW accepts position arguments and keyword arguments, but keys of keyword arguments are packed into a constant tuple. * CALL_FUNCTION_EX is the most generic, it expects a tuple and a dict for positional and keyword arguments. CALL_FUNCTION_VAR and CALL_FUNCTION_VAR_KW opcodes have been removed. 2 tests of test_traceback are currently broken: skip test, the issue #28050 was created to track the issue. Patch by Demur Rumed, design by Serhiy Storchaka, reviewed by Serhiy Storchaka and Victor Stinner. --- Python/compile.c | 159 ++++++++++++++++++++++++------------------------------- 1 file changed, 70 insertions(+), 89 deletions(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index 20fe4bc5b5..36683d30e1 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -991,7 +991,7 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) case BUILD_MAP_UNPACK: return 1 - oparg; case BUILD_MAP_UNPACK_WITH_CALL: - return 1 - (oparg & 0xFF); + return 1 - oparg; case BUILD_MAP: return 1 - 2*oparg; case BUILD_CONST_KEY_MAP: @@ -1038,15 +1038,12 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) case RAISE_VARARGS: return -oparg; -#define NARGS(o) (((o) % 256) + 2*(((o) / 256) % 256)) case CALL_FUNCTION: - return -NARGS(oparg); - case CALL_FUNCTION_VAR: + return -oparg; case CALL_FUNCTION_KW: - return -NARGS(oparg)-1; - case CALL_FUNCTION_VAR_KW: - return -NARGS(oparg)-2; -#undef NARGS + return -oparg-1; + case CALL_FUNCTION_EX: + return - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0); case MAKE_FUNCTION: return -1 - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0) - ((oparg & 0x04) != 0) - ((oparg & 0x08) != 0); @@ -3500,22 +3497,29 @@ compiler_call_helper(struct compiler *c, asdl_seq *args, asdl_seq *keywords) { - int code = 0; - Py_ssize_t nelts, i, nseen; - int nkw; + Py_ssize_t i, nseen, nelts, nkwelts; + int musttupleunpack = 0, mustdictunpack = 0; /* the number of tuples and dictionaries on the stack */ Py_ssize_t nsubargs = 0, nsubkwargs = 0; - nkw = 0; - nseen = 0; /* the number of positional arguments on the stack */ nelts = asdl_seq_LEN(args); + nkwelts = asdl_seq_LEN(keywords); + + for (i = 0; i < nkwelts; i++) { + keyword_ty kw = asdl_seq_GET(keywords, i); + if (kw->arg == NULL) { + mustdictunpack = 1; + break; + } + } + + nseen = n; /* the number of positional arguments on the stack */ for (i = 0; i < nelts; i++) { expr_ty elt = asdl_seq_GET(args, i); if (elt->kind == Starred_kind) { /* A star-arg. If we've seen positional arguments, - pack the positional arguments into a - tuple. */ + pack the positional arguments into a tuple. */ if (nseen) { ADDOP_I(c, BUILD_TUPLE, nseen); nseen = 0; @@ -3523,102 +3527,80 @@ compiler_call_helper(struct compiler *c, } VISIT(c, expr, elt->v.Starred.value); nsubargs++; - } - else if (nsubargs) { - /* We've seen star-args already, so we - count towards items-to-pack-into-tuple. */ - VISIT(c, expr, elt); - nseen++; + musttupleunpack = 1; } else { - /* Positional arguments before star-arguments - are left on the stack. */ VISIT(c, expr, elt); - n++; + nseen++; } } - if (nseen) { - /* Pack up any trailing positional arguments. */ - ADDOP_I(c, BUILD_TUPLE, nseen); - nsubargs++; - } - if (nsubargs) { - code |= 1; - if (nsubargs > 1) { + + /* Same dance again for keyword arguments */ + if (musttupleunpack || mustdictunpack) { + if (nseen) { + /* Pack up any trailing positional arguments. */ + ADDOP_I(c, BUILD_TUPLE, nseen); + nsubargs++; + } + if (musttupleunpack || nsubargs > 1) { /* If we ended up with more than one stararg, we need to concatenate them into a single sequence. */ - ADDOP_I(c, BUILD_LIST_UNPACK, nsubargs); + ADDOP_I(c, BUILD_TUPLE_UNPACK, nsubargs); } - } - - /* Same dance again for keyword arguments */ - nseen = 0; /* the number of keyword arguments on the stack following */ - nelts = asdl_seq_LEN(keywords); - for (i = 0; i < nelts; i++) { - keyword_ty kw = asdl_seq_GET(keywords, i); - if (kw->arg == NULL) { - /* A keyword argument unpacking. */ - if (nseen) { - if (nsubkwargs) { + else if (nsubargs == 0) { + ADDOP_I(c, BUILD_TUPLE, 0); + } + nseen = 0; /* the number of keyword arguments on the stack following */ + for (i = 0; i < nkwelts; i++) { + keyword_ty kw = asdl_seq_GET(keywords, i); + if (kw->arg == NULL) { + /* A keyword argument unpacking. */ + if (nseen) { if (!compiler_subkwargs(c, keywords, i - nseen, i)) return 0; nsubkwargs++; + nseen = 0; } - else { - Py_ssize_t j; - for (j = 0; j < nseen; j++) { - VISIT(c, keyword, asdl_seq_GET(keywords, j)); - } - nkw = nseen; - } - nseen = 0; + VISIT(c, expr, kw->value); + nsubkwargs++; + } + else { + nseen++; } - VISIT(c, expr, kw->value); - nsubkwargs++; - } - else { - nseen++; } - } - if (nseen) { - if (nsubkwargs) { + if (nseen) { /* Pack up any trailing keyword arguments. */ - if (!compiler_subkwargs(c, keywords, nelts - nseen, nelts)) + if (!compiler_subkwargs(c, keywords, nkwelts - nseen, nkwelts)) return 0; nsubkwargs++; } - else { - VISIT_SEQ(c, keyword, keywords); - nkw = nseen; + if (mustdictunpack || nsubkwargs > 1) { + /* Pack it all up */ + ADDOP_I(c, BUILD_MAP_UNPACK_WITH_CALL, nsubkwargs); } + ADDOP_I(c, CALL_FUNCTION_EX, nsubkwargs > 0); + return 1; } - if (nsubkwargs) { - code |= 2; - if (nsubkwargs > 1) { - /* Pack it all up */ - int function_pos = n + (code & 1) + 2 * nkw + 1; - ADDOP_I(c, BUILD_MAP_UNPACK_WITH_CALL, nsubkwargs | (function_pos << 8)); + else if (nkwelts) { + PyObject *names; + VISIT_SEQ(c, keyword, keywords); + names = PyTuple_New(nkwelts); + if (names == NULL) { + return 0; + } + for (i = 0; i < nkwelts; i++) { + keyword_ty kw = asdl_seq_GET(keywords, i); + Py_INCREF(kw->arg); + PyTuple_SET_ITEM(names, i, kw->arg); } + ADDOP_N(c, LOAD_CONST, names, consts); + ADDOP_I(c, CALL_FUNCTION_KW, n + nelts + nkwelts); + return 1; } - assert(n < 1<<8); - assert(nkw < 1<<24); - n |= nkw << 8; - - switch (code) { - case 0: - ADDOP_I(c, CALL_FUNCTION, n); - break; - case 1: - ADDOP_I(c, CALL_FUNCTION_VAR, n); - break; - case 2: - ADDOP_I(c, CALL_FUNCTION_KW, n); - break; - case 3: - ADDOP_I(c, CALL_FUNCTION_VAR_KW, n); - break; + else { + ADDOP_I(c, CALL_FUNCTION, n + nelts); + return 1; } - return 1; } @@ -4040,7 +4022,6 @@ compiler_dictcomp(struct compiler *c, expr_ty e) static int compiler_visit_keyword(struct compiler *c, keyword_ty k) { - ADDOP_O(c, LOAD_CONST, k->arg, consts); VISIT(c, expr, k->value); return 1; } -- cgit v1.2.1 From 0780d6cb0e6e406d0030dee40156a6dd00066147 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 11 Sep 2016 14:45:49 +1000 Subject: Issue #23722: Initialize __class__ from type.__new__() The __class__ cell used by zero-argument super() is now initialized from type.__new__ rather than __build_class__, so class methods relying on that will now work correctly when called from metaclass methods during class creation. Patch by Martin Teichmann. --- Python/compile.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index 36683d30e1..4631dbbf18 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1968,7 +1968,7 @@ compiler_class(struct compiler *c, stmt_ty s) return 0; } if (c->u->u_ste->ste_needs_class_closure) { - /* return the (empty) __class__ cell */ + /* store __classcell__ into class namespace */ str = PyUnicode_InternFromString("__class__"); if (str == NULL) { compiler_exit_scope(c); @@ -1981,15 +1981,20 @@ compiler_class(struct compiler *c, stmt_ty s) return 0; } assert(i == 0); - /* Return the cell where to store __class__ */ + ADDOP_I(c, LOAD_CLOSURE, i); + str = PyUnicode_InternFromString("__classcell__"); + if (!str || !compiler_nameop(c, str, Store)) { + Py_XDECREF(str); + compiler_exit_scope(c); + return 0; + } + Py_DECREF(str); } else { + /* This happens when nobody references the cell. */ assert(PyDict_Size(c->u->u_cellvars) == 0); - /* This happens when nobody references the cell. Return None. */ - ADDOP_O(c, LOAD_CONST, Py_None, consts); } - ADDOP_IN_SCOPE(c, RETURN_VALUE); /* create the code object */ co = assemble(c, 1); } -- cgit v1.2.1 From 2fdfdf47618ed421642a743b67d2616c7dae138e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 11 Sep 2016 13:48:15 +0300 Subject: Issue #27129: Replaced wordcode related magic constants with macros. --- Python/compile.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index 4631dbbf18..6d64c076e1 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4948,7 +4948,7 @@ assemble_lnotab(struct assembler *a, struct instr *i) Py_ssize_t len; unsigned char *lnotab; - d_bytecode = a->a_offset - a->a_lineno_off; + d_bytecode = (a->a_offset - a->a_lineno_off) * sizeof(_Py_CODEUNIT); d_lineno = i->i_lineno - a->a_lineno; assert(d_bytecode >= 0); @@ -5055,21 +5055,21 @@ assemble_emit(struct assembler *a, struct instr *i) { int size, arg = 0; Py_ssize_t len = PyBytes_GET_SIZE(a->a_bytecode); - char *code; + _Py_CODEUNIT *code; arg = i->i_oparg; size = instrsize(arg); if (i->i_lineno && !assemble_lnotab(a, i)) return 0; - if (a->a_offset + size >= len) { + if (a->a_offset + size >= len / (int)sizeof(_Py_CODEUNIT)) { if (len > PY_SSIZE_T_MAX / 2) return 0; if (_PyBytes_Resize(&a->a_bytecode, len * 2) < 0) return 0; } - code = PyBytes_AS_STRING(a->a_bytecode) + a->a_offset; + code = (_Py_CODEUNIT *)PyBytes_AS_STRING(a->a_bytecode) + a->a_offset; a->a_offset += size; - write_op_arg((unsigned char*)code, i->i_opcode, arg, size); + write_op_arg(code, i->i_opcode, arg, size); return 1; } @@ -5106,6 +5106,7 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c) if (instr->i_jrel) { instr->i_oparg -= bsize; } + instr->i_oparg *= sizeof(_Py_CODEUNIT); if (instrsize(instr->i_oparg) != isize) { extended_arg_recompile = 1; } @@ -5351,7 +5352,7 @@ assemble(struct compiler *c, int addNone) if (_PyBytes_Resize(&a.a_lnotab, a.a_lnotab_off) < 0) goto error; - if (_PyBytes_Resize(&a.a_bytecode, a.a_offset) < 0) + if (_PyBytes_Resize(&a.a_bytecode, a.a_offset * sizeof(_Py_CODEUNIT)) < 0) goto error; co = makecode(c, &a); -- cgit v1.2.1 From dbb07196c8bebd4c6563e3416eac0e3c866303c4 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 11 Sep 2016 09:45:24 -0700 Subject: Issue #28076: Variable annotations should be mangled for private names. By Ivan Levkivskyi. --- Python/compile.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index 6d64c076e1..6bab86eb0b 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4562,6 +4562,7 @@ static int compiler_annassign(struct compiler *c, stmt_ty s) { expr_ty targ = s->v.AnnAssign.target; + PyObject* mangled; assert(s->kind == AnnAssign_kind); @@ -4576,8 +4577,13 @@ compiler_annassign(struct compiler *c, stmt_ty s) if (s->v.AnnAssign.simple && (c->u->u_scope_type == COMPILER_SCOPE_MODULE || c->u->u_scope_type == COMPILER_SCOPE_CLASS)) { + mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id); + if (!mangled) { + return 0; + } VISIT(c, expr, s->v.AnnAssign.annotation); - ADDOP_O(c, STORE_ANNOTATION, targ->v.Name.id, names) + /* ADDOP_N decrefs its argument */ + ADDOP_N(c, STORE_ANNOTATION, mangled, names); } break; case Attribute_kind: -- cgit v1.2.1 From e74d571e49d3fdb4389b79fa1392dd75e2acb916 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 12 Sep 2016 00:52:40 +0300 Subject: Issue #27213: Fixed different issues with reworked CALL_FUNCTION* opcodes. * BUILD_TUPLE_UNPACK and BUILD_MAP_UNPACK_WITH_CALL no longer generated with single tuple or dict. * Restored more informative error messages for incorrect var-positional and var-keyword arguments. * Removed code duplications in _PyEval_EvalCodeWithName(). * Removed redundant runtime checks and parameters in _PyStack_AsDict(). * Added a workaround and enabled previously disabled test in test_traceback. * Removed dead code from the dis module. --- Python/compile.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index 6bab86eb0b..9502feef7c 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3503,7 +3503,7 @@ compiler_call_helper(struct compiler *c, asdl_seq *keywords) { Py_ssize_t i, nseen, nelts, nkwelts; - int musttupleunpack = 0, mustdictunpack = 0; + int mustdictunpack = 0; /* the number of tuples and dictionaries on the stack */ Py_ssize_t nsubargs = 0, nsubkwargs = 0; @@ -3532,7 +3532,6 @@ compiler_call_helper(struct compiler *c, } VISIT(c, expr, elt->v.Starred.value); nsubargs++; - musttupleunpack = 1; } else { VISIT(c, expr, elt); @@ -3541,13 +3540,13 @@ compiler_call_helper(struct compiler *c, } /* Same dance again for keyword arguments */ - if (musttupleunpack || mustdictunpack) { + if (nsubargs || mustdictunpack) { if (nseen) { /* Pack up any trailing positional arguments. */ ADDOP_I(c, BUILD_TUPLE, nseen); nsubargs++; } - if (musttupleunpack || nsubargs > 1) { + if (nsubargs > 1) { /* If we ended up with more than one stararg, we need to concatenate them into a single sequence. */ ADDOP_I(c, BUILD_TUPLE_UNPACK, nsubargs); @@ -3579,7 +3578,7 @@ compiler_call_helper(struct compiler *c, return 0; nsubkwargs++; } - if (mustdictunpack || nsubkwargs > 1) { + if (nsubkwargs > 1) { /* Pack it all up */ ADDOP_I(c, BUILD_MAP_UNPACK_WITH_CALL, nsubkwargs); } -- cgit v1.2.1 From a9d40323166c43221f8a4f57170cb057f95ad2dc Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 2 Oct 2016 10:33:46 +0300 Subject: Issue #28257: Improved error message when pass a non-iterable as a var-positional argument. Added opcode BUILD_TUPLE_UNPACK_WITH_CALL. --- Python/compile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index 9502feef7c..c0c81e1228 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -987,9 +987,9 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return 1-oparg; case BUILD_LIST_UNPACK: case BUILD_TUPLE_UNPACK: + case BUILD_TUPLE_UNPACK_WITH_CALL: case BUILD_SET_UNPACK: case BUILD_MAP_UNPACK: - return 1 - oparg; case BUILD_MAP_UNPACK_WITH_CALL: return 1 - oparg; case BUILD_MAP: @@ -3549,7 +3549,7 @@ compiler_call_helper(struct compiler *c, if (nsubargs > 1) { /* If we ended up with more than one stararg, we need to concatenate them into a single sequence. */ - ADDOP_I(c, BUILD_TUPLE_UNPACK, nsubargs); + ADDOP_I(c, BUILD_TUPLE_UNPACK_WITH_CALL, nsubargs); } else if (nsubargs == 0) { ADDOP_I(c, BUILD_TUPLE, 0); -- cgit v1.2.1 From f1b02866add58317c7f9c6c813c282659981a147 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 5 Dec 2016 16:47:55 +1000 Subject: Issue #23722: improve __classcell__ compatibility Handling zero-argument super() in __init_subclass__ and __set_name__ involved moving __class__ initialisation to type.__new__. This requires cooperation from custom metaclasses to ensure that the new __classcell__ entry is passed along appropriately. The initial implementation of that change resulted in abruptly broken zero-argument super() support in metaclasses that didn't adhere to the new requirements (such as Django's metaclass for Model definitions). The updated approach adopted here instead emits a deprecation warning for those cases, and makes them work the same way they did in Python 3.5. This patch also improves the related class machinery documentation to cover these details and to include more reader-friendly cross-references and index entries. --- Python/compile.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index a8d7fcd717..35151cdd59 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1967,8 +1967,9 @@ compiler_class(struct compiler *c, stmt_ty s) compiler_exit_scope(c); return 0; } + /* Return __classcell__ if it is referenced, otherwise return None */ if (c->u->u_ste->ste_needs_class_closure) { - /* store __classcell__ into class namespace */ + /* Store __classcell__ into class namespace & return it */ str = PyUnicode_InternFromString("__class__"); if (str == NULL) { compiler_exit_scope(c); @@ -1983,6 +1984,7 @@ compiler_class(struct compiler *c, stmt_ty s) assert(i == 0); ADDOP_I(c, LOAD_CLOSURE, i); + ADDOP(c, DUP_TOP); str = PyUnicode_InternFromString("__classcell__"); if (!str || !compiler_nameop(c, str, Store)) { Py_XDECREF(str); @@ -1992,9 +1994,11 @@ compiler_class(struct compiler *c, stmt_ty s) Py_DECREF(str); } else { - /* This happens when nobody references the cell. */ + /* No methods referenced __class__, so just return None */ assert(PyDict_Size(c->u->u_cellvars) == 0); + ADDOP_O(c, LOAD_CONST, Py_None, consts); } + ADDOP_IN_SCOPE(c, RETURN_VALUE); /* create the code object */ co = assemble(c, 1); } -- cgit v1.2.1 From 3759dfc49056abec76793c75729aa85eea4f32cd Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 11 Dec 2016 19:37:19 +0200 Subject: Issue #28739: f-string expressions no longer accepted as docstrings and by ast.literal_eval() even if they do not include subexpressions. --- Python/compile.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/compile.c') diff --git a/Python/compile.c b/Python/compile.c index 35151cdd59..0e16075852 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3415,7 +3415,8 @@ static int compiler_joined_str(struct compiler *c, expr_ty e) { VISIT_SEQ(c, expr, e->v.JoinedStr.values); - ADDOP_I(c, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values)); + if (asdl_seq_LEN(e->v.JoinedStr.values) != 1) + ADDOP_I(c, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values)); return 1; } -- cgit v1.2.1