diff options
Diffstat (limited to 'Python/compile.c')
| -rw-r--r-- | Python/compile.c | 1322 |
1 files changed, 975 insertions, 347 deletions
diff --git a/Python/compile.c b/Python/compile.c index adc33ac54c..0e16075852 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -29,6 +29,7 @@ #include "code.h" #include "symtable.h" #include "opcode.h" +#include "wordcode_helpers.h" #define DEFAULT_BLOCK_SIZE 16 #define DEFAULT_BLOCKS 8 @@ -43,7 +44,6 @@ struct instr { unsigned i_jabs : 1; unsigned i_jrel : 1; - unsigned i_hasarg : 1; unsigned char i_opcode; int i_oparg; struct basicblock_ *i_target; /* target block (if jump instruction) */ @@ -171,7 +171,6 @@ static int compiler_addop(struct compiler *, int); static int compiler_addop_o(struct compiler *, int, PyObject *, PyObject *); static int compiler_addop_i(struct compiler *, int, Py_ssize_t); static int compiler_addop_j(struct compiler *, int, basicblock *, int); -static basicblock *compiler_use_new_block(struct compiler *); static int compiler_error(struct compiler *, const char *); static int compiler_nameop(struct compiler *, identifier, expr_context_ty); @@ -180,6 +179,7 @@ static int compiler_visit_stmt(struct compiler *, stmt_ty); static int compiler_visit_keyword(struct compiler *, keyword_ty); static int compiler_visit_expr(struct compiler *, expr_ty); static int compiler_augassign(struct compiler *, stmt_ty); +static int compiler_annassign(struct compiler *, stmt_ty); static int compiler_visit_slice(struct compiler *, slice_ty, expr_context_ty); @@ -196,12 +196,22 @@ static int expr_constant(struct compiler *, expr_ty); static int compiler_with(struct compiler *, stmt_ty, int); static int compiler_async_with(struct compiler *, stmt_ty, int); static int compiler_async_for(struct compiler *, stmt_ty); -static int compiler_call_helper(struct compiler *c, Py_ssize_t n, +static int compiler_call_helper(struct compiler *c, int n, asdl_seq *args, asdl_seq *keywords); static int compiler_try_except(struct compiler *, stmt_ty); static int compiler_set_qualname(struct compiler *); +static int compiler_sync_comprehension_generator( + struct compiler *c, + asdl_seq *generators, int gen_index, + expr_ty elt, expr_ty val, int type); + +static int compiler_async_comprehension_generator( + struct compiler *c, + asdl_seq *generators, int gen_index, + expr_ty elt, expr_ty val, int type); + static PyCodeObject *assemble(struct compiler *, int addNone); static PyObject *__doc__; @@ -477,9 +487,9 @@ compiler_unit_check(struct compiler_unit *u) { basicblock *block; for (block = u->u_blocks; block != NULL; block = block->b_list) { - assert((void *)block != (void *)0xcbcbcbcb); - assert((void *)block != (void *)0xfbfbfbfb); - assert((void *)block != (void *)0xdbdbdbdb); + assert((uintptr_t)block != 0xcbcbcbcbU); + assert((uintptr_t)block != 0xfbfbfbfbU); + assert((uintptr_t)block != 0xdbdbdbdbU); if (block->b_instr != NULL) { assert(block->b_ialloc > 0); assert(block->b_iused > 0); @@ -523,6 +533,7 @@ compiler_enter_scope(struct compiler *c, identifier name, int scope_type, void *key, int lineno) { struct compiler_unit *u; + basicblock *block; u = (struct compiler_unit *)PyObject_Malloc(sizeof( struct compiler_unit)); @@ -620,8 +631,11 @@ compiler_enter_scope(struct compiler *c, identifier name, c->u = u; c->c_nestlevel++; - if (compiler_use_new_block(c) == NULL) + + block = compiler_new_block(c); + if (block == NULL) return 0; + c->u->u_curblock = block; if (u->u_scope_type != COMPILER_SCOPE_MODULE) { if (!compiler_set_qualname(c)) @@ -731,6 +745,7 @@ compiler_set_qualname(struct compiler *c) return 1; } + /* Allocate a new block and return a pointer to it. Returns NULL on error. */ @@ -755,16 +770,6 @@ compiler_new_block(struct compiler *c) } static basicblock * -compiler_use_new_block(struct compiler *c) -{ - basicblock *block = compiler_new_block(c); - if (block == NULL) - return NULL; - c->u->u_curblock = block; - return block; -} - -static basicblock * compiler_next_block(struct compiler *c) { basicblock *block = compiler_new_block(c); @@ -810,7 +815,7 @@ compiler_next_instr(struct compiler *c, basicblock *b) oldsize = b->b_ialloc * sizeof(struct instr); newsize = oldsize << 1; - if (oldsize > (PY_SIZE_MAX >> 1)) { + if (oldsize > (SIZE_MAX >> 1)) { PyErr_NoMemory(); return -1; } @@ -939,6 +944,8 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return -1; case IMPORT_STAR: return -1; + case SETUP_ANNOTATIONS: + return 0; case YIELD_VALUE: return 0; case YIELD_FROM: @@ -976,16 +983,19 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) case BUILD_TUPLE: case BUILD_LIST: case BUILD_SET: + case BUILD_STRING: return 1-oparg; case BUILD_LIST_UNPACK: case BUILD_TUPLE_UNPACK: + case BUILD_TUPLE_UNPACK_WITH_CALL: case BUILD_SET_UNPACK: case BUILD_MAP_UNPACK: - return 1 - oparg; case BUILD_MAP_UNPACK_WITH_CALL: - return 1 - (oparg & 0xFF); + return 1 - oparg; case BUILD_MAP: return 1 - 2*oparg; + case BUILD_CONST_KEY_MAP: + return -oparg; case LOAD_ATTR: return 0; case COMPARE_OP: @@ -1023,22 +1033,20 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return -1; case DELETE_FAST: return 0; + case STORE_ANNOTATION: + return -1; case RAISE_VARARGS: return -oparg; -#define NARGS(o) (((o) % 256) + 2*(((o) / 256) % 256)) case CALL_FUNCTION: - return -NARGS(oparg); - case CALL_FUNCTION_VAR: + return -oparg; case CALL_FUNCTION_KW: - return -NARGS(oparg)-1; - case CALL_FUNCTION_VAR_KW: - return -NARGS(oparg)-2; + return -oparg-1; + case CALL_FUNCTION_EX: + return - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0); case MAKE_FUNCTION: - return -1 -NARGS(oparg) - ((oparg >> 16) & 0xffff); - case MAKE_CLOSURE: - return -2 - NARGS(oparg) - ((oparg >> 16) & 0xffff); -#undef NARGS + return -1 - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0) - + ((oparg & 0x04) != 0) - ((oparg & 0x08) != 0); case BUILD_SLICE: if (oparg == 3) return -2; @@ -1066,6 +1074,10 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return 1; case GET_YIELD_FROM_ITER: return 0; + case FORMAT_VALUE: + /* If there's a fmt_spec on the stack, we go from 2->1, + else 1->1. */ + return (oparg & FVS_MASK) == FVS_HAVE_SPEC ? -1 : 0; default: return PY_INVALID_STACK_EFFECT; } @@ -1082,13 +1094,14 @@ compiler_addop(struct compiler *c, int opcode) basicblock *b; struct instr *i; int off; + assert(!HAS_ARG(opcode)); off = compiler_next_instr(c, c->u->u_curblock); if (off < 0) return 0; b = c->u->u_curblock; i = &b->b_instr[off]; i->i_opcode = opcode; - i->i_hasarg = 0; + i->i_oparg = 0; if (opcode == RETURN_VALUE) b->b_return = 1; compiler_set_lineno(c, off); @@ -1165,10 +1178,15 @@ compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg) struct instr *i; int off; - /* Integer arguments are limit to 16-bit. There is an extension for 32-bit - integer arguments. */ - assert((-2147483647-1) <= oparg); - assert(oparg <= 2147483647); + /* oparg value is unsigned, but a signed C int is usually used to store + it in the C code (like Python/ceval.c). + + Limit to 32-bit signed C int (rather than INT_MAX) for portability. + + The argument of a concrete bytecode instruction is limited to 8-bit. + EXTENDED_ARG is used for 16, 24, and 32-bit arguments. */ + assert(HAS_ARG(opcode)); + assert(0 <= oparg && oparg <= 2147483647); off = compiler_next_instr(c, c->u->u_curblock); if (off < 0) @@ -1176,7 +1194,6 @@ compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg) i = &c->u->u_curblock->b_instr[off]; i->i_opcode = opcode; i->i_oparg = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int); - i->i_hasarg = 1; compiler_set_lineno(c, off); return 1; } @@ -1187,6 +1204,7 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute) struct instr *i; int off; + assert(HAS_ARG(opcode)); assert(b != NULL); off = compiler_next_instr(c, c->u->u_curblock); if (off < 0) @@ -1194,7 +1212,6 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute) i = &c->u->u_curblock->b_instr[off]; i->i_opcode = opcode; i->i_target = b; - i->i_hasarg = 1; if (absolute) i->i_jabs = 1; else @@ -1203,22 +1220,12 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute) return 1; } -/* The distinction between NEW_BLOCK and NEXT_BLOCK is subtle. (I'd - like to find better names.) NEW_BLOCK() creates a new block and sets - it as the current block. NEXT_BLOCK() also creates an implicit jump - from the current block to the new block. -*/ +/* NEXT_BLOCK() creates an implicit jump from the current block + to the new block. -/* The returns inside these macros make it impossible to decref objects - created in the local function. Local objects should use the arena. + The returns inside this macro make it impossible to decref objects + created in the local function. Local objects should use the arena. */ - - -#define NEW_BLOCK(C) { \ - if (compiler_use_new_block((C)) == NULL) \ - return 0; \ -} - #define NEXT_BLOCK(C) { \ if (compiler_next_block((C)) == NULL) \ return 0; \ @@ -1241,6 +1248,15 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute) return 0; \ } +/* Same as ADDOP_O, but steals a reference. */ +#define ADDOP_N(C, OP, O, TYPE) { \ + if (!compiler_addop_o((C), (OP), (C)->u->u_ ## TYPE, (O))) { \ + Py_DECREF((O)); \ + return 0; \ + } \ + Py_DECREF((O)); \ +} + #define ADDOP_NAME(C, OP, O, TYPE) { \ if (!compiler_addop_name((C), (OP), (C)->u->u_ ## TYPE, (O))) \ return 0; \ @@ -1309,10 +1325,110 @@ compiler_isdocstring(stmt_ty s) { if (s->kind != Expr_kind) return 0; - return s->v.Expr.value->kind == Str_kind; + if (s->v.Expr.value->kind == Str_kind) + return 1; + if (s->v.Expr.value->kind == Constant_kind) + return PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value); + return 0; +} + +static int +is_const(expr_ty e) +{ + switch (e->kind) { + case Constant_kind: + case Num_kind: + case Str_kind: + case Bytes_kind: + case Ellipsis_kind: + case NameConstant_kind: + return 1; + default: + return 0; + } +} + +static PyObject * +get_const_value(expr_ty e) +{ + switch (e->kind) { + case Constant_kind: + return e->v.Constant.value; + case Num_kind: + return e->v.Num.n; + case Str_kind: + return e->v.Str.s; + case Bytes_kind: + return e->v.Bytes.s; + case Ellipsis_kind: + return Py_Ellipsis; + case NameConstant_kind: + return e->v.NameConstant.value; + default: + assert(!is_const(e)); + return NULL; + } +} + +/* Search if variable annotations are present statically in a block. */ + +static int +find_ann(asdl_seq *stmts) +{ + int i, j, res = 0; + stmt_ty st; + + for (i = 0; i < asdl_seq_LEN(stmts); i++) { + st = (stmt_ty)asdl_seq_GET(stmts, i); + switch (st->kind) { + case AnnAssign_kind: + return 1; + case For_kind: + res = find_ann(st->v.For.body) || + find_ann(st->v.For.orelse); + break; + case AsyncFor_kind: + res = find_ann(st->v.AsyncFor.body) || + find_ann(st->v.AsyncFor.orelse); + break; + case While_kind: + res = find_ann(st->v.While.body) || + find_ann(st->v.While.orelse); + break; + case If_kind: + res = find_ann(st->v.If.body) || + find_ann(st->v.If.orelse); + break; + case With_kind: + res = find_ann(st->v.With.body); + break; + case AsyncWith_kind: + res = find_ann(st->v.AsyncWith.body); + break; + case Try_kind: + for (j = 0; j < asdl_seq_LEN(st->v.Try.handlers); j++) { + excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( + st->v.Try.handlers, j); + if (find_ann(handler->v.ExceptHandler.body)) { + return 1; + } + } + res = find_ann(st->v.Try.body) || + find_ann(st->v.Try.finalbody) || + find_ann(st->v.Try.orelse); + break; + default: + res = 0; + } + if (res) { + break; + } + } + return res; } -/* Compile a sequence of statements, checking for a docstring. */ +/* Compile a sequence of statements, checking for a docstring + and for annotations. */ static int compiler_body(struct compiler *c, asdl_seq *stmts) @@ -1320,6 +1436,19 @@ compiler_body(struct compiler *c, asdl_seq *stmts) int i = 0; stmt_ty st; + /* Set current line number to the line number of first statement. + This way line number for SETUP_ANNOTATIONS will always + coincide with the line number of first "real" statement in module. + If body is empy, then lineno will be set later in assemble. */ + if (c->u->u_scope_type == COMPILER_SCOPE_MODULE && + !c->u->u_lineno && asdl_seq_LEN(stmts)) { + st = (stmt_ty)asdl_seq_GET(stmts, 0); + c->u->u_lineno = st->lineno; + } + /* Every annotated class and module should have __annotations__. */ + if (find_ann(stmts)) { + ADDOP(c, SETUP_ANNOTATIONS); + } if (!asdl_seq_LEN(stmts)) return 1; st = (stmt_ty)asdl_seq_GET(stmts, 0); @@ -1357,6 +1486,9 @@ compiler_mod(struct compiler *c, mod_ty mod) } break; case Interactive_kind: + if (find_ann(mod->v.Interactive.body)) { + ADDOP(c, SETUP_ANNOTATIONS); + } c->c_interactive = 1; VISIT_SEQ_IN_SCOPE(c, stmt, mod->v.Interactive.body); @@ -1426,53 +1558,50 @@ compiler_lookup_arg(PyObject *dict, PyObject *name) } static int -compiler_make_closure(struct compiler *c, PyCodeObject *co, Py_ssize_t args, PyObject *qualname) +compiler_make_closure(struct compiler *c, PyCodeObject *co, Py_ssize_t flags, PyObject *qualname) { Py_ssize_t i, free = PyCode_GetNumFree(co); if (qualname == NULL) qualname = co->co_name; - if (free == 0) { - ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts); - ADDOP_O(c, LOAD_CONST, qualname, consts); - ADDOP_I(c, MAKE_FUNCTION, args); - return 1; - } - for (i = 0; i < free; ++i) { - /* Bypass com_addop_varname because it will generate - LOAD_DEREF but LOAD_CLOSURE is needed. - */ - PyObject *name = PyTuple_GET_ITEM(co->co_freevars, i); - int arg, reftype; - - /* Special case: If a class contains a method with a - free variable that has the same name as a method, - the name will be considered free *and* local in the - class. It should be handled by the closure, as - well as by the normal name loookup logic. - */ - reftype = get_ref_type(c, name); - if (reftype == CELL) - arg = compiler_lookup_arg(c->u->u_cellvars, name); - else /* (reftype == FREE) */ - arg = compiler_lookup_arg(c->u->u_freevars, name); - if (arg == -1) { - fprintf(stderr, - "lookup %s in %s %d %d\n" - "freevars of %s: %s\n", - PyUnicode_AsUTF8(PyObject_Repr(name)), - PyUnicode_AsUTF8(c->u->u_name), - reftype, arg, - PyUnicode_AsUTF8(co->co_name), - PyUnicode_AsUTF8(PyObject_Repr(co->co_freevars))); - Py_FatalError("compiler_make_closure()"); + if (free) { + for (i = 0; i < free; ++i) { + /* Bypass com_addop_varname because it will generate + LOAD_DEREF but LOAD_CLOSURE is needed. + */ + PyObject *name = PyTuple_GET_ITEM(co->co_freevars, i); + int arg, reftype; + + /* Special case: If a class contains a method with a + free variable that has the same name as a method, + the name will be considered free *and* local in the + class. It should be handled by the closure, as + well as by the normal name loookup logic. + */ + reftype = get_ref_type(c, name); + if (reftype == CELL) + arg = compiler_lookup_arg(c->u->u_cellvars, name); + else /* (reftype == FREE) */ + arg = compiler_lookup_arg(c->u->u_freevars, name); + if (arg == -1) { + fprintf(stderr, + "lookup %s in %s %d %d\n" + "freevars of %s: %s\n", + PyUnicode_AsUTF8(PyObject_Repr(name)), + PyUnicode_AsUTF8(c->u->u_name), + reftype, arg, + PyUnicode_AsUTF8(co->co_name), + PyUnicode_AsUTF8(PyObject_Repr(co->co_freevars))); + Py_FatalError("compiler_make_closure()"); + } + ADDOP_I(c, LOAD_CLOSURE, arg); } - ADDOP_I(c, LOAD_CLOSURE, arg); + flags |= 0x08; + ADDOP_I(c, BUILD_TUPLE, free); } - ADDOP_I(c, BUILD_TUPLE, free); ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts); ADDOP_O(c, LOAD_CONST, qualname, consts); - ADDOP_I(c, MAKE_CLOSURE, args); + ADDOP_I(c, MAKE_FUNCTION, flags); return 1; } @@ -1494,26 +1623,60 @@ static int compiler_visit_kwonlydefaults(struct compiler *c, asdl_seq *kwonlyargs, asdl_seq *kw_defaults) { - /* Return the number of defaults + 1. - Returns 0 on error. - */ - int i, default_count = 0; + /* Push a dict of keyword-only default values. + + Return 0 on error, -1 if no dict pushed, 1 if a dict is pushed. + */ + int i; + PyObject *keys = NULL; + for (i = 0; i < asdl_seq_LEN(kwonlyargs); i++) { arg_ty arg = asdl_seq_GET(kwonlyargs, i); expr_ty default_ = asdl_seq_GET(kw_defaults, i); if (default_) { PyObject *mangled = _Py_Mangle(c->u->u_private, arg->arg); - if (!mangled) - return 0; - ADDOP_O(c, LOAD_CONST, mangled, consts); - Py_DECREF(mangled); + if (!mangled) { + goto error; + } + if (keys == NULL) { + keys = PyList_New(1); + if (keys == NULL) { + Py_DECREF(mangled); + return 0; + } + PyList_SET_ITEM(keys, 0, mangled); + } + else { + int res = PyList_Append(keys, mangled); + Py_DECREF(mangled); + if (res == -1) { + goto error; + } + } if (!compiler_visit_expr(c, default_)) { - return 0; + goto error; } - default_count++; } } - return default_count + 1; + if (keys != NULL) { + Py_ssize_t default_count = PyList_GET_SIZE(keys); + PyObject *keys_tuple = PyList_AsTuple(keys); + Py_DECREF(keys); + if (keys_tuple == NULL) { + return 0; + } + ADDOP_N(c, LOAD_CONST, keys_tuple, consts); + ADDOP_I(c, BUILD_CONST_KEY_MAP, default_count); + assert(default_count > 0); + return 1; + } + else { + return -1; + } + +error: + Py_XDECREF(keys); + return 0; } static int @@ -1556,11 +1719,10 @@ static int compiler_visit_annotations(struct compiler *c, arguments_ty args, expr_ty returns) { - /* Push arg annotations and a list of the argument names. Return the # - of items pushed + 1. The expressions are evaluated out-of-order wrt the - source code. + /* Push arg annotation dict. + The expressions are evaluated out-of-order wrt the source code. - More than 2^16-1 annotations is a SyntaxError. Returns 0 on error. + Return 0 on error, -1 if no dict pushed, 1 if a dict is pushed. */ static identifier return_str; PyObject *names; @@ -1592,32 +1754,20 @@ compiler_visit_annotations(struct compiler *c, arguments_ty args, } len = PyList_GET_SIZE(names); - if (len > 65534) { - /* len must fit in 16 bits, and len is incremented below */ - PyErr_SetString(PyExc_SyntaxError, - "too many annotations"); - goto error; - } if (len) { - /* convert names to a tuple and place on stack */ - PyObject *elt; - Py_ssize_t i; - PyObject *s = PyTuple_New(len); - if (!s) - goto error; - for (i = 0; i < len; i++) { - elt = PyList_GET_ITEM(names, i); - Py_INCREF(elt); - PyTuple_SET_ITEM(s, i, elt); + PyObject *keytuple = PyList_AsTuple(names); + Py_DECREF(names); + if (keytuple == NULL) { + return 0; } - ADDOP_O(c, LOAD_CONST, s, consts); - Py_DECREF(s); - len++; /* include the just-pushed tuple */ + ADDOP_N(c, LOAD_CONST, keytuple, consts); + ADDOP_I(c, BUILD_CONST_KEY_MAP, len); + return 1; + } + else { + Py_DECREF(names); + return -1; } - Py_DECREF(names); - - /* We just checked that len <= 65535, see above */ - return Py_SAFE_DOWNCAST(len + 1, Py_ssize_t, int); error: Py_DECREF(names); @@ -1625,6 +1775,36 @@ error: } static int +compiler_visit_defaults(struct compiler *c, arguments_ty args) +{ + VISIT_SEQ(c, expr, args->defaults); + ADDOP_I(c, BUILD_TUPLE, asdl_seq_LEN(args->defaults)); + return 1; +} + +static Py_ssize_t +compiler_default_arguments(struct compiler *c, arguments_ty args) +{ + Py_ssize_t funcflags = 0; + if (args->defaults && asdl_seq_LEN(args->defaults) > 0) { + if (!compiler_visit_defaults(c, args)) + return -1; + funcflags |= 0x01; + } + if (args->kwonlyargs) { + int res = compiler_visit_kwonlydefaults(c, args->kwonlyargs, + args->kw_defaults); + if (res == 0) { + return -1; + } + else if (res > 0) { + funcflags |= 0x02; + } + } + return funcflags; +} + +static int compiler_function(struct compiler *c, stmt_ty s, int is_async) { PyCodeObject *co; @@ -1635,12 +1815,11 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) asdl_seq* decos; asdl_seq *body; stmt_ty st; - Py_ssize_t i, n, arglength; - int docstring, kw_default_count = 0; - int num_annotations; + Py_ssize_t i, n, funcflags; + int docstring; + int annotations; int scope_type; - if (is_async) { assert(s->kind == AsyncFunctionDef_kind); @@ -1665,30 +1844,32 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (!compiler_decorators(c, decos)) return 0; - if (args->defaults) - VISIT_SEQ(c, expr, args->defaults); - if (args->kwonlyargs) { - int res = compiler_visit_kwonlydefaults(c, args->kwonlyargs, - args->kw_defaults); - if (res == 0) - return 0; - kw_default_count = res - 1; + + funcflags = compiler_default_arguments(c, args); + if (funcflags == -1) { + return 0; } - num_annotations = compiler_visit_annotations(c, args, returns); - if (num_annotations == 0) + + annotations = compiler_visit_annotations(c, args, returns); + if (annotations == 0) { return 0; - num_annotations--; - assert((num_annotations & 0xFFFF) == num_annotations); + } + else if (annotations > 0) { + funcflags |= 0x04; + } - if (!compiler_enter_scope(c, name, - scope_type, (void *)s, - s->lineno)) + if (!compiler_enter_scope(c, name, scope_type, (void *)s, s->lineno)) { return 0; + } st = (stmt_ty)asdl_seq_GET(body, 0); docstring = compiler_isdocstring(st); - if (docstring && c->c_optimize < 2) - first_const = st->v.Expr.value->v.Str.s; + if (docstring && c->c_optimize < 2) { + if (st->v.Expr.value->kind == Constant_kind) + first_const = st->v.Expr.value->v.Constant.value; + else + first_const = st->v.Expr.value->v.Str.s; + } if (compiler_add_o(c, c->u->u_consts, first_const) < 0) { compiler_exit_scope(c); return 0; @@ -1712,12 +1893,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) return 0; } - arglength = asdl_seq_LEN(args->defaults); - arglength |= kw_default_count << 8; - arglength |= num_annotations << 16; - if (is_async) - co->co_flags |= CO_COROUTINE; - compiler_make_closure(c, co, arglength, qualname); + compiler_make_closure(c, co, funcflags, qualname); Py_DECREF(qualname); Py_DECREF(co); @@ -1791,8 +1967,9 @@ compiler_class(struct compiler *c, stmt_ty s) compiler_exit_scope(c); return 0; } + /* Return __classcell__ if it is referenced, otherwise return None */ if (c->u->u_ste->ste_needs_class_closure) { - /* return the (empty) __class__ cell */ + /* Store __classcell__ into class namespace & return it */ str = PyUnicode_InternFromString("__class__"); if (str == NULL) { compiler_exit_scope(c); @@ -1805,12 +1982,20 @@ compiler_class(struct compiler *c, stmt_ty s) return 0; } assert(i == 0); - /* Return the cell where to store __class__ */ + ADDOP_I(c, LOAD_CLOSURE, i); + ADDOP(c, DUP_TOP); + str = PyUnicode_InternFromString("__classcell__"); + if (!str || !compiler_nameop(c, str, Store)) { + Py_XDECREF(str); + compiler_exit_scope(c); + return 0; + } + Py_DECREF(str); } else { + /* No methods referenced __class__, so just return None */ assert(PyDict_Size(c->u->u_cellvars) == 0); - /* This happens when nobody references the cell. Return None. */ ADDOP_O(c, LOAD_CONST, Py_None, consts); } ADDOP_IN_SCOPE(c, RETURN_VALUE); @@ -1877,8 +2062,7 @@ compiler_lambda(struct compiler *c, expr_ty e) PyCodeObject *co; PyObject *qualname; static identifier name; - int kw_default_count = 0; - Py_ssize_t arglength; + Py_ssize_t funcflags; arguments_ty args = e->v.Lambda.args; assert(e->kind == Lambda_kind); @@ -1888,14 +2072,11 @@ compiler_lambda(struct compiler *c, expr_ty e) return 0; } - if (args->defaults) - VISIT_SEQ(c, expr, args->defaults); - if (args->kwonlyargs) { - int res = compiler_visit_kwonlydefaults(c, args->kwonlyargs, - args->kw_defaults); - if (res == 0) return 0; - kw_default_count = res - 1; + funcflags = compiler_default_arguments(c, args); + if (funcflags == -1) { + return 0; } + if (!compiler_enter_scope(c, name, COMPILER_SCOPE_LAMBDA, (void *)e, e->lineno)) return 0; @@ -1921,9 +2102,7 @@ compiler_lambda(struct compiler *c, expr_ty e) if (co == NULL) return 0; - arglength = asdl_seq_LEN(args->defaults); - arglength |= kw_default_count << 8; - compiler_make_closure(c, co, arglength, qualname); + compiler_make_closure(c, co, funcflags, qualname); Py_DECREF(qualname); Py_DECREF(co); @@ -2002,14 +2181,14 @@ compiler_for(struct compiler *c, stmt_ty s) static int compiler_async_for(struct compiler *c, stmt_ty s) { - static PyObject *stopiter_error = NULL; + _Py_IDENTIFIER(StopAsyncIteration); + basicblock *try, *except, *end, *after_try, *try_cleanup, *after_loop, *after_loop_else; - if (stopiter_error == NULL) { - stopiter_error = PyUnicode_InternFromString("StopAsyncIteration"); - if (stopiter_error == NULL) - return 0; + PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration); + if (stop_aiter_error == NULL) { + return 0; } try = compiler_new_block(c); @@ -2051,7 +2230,7 @@ compiler_async_for(struct compiler *c, stmt_ty s) compiler_use_next_block(c, except); ADDOP(c, DUP_TOP); - ADDOP_O(c, LOAD_GLOBAL, stopiter_error, names); + ADDOP_O(c, LOAD_GLOBAL, stop_aiter_error, names); ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH); ADDOP_JABS(c, POP_JUMP_IF_FALSE, try_cleanup); @@ -2599,6 +2778,25 @@ compiler_assert(struct compiler *c, stmt_ty s) } static int +compiler_visit_stmt_expr(struct compiler *c, expr_ty value) +{ + if (c->c_interactive && c->c_nestlevel <= 1) { + VISIT(c, expr, value); + ADDOP(c, PRINT_EXPR); + return 1; + } + + if (is_const(value)) { + /* ignore constant statement */ + return 1; + } + + VISIT(c, expr, value); + ADDOP(c, POP_TOP); + return 1; +} + +static int compiler_visit_stmt(struct compiler *c, stmt_ty s) { Py_ssize_t i, n; @@ -2617,6 +2815,9 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) if (c->u->u_ste->ste_type != FunctionBlock) return compiler_error(c, "'return' outside function"); if (s->v.Return.value) { + if (c->u->u_ste->ste_coroutine && c->u->u_ste->ste_generator) + return compiler_error( + c, "'return' with value in async generator"); VISIT(c, expr, s->v.Return.value); } else @@ -2638,6 +2839,8 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) break; case AugAssign_kind: return compiler_augassign(c, s); + case AnnAssign_kind: + return compiler_annassign(c, s); case For_kind: return compiler_for(c, s); case While_kind: @@ -2668,16 +2871,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) case Nonlocal_kind: break; case Expr_kind: - if (c->c_interactive && c->c_nestlevel <= 1) { - VISIT(c, expr, s->v.Expr.value); - ADDOP(c, PRINT_EXPR); - } - else if (s->v.Expr.value->kind != Str_kind && - s->v.Expr.value->kind != Num_kind) { - VISIT(c, expr, s->v.Expr.value); - ADDOP(c, POP_TOP); - } - break; + return compiler_visit_stmt_expr(c, s->v.Expr.value); case Pass_kind: break; case Break_kind: @@ -3079,9 +3273,53 @@ compiler_set(struct compiler *c, expr_ty e) } static int +are_all_items_const(asdl_seq *seq, Py_ssize_t begin, Py_ssize_t end) +{ + Py_ssize_t i; + for (i = begin; i < end; i++) { + expr_ty key = (expr_ty)asdl_seq_GET(seq, i); + if (key == NULL || !is_const(key)) + return 0; + } + return 1; +} + +static int +compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end) +{ + Py_ssize_t i, n = end - begin; + PyObject *keys, *key; + if (n > 1 && are_all_items_const(e->v.Dict.keys, begin, end)) { + for (i = begin; i < end; i++) { + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); + } + keys = PyTuple_New(n); + if (keys == NULL) { + return 0; + } + for (i = begin; i < end; i++) { + key = get_const_value((expr_ty)asdl_seq_GET(e->v.Dict.keys, i)); + Py_INCREF(key); + PyTuple_SET_ITEM(keys, i - begin, key); + } + ADDOP_N(c, LOAD_CONST, keys, consts); + ADDOP_I(c, BUILD_CONST_KEY_MAP, n); + } + else { + for (i = begin; i < end; i++) { + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i)); + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); + } + ADDOP_I(c, BUILD_MAP, n); + } + return 1; +} + +static int compiler_dict(struct compiler *c, expr_ty e) { - Py_ssize_t i, n, containers, elements; + Py_ssize_t i, n, elements; + int containers; int is_unpacking = 0; n = asdl_seq_LEN(e->v.Dict.values); containers = 0; @@ -3089,7 +3327,8 @@ compiler_dict(struct compiler *c, expr_ty e) for (i = 0; i < n; i++) { is_unpacking = (expr_ty)asdl_seq_GET(e->v.Dict.keys, i) == NULL; if (elements == 0xFFFF || (elements && is_unpacking)) { - ADDOP_I(c, BUILD_MAP, elements); + if (!compiler_subdict(c, e, i - elements, i)) + return 0; containers++; elements = 0; } @@ -3098,13 +3337,12 @@ compiler_dict(struct compiler *c, expr_ty e) containers++; } else { - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i)); - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); elements++; } } if (elements || containers == 0) { - ADDOP_I(c, BUILD_MAP, elements); + if (!compiler_subdict(c, e, n - elements, n)) + return 0; containers++; } /* If there is more than one dict, they need to be merged into a new @@ -3173,28 +3411,125 @@ compiler_call(struct compiler *c, expr_ty e) e->v.Call.keywords); } +static int +compiler_joined_str(struct compiler *c, expr_ty e) +{ + VISIT_SEQ(c, expr, e->v.JoinedStr.values); + if (asdl_seq_LEN(e->v.JoinedStr.values) != 1) + ADDOP_I(c, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values)); + return 1; +} + +/* Used to implement f-strings. Format a single value. */ +static int +compiler_formatted_value(struct compiler *c, expr_ty e) +{ + /* Our oparg encodes 2 pieces of information: the conversion + character, and whether or not a format_spec was provided. + + Convert the conversion char to 2 bits: + None: 000 0x0 FVC_NONE + !s : 001 0x1 FVC_STR + !r : 010 0x2 FVC_REPR + !a : 011 0x3 FVC_ASCII + + next bit is whether or not we have a format spec: + yes : 100 0x4 + no : 000 0x0 + */ + + int oparg; + + /* Evaluate the expression to be formatted. */ + VISIT(c, expr, e->v.FormattedValue.value); + + switch (e->v.FormattedValue.conversion) { + case 's': oparg = FVC_STR; break; + case 'r': oparg = FVC_REPR; break; + case 'a': oparg = FVC_ASCII; break; + case -1: oparg = FVC_NONE; break; + default: + PyErr_SetString(PyExc_SystemError, + "Unrecognized conversion character"); + return 0; + } + if (e->v.FormattedValue.format_spec) { + /* Evaluate the format spec, and update our opcode arg. */ + VISIT(c, expr, e->v.FormattedValue.format_spec); + oparg |= FVS_HAVE_SPEC; + } + + /* And push our opcode and oparg */ + ADDOP_I(c, FORMAT_VALUE, oparg); + return 1; +} + +static int +compiler_subkwargs(struct compiler *c, asdl_seq *keywords, Py_ssize_t begin, Py_ssize_t end) +{ + Py_ssize_t i, n = end - begin; + keyword_ty kw; + PyObject *keys, *key; + assert(n > 0); + if (n > 1) { + for (i = begin; i < end; i++) { + kw = asdl_seq_GET(keywords, i); + VISIT(c, expr, kw->value); + } + keys = PyTuple_New(n); + if (keys == NULL) { + return 0; + } + for (i = begin; i < end; i++) { + key = ((keyword_ty) asdl_seq_GET(keywords, i))->arg; + Py_INCREF(key); + PyTuple_SET_ITEM(keys, i - begin, key); + } + ADDOP_N(c, LOAD_CONST, keys, consts); + ADDOP_I(c, BUILD_CONST_KEY_MAP, n); + } + else { + /* a for loop only executes once */ + for (i = begin; i < end; i++) { + kw = asdl_seq_GET(keywords, i); + ADDOP_O(c, LOAD_CONST, kw->arg, consts); + VISIT(c, expr, kw->value); + } + ADDOP_I(c, BUILD_MAP, n); + } + return 1; +} + /* shared code between compiler_call and compiler_class */ static int compiler_call_helper(struct compiler *c, - Py_ssize_t n, /* Args already pushed */ + int n, /* Args already pushed */ asdl_seq *args, asdl_seq *keywords) { - int code = 0; - Py_ssize_t nelts, i, nseen, nkw; + Py_ssize_t i, nseen, nelts, nkwelts; + int mustdictunpack = 0; /* the number of tuples and dictionaries on the stack */ Py_ssize_t nsubargs = 0, nsubkwargs = 0; - nkw = 0; - nseen = 0; /* the number of positional arguments on the stack */ nelts = asdl_seq_LEN(args); + nkwelts = asdl_seq_LEN(keywords); + + for (i = 0; i < nkwelts; i++) { + keyword_ty kw = asdl_seq_GET(keywords, i); + if (kw->arg == NULL) { + mustdictunpack = 1; + break; + } + } + + nseen = n; /* the number of positional arguments on the stack */ for (i = 0; i < nelts; i++) { expr_ty elt = asdl_seq_GET(args, i); if (elt->kind == Starred_kind) { /* A star-arg. If we've seen positional arguments, - pack the positional arguments into a - tuple. */ + pack the positional arguments into a tuple. */ if (nseen) { ADDOP_I(c, BUILD_TUPLE, nseen); nseen = 0; @@ -3203,92 +3538,78 @@ compiler_call_helper(struct compiler *c, VISIT(c, expr, elt->v.Starred.value); nsubargs++; } - else if (nsubargs) { - /* We've seen star-args already, so we - count towards items-to-pack-into-tuple. */ - VISIT(c, expr, elt); - nseen++; - } else { - /* Positional arguments before star-arguments - are left on the stack. */ VISIT(c, expr, elt); - n++; + nseen++; } } - if (nseen) { - /* Pack up any trailing positional arguments. */ - ADDOP_I(c, BUILD_TUPLE, nseen); - nsubargs++; - } - if (nsubargs) { - code |= 1; + + /* Same dance again for keyword arguments */ + if (nsubargs || mustdictunpack) { + if (nseen) { + /* Pack up any trailing positional arguments. */ + ADDOP_I(c, BUILD_TUPLE, nseen); + nsubargs++; + } if (nsubargs > 1) { /* If we ended up with more than one stararg, we need to concatenate them into a single sequence. */ - ADDOP_I(c, BUILD_LIST_UNPACK, nsubargs); + ADDOP_I(c, BUILD_TUPLE_UNPACK_WITH_CALL, nsubargs); } - } - - /* Same dance again for keyword arguments */ - nseen = 0; /* the number of keyword arguments on the stack following */ - nelts = asdl_seq_LEN(keywords); - for (i = 0; i < nelts; i++) { - keyword_ty kw = asdl_seq_GET(keywords, i); - if (kw->arg == NULL) { - /* A keyword argument unpacking. */ - if (nseen) { - ADDOP_I(c, BUILD_MAP, nseen); - nseen = 0; + else if (nsubargs == 0) { + ADDOP_I(c, BUILD_TUPLE, 0); + } + nseen = 0; /* the number of keyword arguments on the stack following */ + for (i = 0; i < nkwelts; i++) { + keyword_ty kw = asdl_seq_GET(keywords, i); + if (kw->arg == NULL) { + /* A keyword argument unpacking. */ + if (nseen) { + if (!compiler_subkwargs(c, keywords, i - nseen, i)) + return 0; + nsubkwargs++; + nseen = 0; + } + VISIT(c, expr, kw->value); nsubkwargs++; } - VISIT(c, expr, kw->value); - nsubkwargs++; - } - else if (nsubkwargs) { - /* A keyword argument and we already have a dict. */ - ADDOP_O(c, LOAD_CONST, kw->arg, consts); - VISIT(c, expr, kw->value); - nseen++; + else { + nseen++; + } } - else { - /* keyword argument */ - VISIT(c, keyword, kw) - nkw++; + if (nseen) { + /* Pack up any trailing keyword arguments. */ + if (!compiler_subkwargs(c, keywords, nkwelts - nseen, nkwelts)) + return 0; + nsubkwargs++; } - } - if (nseen) { - /* Pack up any trailing keyword arguments. */ - ADDOP_I(c, BUILD_MAP, nseen); - nsubkwargs++; - } - if (nsubkwargs) { - code |= 2; if (nsubkwargs > 1) { /* Pack it all up */ - int function_pos = n + (code & 1) + 2 * nkw + 1; - ADDOP_I(c, BUILD_MAP_UNPACK_WITH_CALL, nsubkwargs | (function_pos << 8)); + ADDOP_I(c, BUILD_MAP_UNPACK_WITH_CALL, nsubkwargs); } + ADDOP_I(c, CALL_FUNCTION_EX, nsubkwargs > 0); + return 1; } - assert(n < 1<<8); - assert(nkw < 1<<24); - n |= nkw << 8; - - switch (code) { - case 0: - ADDOP_I(c, CALL_FUNCTION, n); - break; - case 1: - ADDOP_I(c, CALL_FUNCTION_VAR, n); - break; - case 2: - ADDOP_I(c, CALL_FUNCTION_KW, n); - break; - case 3: - ADDOP_I(c, CALL_FUNCTION_VAR_KW, n); - break; + else if (nkwelts) { + PyObject *names; + VISIT_SEQ(c, keyword, keywords); + names = PyTuple_New(nkwelts); + if (names == NULL) { + return 0; + } + for (i = 0; i < nkwelts; i++) { + keyword_ty kw = asdl_seq_GET(keywords, i); + Py_INCREF(kw->arg); + PyTuple_SET_ITEM(names, i, kw->arg); + } + ADDOP_N(c, LOAD_CONST, names, consts); + ADDOP_I(c, CALL_FUNCTION_KW, n + nelts + nkwelts); + return 1; + } + else { + ADDOP_I(c, CALL_FUNCTION, n + nelts); + return 1; } - return 1; } @@ -3307,11 +3628,28 @@ compiler_call_helper(struct compiler *c, - iterate over the generator sequence instead of using recursion */ + static int compiler_comprehension_generator(struct compiler *c, asdl_seq *generators, int gen_index, expr_ty elt, expr_ty val, int type) { + comprehension_ty gen; + gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); + if (gen->is_async) { + return compiler_async_comprehension_generator( + c, generators, gen_index, elt, val, type); + } else { + return compiler_sync_comprehension_generator( + c, generators, gen_index, elt, val, type); + } +} + +static int +compiler_sync_comprehension_generator(struct compiler *c, + asdl_seq *generators, int gen_index, + expr_ty elt, expr_ty val, int type) +{ /* generate code for the iterator, then each of the ifs, and then write to the element */ @@ -3398,20 +3736,167 @@ compiler_comprehension_generator(struct compiler *c, } static int +compiler_async_comprehension_generator(struct compiler *c, + asdl_seq *generators, int gen_index, + expr_ty elt, expr_ty val, int type) +{ + _Py_IDENTIFIER(StopAsyncIteration); + + comprehension_ty gen; + basicblock *anchor, *skip, *if_cleanup, *try, + *after_try, *except, *try_cleanup; + Py_ssize_t i, n; + + PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration); + if (stop_aiter_error == NULL) { + return 0; + } + + try = compiler_new_block(c); + after_try = compiler_new_block(c); + try_cleanup = compiler_new_block(c); + except = compiler_new_block(c); + skip = compiler_new_block(c); + if_cleanup = compiler_new_block(c); + anchor = compiler_new_block(c); + + if (skip == NULL || if_cleanup == NULL || anchor == NULL || + try == NULL || after_try == NULL || + except == NULL || after_try == NULL) { + return 0; + } + + gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); + + if (gen_index == 0) { + /* Receive outermost iter as an implicit argument */ + c->u->u_argcount = 1; + ADDOP_I(c, LOAD_FAST, 0); + } + else { + /* Sub-iter - calculate on the fly */ + VISIT(c, expr, gen->iter); + ADDOP(c, GET_AITER); + ADDOP_O(c, LOAD_CONST, Py_None, consts); + ADDOP(c, YIELD_FROM); + } + + compiler_use_next_block(c, try); + + + ADDOP_JREL(c, SETUP_EXCEPT, except); + if (!compiler_push_fblock(c, EXCEPT, try)) + return 0; + + ADDOP(c, GET_ANEXT); + ADDOP_O(c, LOAD_CONST, Py_None, consts); + ADDOP(c, YIELD_FROM); + VISIT(c, expr, gen->target); + ADDOP(c, POP_BLOCK); + compiler_pop_fblock(c, EXCEPT, try); + ADDOP_JREL(c, JUMP_FORWARD, after_try); + + + compiler_use_next_block(c, except); + ADDOP(c, DUP_TOP); + ADDOP_O(c, LOAD_GLOBAL, stop_aiter_error, names); + ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH); + ADDOP_JABS(c, POP_JUMP_IF_FALSE, try_cleanup); + + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */ + ADDOP_JABS(c, JUMP_ABSOLUTE, anchor); + + + compiler_use_next_block(c, try_cleanup); + ADDOP(c, END_FINALLY); + + compiler_use_next_block(c, after_try); + + n = asdl_seq_LEN(gen->ifs); + for (i = 0; i < n; i++) { + expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); + VISIT(c, expr, e); + ADDOP_JABS(c, POP_JUMP_IF_FALSE, if_cleanup); + NEXT_BLOCK(c); + } + + if (++gen_index < asdl_seq_LEN(generators)) + if (!compiler_comprehension_generator(c, + generators, gen_index, + elt, val, type)) + return 0; + + /* only append after the last for generator */ + if (gen_index >= asdl_seq_LEN(generators)) { + /* comprehension specific code */ + switch (type) { + case COMP_GENEXP: + VISIT(c, expr, elt); + ADDOP(c, YIELD_VALUE); + ADDOP(c, POP_TOP); + break; + case COMP_LISTCOMP: + VISIT(c, expr, elt); + ADDOP_I(c, LIST_APPEND, gen_index + 1); + break; + case COMP_SETCOMP: + VISIT(c, expr, elt); + ADDOP_I(c, SET_ADD, gen_index + 1); + break; + case COMP_DICTCOMP: + /* With 'd[k] = v', v is evaluated before k, so we do + the same. */ + VISIT(c, expr, val); + VISIT(c, expr, elt); + ADDOP_I(c, MAP_ADD, gen_index + 1); + break; + default: + return 0; + } + + compiler_use_next_block(c, skip); + } + compiler_use_next_block(c, if_cleanup); + ADDOP_JABS(c, JUMP_ABSOLUTE, try); + compiler_use_next_block(c, anchor); + ADDOP(c, POP_TOP); + + return 1; +} + +static int compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name, asdl_seq *generators, expr_ty elt, expr_ty val) { PyCodeObject *co = NULL; - expr_ty outermost_iter; + comprehension_ty outermost; PyObject *qualname = NULL; + int is_async_function = c->u->u_ste->ste_coroutine; + int is_async_generator = 0; - outermost_iter = ((comprehension_ty) - asdl_seq_GET(generators, 0))->iter; + outermost = (comprehension_ty) asdl_seq_GET(generators, 0); if (!compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION, (void *)e, e->lineno)) + { goto error; + } + + is_async_generator = c->u->u_ste->ste_coroutine; + + if (is_async_generator && !is_async_function) { + if (e->lineno > c->u->u_lineno) { + c->u->u_lineno = e->lineno; + c->u->u_lineno_set = 0; + } + compiler_error(c, "asynchronous comprehension outside of " + "an asynchronous function"); + goto error_in_scope; + } if (type != COMP_GENEXP) { int op; @@ -3454,9 +3939,24 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, Py_DECREF(qualname); Py_DECREF(co); - VISIT(c, expr, outermost_iter); - ADDOP(c, GET_ITER); + VISIT(c, expr, outermost->iter); + + if (outermost->is_async) { + ADDOP(c, GET_AITER); + ADDOP_O(c, LOAD_CONST, Py_None, consts); + ADDOP(c, YIELD_FROM); + } else { + ADDOP(c, GET_ITER); + } + ADDOP_I(c, CALL_FUNCTION, 1); + + if (is_async_generator && type != COMP_GENEXP) { + ADDOP(c, GET_AWAITABLE); + ADDOP_O(c, LOAD_CONST, Py_None, consts); + ADDOP(c, YIELD_FROM); + } + return 1; error_in_scope: compiler_exit_scope(c); @@ -3531,7 +4031,6 @@ compiler_dictcomp(struct compiler *c, expr_ty e) static int compiler_visit_keyword(struct compiler *c, keyword_ty k) { - ADDOP_O(c, LOAD_CONST, k->arg, consts); VISIT(c, expr, k->value); return 1; } @@ -3549,6 +4048,8 @@ expr_constant(struct compiler *c, expr_ty e) switch (e->kind) { case Ellipsis_kind: return 1; + case Constant_kind: + return PyObject_IsTrue(e->v.Constant.value); case Num_kind: return PyObject_IsTrue(e->v.Num.n); case Str_kind: @@ -3592,9 +4093,9 @@ expr_constant(struct compiler *c, expr_ty e) BLOCK finally: if an exception was raised: - exc = copy of (exception, instance, traceback) + exc = copy of (exception, instance, traceback) else: - exc = (None, None, None) + exc = (None, None, None) if not (await exit(*exc)): raise */ @@ -3794,8 +4295,6 @@ compiler_visit_expr(struct compiler *c, expr_ty e) case Yield_kind: if (c->u->u_ste->ste_type != FunctionBlock) return compiler_error(c, "'yield' outside function"); - if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION) - return compiler_error(c, "'yield' inside async function"); if (e->v.Yield.value) { VISIT(c, expr, e->v.Yield.value); } @@ -3820,11 +4319,8 @@ compiler_visit_expr(struct compiler *c, expr_ty e) if (c->u->u_ste->ste_type != FunctionBlock) return compiler_error(c, "'await' outside function"); - if (c->u->u_scope_type == COMPILER_SCOPE_COMPREHENSION) - return compiler_error( - c, "'await' expressions in comprehensions are not supported"); - - if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) + if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && + c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION) return compiler_error(c, "'await' outside async function"); VISIT(c, expr, e->v.Await.value); @@ -3836,12 +4332,19 @@ compiler_visit_expr(struct compiler *c, expr_ty e) return compiler_compare(c, e); case Call_kind: return compiler_call(c, e); + case Constant_kind: + ADDOP_O(c, LOAD_CONST, e->v.Constant.value, consts); + break; case Num_kind: ADDOP_O(c, LOAD_CONST, e->v.Num.n, consts); break; case Str_kind: ADDOP_O(c, LOAD_CONST, e->v.Str.s, consts); break; + case JoinedStr_kind: + return compiler_joined_str(c, e); + case FormattedValue_kind: + return compiler_formatted_value(c, e); case Bytes_kind: ADDOP_O(c, LOAD_CONST, e->v.Bytes.s, consts); break; @@ -3976,6 +4479,144 @@ compiler_augassign(struct compiler *c, stmt_ty s) } static int +check_ann_expr(struct compiler *c, expr_ty e) +{ + VISIT(c, expr, e); + ADDOP(c, POP_TOP); + return 1; +} + +static int +check_annotation(struct compiler *c, stmt_ty s) +{ + /* Annotations are only evaluated in a module or class. */ + if (c->u->u_scope_type == COMPILER_SCOPE_MODULE || + c->u->u_scope_type == COMPILER_SCOPE_CLASS) { + return check_ann_expr(c, s->v.AnnAssign.annotation); + } + return 1; +} + +static int +check_ann_slice(struct compiler *c, slice_ty sl) +{ + switch(sl->kind) { + case Index_kind: + return check_ann_expr(c, sl->v.Index.value); + case Slice_kind: + if (sl->v.Slice.lower && !check_ann_expr(c, sl->v.Slice.lower)) { + return 0; + } + if (sl->v.Slice.upper && !check_ann_expr(c, sl->v.Slice.upper)) { + return 0; + } + if (sl->v.Slice.step && !check_ann_expr(c, sl->v.Slice.step)) { + return 0; + } + break; + default: + PyErr_SetString(PyExc_SystemError, + "unexpected slice kind"); + return 0; + } + return 1; +} + +static int +check_ann_subscr(struct compiler *c, slice_ty sl) +{ + /* We check that everything in a subscript is defined at runtime. */ + Py_ssize_t i, n; + + switch (sl->kind) { + case Index_kind: + case Slice_kind: + if (!check_ann_slice(c, sl)) { + return 0; + } + break; + case ExtSlice_kind: + n = asdl_seq_LEN(sl->v.ExtSlice.dims); + for (i = 0; i < n; i++) { + slice_ty subsl = (slice_ty)asdl_seq_GET(sl->v.ExtSlice.dims, i); + switch (subsl->kind) { + case Index_kind: + case Slice_kind: + if (!check_ann_slice(c, subsl)) { + return 0; + } + break; + case ExtSlice_kind: + default: + PyErr_SetString(PyExc_SystemError, + "extended slice invalid in nested slice"); + return 0; + } + } + break; + default: + PyErr_Format(PyExc_SystemError, + "invalid subscript kind %d", sl->kind); + return 0; + } + return 1; +} + +static int +compiler_annassign(struct compiler *c, stmt_ty s) +{ + expr_ty targ = s->v.AnnAssign.target; + PyObject* mangled; + + assert(s->kind == AnnAssign_kind); + + /* We perform the actual assignment first. */ + if (s->v.AnnAssign.value) { + VISIT(c, expr, s->v.AnnAssign.value); + VISIT(c, expr, targ); + } + switch (targ->kind) { + case Name_kind: + /* If we have a simple name in a module or class, store annotation. */ + if (s->v.AnnAssign.simple && + (c->u->u_scope_type == COMPILER_SCOPE_MODULE || + c->u->u_scope_type == COMPILER_SCOPE_CLASS)) { + mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id); + if (!mangled) { + return 0; + } + VISIT(c, expr, s->v.AnnAssign.annotation); + /* ADDOP_N decrefs its argument */ + ADDOP_N(c, STORE_ANNOTATION, mangled, names); + } + break; + case Attribute_kind: + if (!s->v.AnnAssign.value && + !check_ann_expr(c, targ->v.Attribute.value)) { + return 0; + } + break; + case Subscript_kind: + if (!s->v.AnnAssign.value && + (!check_ann_expr(c, targ->v.Subscript.value) || + !check_ann_subscr(c, targ->v.Subscript.slice))) { + return 0; + } + break; + default: + PyErr_Format(PyExc_SystemError, + "invalid node type (%d) for annotated assignment", + targ->kind); + return 0; + } + /* Annotation is evaluated last. */ + if (!s->v.AnnAssign.simple && !check_annotation(c, s)) { + return 0; + } + return 1; +} + +static int compiler_push_fblock(struct compiler *c, enum fblocktype t, basicblock *b) { struct fblockinfo *f; @@ -4273,7 +4914,7 @@ assemble_init(struct assembler *a, int nblocks, int firstlineno) a->a_lnotab = PyBytes_FromStringAndSize(NULL, DEFAULT_LNOTAB_SIZE); if (!a->a_lnotab) return 0; - if ((size_t)nblocks > PY_SIZE_MAX / sizeof(basicblock *)) { + if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) { PyErr_NoMemory(); return 0; } @@ -4295,18 +4936,6 @@ assemble_free(struct assembler *a) PyObject_Free(a->a_postorder); } -/* Return the size of a basic block in bytes. */ - -static int -instrsize(struct instr *instr) -{ - if (!instr->i_hasarg) - return 1; /* 1 byte for the opcode*/ - if (instr->i_oparg > 0xffff) - return 6; /* 1 (opcode) + 1 (EXTENDED_ARG opcode) + 2 (oparg) + 2(oparg extended) */ - return 3; /* 1 (opcode) + 2 (oparg) */ -} - static int blocksize(basicblock *b) { @@ -4314,7 +4943,7 @@ blocksize(basicblock *b) int size = 0; for (i = 0; i < b->b_iused; i++) - size += instrsize(&b->b_instr[i]); + size += instrsize(b->b_instr[i].i_oparg); return size; } @@ -4329,11 +4958,10 @@ assemble_lnotab(struct assembler *a, struct instr *i) Py_ssize_t len; unsigned char *lnotab; - d_bytecode = a->a_offset - a->a_lineno_off; + d_bytecode = (a->a_offset - a->a_lineno_off) * sizeof(_Py_CODEUNIT); d_lineno = i->i_lineno - a->a_lineno; assert(d_bytecode >= 0); - assert(d_lineno >= 0); if(d_bytecode == 0 && d_lineno == 0) return 1; @@ -4363,9 +4991,21 @@ assemble_lnotab(struct assembler *a, struct instr *i) d_bytecode -= ncodes * 255; a->a_lnotab_off += ncodes * 2; } - assert(d_bytecode <= 255); - if (d_lineno > 255) { - int j, nbytes, ncodes = d_lineno / 255; + assert(0 <= d_bytecode && d_bytecode <= 255); + + if (d_lineno < -128 || 127 < d_lineno) { + int j, nbytes, ncodes, k; + if (d_lineno < 0) { + k = -128; + /* use division on positive numbers */ + ncodes = (-d_lineno) / 128; + } + else { + k = 127; + ncodes = d_lineno / 127; + } + d_lineno -= ncodes * k; + assert(ncodes >= 1); nbytes = a->a_lnotab_off + 2 * ncodes; len = PyBytes_GET_SIZE(a->a_lnotab); if (nbytes >= len) { @@ -4383,15 +5023,15 @@ assemble_lnotab(struct assembler *a, struct instr *i) lnotab = (unsigned char *) PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off; *lnotab++ = d_bytecode; - *lnotab++ = 255; + *lnotab++ = k; d_bytecode = 0; for (j = 1; j < ncodes; j++) { *lnotab++ = 0; - *lnotab++ = 255; + *lnotab++ = k; } - d_lineno -= ncodes * 255; a->a_lnotab_off += ncodes * 2; } + assert(-128 <= d_lineno && d_lineno <= 127); len = PyBytes_GET_SIZE(a->a_lnotab); if (a->a_lnotab_off + 2 >= len) { @@ -4423,38 +5063,23 @@ assemble_lnotab(struct assembler *a, struct instr *i) static int assemble_emit(struct assembler *a, struct instr *i) { - int size, arg = 0, ext = 0; + int size, arg = 0; Py_ssize_t len = PyBytes_GET_SIZE(a->a_bytecode); - char *code; + _Py_CODEUNIT *code; - size = instrsize(i); - if (i->i_hasarg) { - arg = i->i_oparg; - ext = arg >> 16; - } + arg = i->i_oparg; + size = instrsize(arg); if (i->i_lineno && !assemble_lnotab(a, i)) return 0; - if (a->a_offset + size >= len) { + if (a->a_offset + size >= len / (int)sizeof(_Py_CODEUNIT)) { if (len > PY_SSIZE_T_MAX / 2) return 0; if (_PyBytes_Resize(&a->a_bytecode, len * 2) < 0) return 0; } - code = PyBytes_AS_STRING(a->a_bytecode) + a->a_offset; + code = (_Py_CODEUNIT *)PyBytes_AS_STRING(a->a_bytecode) + a->a_offset; a->a_offset += size; - if (size == 6) { - assert(i->i_hasarg); - *code++ = (char)EXTENDED_ARG; - *code++ = ext & 0xff; - *code++ = ext >> 8; - arg &= 0xffff; - } - *code++ = i->i_opcode; - if (i->i_hasarg) { - assert(size == 3 || size == 6); - *code++ = arg & 0xff; - *code++ = arg >> 8; - } + write_op_arg(code, i->i_opcode, arg, size); return 1; } @@ -4462,7 +5087,7 @@ static void assemble_jump_offsets(struct assembler *a, struct compiler *c) { basicblock *b; - int bsize, totsize, extended_arg_count = 0, last_extended_arg_count; + int bsize, totsize, extended_arg_recompile; int i; /* Compute the size of each block and fixup jump args. @@ -4475,27 +5100,27 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c) b->b_offset = totsize; totsize += bsize; } - last_extended_arg_count = extended_arg_count; - extended_arg_count = 0; + extended_arg_recompile = 0; for (b = c->u->u_blocks; b != NULL; b = b->b_list) { bsize = b->b_offset; for (i = 0; i < b->b_iused; i++) { struct instr *instr = &b->b_instr[i]; + int isize = instrsize(instr->i_oparg); /* Relative jumps are computed relative to the instruction pointer after fetching the jump instruction. */ - bsize += instrsize(instr); - if (instr->i_jabs) + bsize += isize; + if (instr->i_jabs || instr->i_jrel) { instr->i_oparg = instr->i_target->b_offset; - else if (instr->i_jrel) { - int delta = instr->i_target->b_offset - bsize; - instr->i_oparg = delta; + if (instr->i_jrel) { + instr->i_oparg -= bsize; + } + instr->i_oparg *= sizeof(_Py_CODEUNIT); + if (instrsize(instr->i_oparg) != isize) { + extended_arg_recompile = 1; + } } - else - continue; - if (instr->i_oparg > 0xffff) - extended_arg_count++; } } @@ -4505,7 +5130,7 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c) The issue is that in the first loop blocksize() is called which calls instrsize() which requires i_oparg be set - appropriately. There is a bootstrap problem because + appropriately. There is a bootstrap problem because i_oparg is calculated in the second loop above. So we loop until we stop seeing new EXTENDED_ARGs. @@ -4513,7 +5138,7 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c) ones in jump instructions. So this should converge fairly quickly. */ - } while (last_extended_arg_count != extended_arg_count); + } while (extended_arg_recompile); } static PyObject * @@ -4549,8 +5174,12 @@ compute_code_flags(struct compiler *c) flags |= CO_NEWLOCALS | CO_OPTIMIZED; if (ste->ste_nested) flags |= CO_NESTED; - if (ste->ste_generator) + if (ste->ste_generator && !ste->ste_coroutine) flags |= CO_GENERATOR; + if (!ste->ste_generator && ste->ste_coroutine) + flags |= CO_COROUTINE; + if (ste->ste_generator && ste->ste_coroutine) + flags |= CO_ASYNC_GENERATOR; if (ste->ste_varargs) flags |= CO_VARARGS; if (ste->ste_varkeywords) @@ -4659,9 +5288,9 @@ dump_instr(const struct instr *i) char arg[128]; *arg = '\0'; - if (i->i_hasarg) + if (HAS_ARG(i->i_opcode)) { sprintf(arg, "arg: %d ", i->i_oparg); - + } fprintf(stderr, "line: %d, opcode: %d %s%s%s\n", i->i_lineno, i->i_opcode, arg, jabs, jrel); } @@ -4711,7 +5340,7 @@ assemble(struct compiler *c, int addNone) /* Set firstlineno if it wasn't explicitly set. */ if (!c->u->u_firstlineno) { - if (entryblock && entryblock->b_instr) + if (entryblock && entryblock->b_instr && entryblock->b_instr->i_lineno) c->u->u_firstlineno = entryblock->b_instr->i_lineno; else c->u->u_firstlineno = 1; @@ -4733,7 +5362,7 @@ assemble(struct compiler *c, int addNone) if (_PyBytes_Resize(&a.a_lnotab, a.a_lnotab_off) < 0) goto error; - if (_PyBytes_Resize(&a.a_bytecode, a.a_offset) < 0) + if (_PyBytes_Resize(&a.a_bytecode, a.a_offset * sizeof(_Py_CODEUNIT)) < 0) goto error; co = makecode(c, &a); @@ -4749,4 +5378,3 @@ PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, { return PyAST_CompileEx(mod, filename, flags, -1, arena); } - |