Issue #29371: merge with 3.5

1 files changed, 1034 insertions, 913 deletions

diff --git a/Python/ceval.c b/Python/ceval.c
index 7b405188d3..d5172b9631 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -15,87 +15,12 @@
 #include "dictobject.h"
 #include "frameobject.h"
 #include "opcode.h"
+#include "pydtrace.h"
 #include "setobject.h"
 #include "structmember.h"
 
 #include <ctype.h>
 
-#ifndef WITH_TSC
-
-#define READ_TIMESTAMP(var)
-
-#else
-
-typedef unsigned long long uint64;
-
-/* PowerPC support.
-   "__ppc__" appears to be the preprocessor definition to detect on OS X, whereas
-   "__powerpc__" appears to be the correct one for Linux with GCC
-*/
-#if defined(__ppc__) || defined (__powerpc__)
-
-#define READ_TIMESTAMP(var) ppc_getcounter(&var)
-
-static void
-ppc_getcounter(uint64 *v)
-{
-    unsigned long tbu, tb, tbu2;
-
-  loop:
-    asm volatile ("mftbu %0" : "=r" (tbu) );
-    asm volatile ("mftb  %0" : "=r" (tb)  );
-    asm volatile ("mftbu %0" : "=r" (tbu2));
-    if (__builtin_expect(tbu != tbu2, 0)) goto loop;
-
-    /* The slightly peculiar way of writing the next lines is
-       compiled better by GCC than any other way I tried. */
-    ((long*)(v))[0] = tbu;
-    ((long*)(v))[1] = tb;
-}
-
-#elif defined(__i386__)
-
-/* this is for linux/x86 (and probably any other GCC/x86 combo) */
-
-#define READ_TIMESTAMP(val) \
-     __asm__ __volatile__("rdtsc" : "=A" (val))
-
-#elif defined(__x86_64__)
-
-/* for gcc/x86_64, the "A" constraint in DI mode means *either* rax *or* rdx;
-   not edx:eax as it does for i386.  Since rdtsc puts its result in edx:eax
-   even in 64-bit mode, we need to use "a" and "d" for the lower and upper
-   32-bit pieces of the result. */
-
-#define READ_TIMESTAMP(val) do {                        \
-    unsigned int h, l;                                  \
-    __asm__ __volatile__("rdtsc" : "=a" (l), "=d" (h)); \
-    (val) = ((uint64)l) | (((uint64)h) << 32);          \
-    } while(0)
-
-
-#else
-
-#error "Don't know how to implement timestamp counter for this architecture"
-
-#endif
-
-void dump_tsc(int opcode, int ticked, uint64 inst0, uint64 inst1,
-              uint64 loop0, uint64 loop1, uint64 intr0, uint64 intr1)
-{
-    uint64 intr, inst, loop;
-    PyThreadState *tstate = PyThreadState_Get();
-    if (!tstate->interp->tscdump)
-        return;
-    intr = intr1 - intr0;
-    inst = inst1 - inst0 - intr;
-    loop = loop1 - loop0 - intr;
-    fprintf(stderr, "opcode=%03d t=%d inst=%06lld loop=%06lld\n",
-            opcode, ticked, inst, loop);
-}
-
-#endif
-
 /* Turn this on if your compiler chokes on the big switch: */
 /* #define CASE_TOO_BIG 1 */
 
@@ -108,24 +33,13 @@ void dump_tsc(int opcode, int ticked, uint64 inst0, uint64 inst1,
 typedef PyObject *(*callproc)(PyObject *, PyObject *, PyObject *);
 
 /* Forward declarations */
-#ifdef WITH_TSC
-static PyObject * call_function(PyObject ***, int, uint64*, uint64*);
-#else
-static PyObject * call_function(PyObject ***, int);
-#endif
-static PyObject * fast_function(PyObject *, PyObject ***, int, int, int);
-static PyObject * do_call(PyObject *, PyObject ***, int, int);
-static PyObject * ext_do_call(PyObject *, PyObject ***, int, int, int);
-static PyObject * update_keyword_args(PyObject *, int, PyObject ***,
-                                      PyObject *);
-static PyObject * update_star_args(int, int, PyObject *, PyObject ***);
-static PyObject * load_args(PyObject ***, int);
-#define CALL_FLAG_VAR 1
-#define CALL_FLAG_KW 2
+static PyObject * call_function(PyObject ***, Py_ssize_t, PyObject *);
+static PyObject * fast_function(PyObject *, PyObject **, Py_ssize_t, PyObject *);
+static PyObject * do_call_core(PyObject *, PyObject *, PyObject *);
 
 #ifdef LLTRACE
 static int lltrace;
-static int prtrace(PyObject *, char *);
+static int prtrace(PyObject *, const char *);
 #endif
 static int call_trace(Py_tracefunc, PyObject *,
                       PyThreadState *, PyFrameObject *,
@@ -137,14 +51,18 @@ static void call_exc_trace(Py_tracefunc, PyObject *,
                            PyThreadState *, PyFrameObject *);
 static int maybe_call_line_trace(Py_tracefunc, PyObject *,
                                  PyThreadState *, PyFrameObject *, int *, int *, int *);
+static void maybe_dtrace_line(PyFrameObject *, int *, int *, int *);
+static void dtrace_function_entry(PyFrameObject *);
+static void dtrace_function_return(PyFrameObject *);
 
 static PyObject * cmp_outcome(int, PyObject *, PyObject *);
+static PyObject * import_name(PyFrameObject *, PyObject *, PyObject *, PyObject *);
 static PyObject * import_from(PyObject *, PyObject *);
 static int import_all_from(PyObject *, PyObject *);
 static void format_exc_check_arg(PyObject *, const char *, PyObject *);
 static void format_exc_unbound(PyCodeObject *co, int oparg);
 static PyObject * unicode_concatenate(PyObject *, PyObject *,
-                                      PyFrameObject *, unsigned char *);
+                                      PyFrameObject *, const _Py_CODEUNIT *);
 static PyObject * special_lookup(PyObject *, _Py_Identifier *);
 
 #define NAME_ERROR_MSG \
@@ -796,11 +714,18 @@ PyEval_EvalFrame(PyFrameObject *f) {
 PyObject *
 PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 {
+    PyThreadState *tstate = PyThreadState_GET();
+    return tstate->interp->eval_frame(f, throwflag);
+}
+
+PyObject *
+_PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
+{
 #ifdef DXPAIRS
     int lastopcode = 0;
 #endif
     PyObject **stack_pointer;  /* Next free slot in value stack */
-    unsigned char *next_instr;
+    const _Py_CODEUNIT *next_instr;
     int opcode;        /* Current opcode */
     int oparg;         /* Current opcode argument, if any */
     enum why_code why; /* Reason for block stack unwind */
@@ -818,7 +743,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
        time it is tested. */
     int instr_ub = -1, instr_lb = 0, instr_prev = -1;
 
-    unsigned char *first_instr;
+    const _Py_CODEUNIT *first_instr;
     PyObject *names;
     PyObject *consts;
 
@@ -886,24 +811,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 /* Import the static jump table */
 #include "opcode_targets.h"
 
-/* This macro is used when several opcodes defer to the same implementation
-   (e.g. SETUP_LOOP, SETUP_FINALLY) */
-#define TARGET_WITH_IMPL(op, impl) \
-    TARGET_##op: \
-        opcode = op; \
-        if (HAS_ARG(op)) \
-            oparg = NEXTARG(); \
-    case op: \
-        goto impl; \
-
 #define TARGET(op) \
     TARGET_##op: \
-        opcode = op; \
-        if (HAS_ARG(op)) \
-            oparg = NEXTARG(); \
     case op:
 
-
 #define DISPATCH() \
     { \
         if (!_Py_atomic_load_relaxed(&eval_breaker)) {      \
@@ -915,18 +826,20 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 #ifdef LLTRACE
 #define FAST_DISPATCH() \
     { \
-        if (!lltrace && !_Py_TracingPossible) { \
+        if (!lltrace && !_Py_TracingPossible && !PyDTrace_LINE_ENABLED()) { \
             f->f_lasti = INSTR_OFFSET(); \
-            goto *opcode_targets[*next_instr++]; \
+            NEXTOPARG(); \
+            goto *opcode_targets[opcode]; \
         } \
         goto fast_next_opcode; \
     }
 #else
 #define FAST_DISPATCH() \
     { \
-        if (!_Py_TracingPossible) { \
+        if (!_Py_TracingPossible && !PyDTrace_LINE_ENABLED()) { \
             f->f_lasti = INSTR_OFFSET(); \
-            goto *opcode_targets[*next_instr++]; \
+            NEXTOPARG(); \
+            goto *opcode_targets[opcode]; \
         } \
         goto fast_next_opcode; \
     }
@@ -935,10 +848,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 #else
 #define TARGET(op) \
     case op:
-#define TARGET_WITH_IMPL(op, impl) \
-    /* silence compiler warnings about `impl` unused */ \
-    if (0) goto impl; \
-    case op:
+
 #define DISPATCH() continue
 #define FAST_DISPATCH() goto fast_next_opcode
 #endif
@@ -952,70 +862,33 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 #define GETITEM(v, i) PyTuple_GetItem((v), (i))
 #endif
 
-#ifdef WITH_TSC
-/* Use Pentium timestamp counter to mark certain events:
-   inst0 -- beginning of switch statement for opcode dispatch
-   inst1 -- end of switch statement (may be skipped)
-   loop0 -- the top of the mainloop
-   loop1 -- place where control returns again to top of mainloop
-            (may be skipped)
-   intr1 -- beginning of long interruption
-   intr2 -- end of long interruption
-
-   Many opcodes call out to helper C functions.  In some cases, the
-   time in those functions should be counted towards the time for the
-   opcode, but not in all cases.  For example, a CALL_FUNCTION opcode
-   calls another Python function; there's no point in charge all the
-   bytecode executed by the called function to the caller.
-
-   It's hard to make a useful judgement statically.  In the presence
-   of operator overloading, it's impossible to tell if a call will
-   execute new Python code or not.
-
-   It's a case-by-case judgement.  I'll use intr1 for the following
-   cases:
-
-   IMPORT_STAR
-   IMPORT_FROM
-   CALL_FUNCTION (and friends)
-
- */
-    uint64 inst0, inst1, loop0, loop1, intr0 = 0, intr1 = 0;
-    int ticked = 0;
-
-    READ_TIMESTAMP(inst0);
-    READ_TIMESTAMP(inst1);
-    READ_TIMESTAMP(loop0);
-    READ_TIMESTAMP(loop1);
-
-    /* shut up the compiler */
-    opcode = 0;
-#endif
-
 /* Code access macros */
 
-#define INSTR_OFFSET()  ((int)(next_instr - first_instr))
-#define NEXTOP()        (*next_instr++)
-#define NEXTARG()       (next_instr += 2, (next_instr[-1]<<8) + next_instr[-2])
-#define PEEKARG()       ((next_instr[2]<<8) + next_instr[1])
-#define JUMPTO(x)       (next_instr = first_instr + (x))
-#define JUMPBY(x)       (next_instr += (x))
+/* The integer overflow is checked by an assertion below. */
+#define INSTR_OFFSET()  (sizeof(_Py_CODEUNIT) * (int)(next_instr - first_instr))
+#define NEXTOPARG()  do { \
+        _Py_CODEUNIT word = *next_instr; \
+        opcode = _Py_OPCODE(word); \
+        oparg = _Py_OPARG(word); \
+        next_instr++; \
+    } while (0)
+#define JUMPTO(x)       (next_instr = first_instr + (x) / sizeof(_Py_CODEUNIT))
+#define JUMPBY(x)       (next_instr += (x) / sizeof(_Py_CODEUNIT))
 
 /* OpCode prediction macros
     Some opcodes tend to come in pairs thus making it possible to
     predict the second code when the first is run.  For example,
-    COMPARE_OP is often followed by JUMP_IF_FALSE or JUMP_IF_TRUE.  And,
-    those opcodes are often followed by a POP_TOP.
+    COMPARE_OP is often followed by POP_JUMP_IF_FALSE or POP_JUMP_IF_TRUE.
 
     Verifying the prediction costs a single high-speed test of a register
     variable against a constant.  If the pairing was good, then the
     processor's own internal branch predication has a high likelihood of
     success, resulting in a nearly zero-overhead transition to the
     next opcode.  A successful prediction saves a trip through the eval-loop
-    including its two unpredictable branches, the HAS_ARG test and the
-    switch-case.  Combined with the processor's internal branch prediction,
-    a successful PREDICT has the effect of making the two opcodes run as if
-    they were a single new opcode with the bodies combined.
+    including its unpredictable switch-case branch.  Combined with the
+    processor's internal branch prediction, a successful PREDICT has the
+    effect of making the two opcodes run as if they were a single new opcode
+    with the bodies combined.
 
     If collecting opcode statistics, your choices are to either keep the
     predictions turned-on and interpret the results as if some opcodes
@@ -1030,13 +903,19 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 
 #if defined(DYNAMIC_EXECUTION_PROFILE) || USE_COMPUTED_GOTOS
 #define PREDICT(op)             if (0) goto PRED_##op
-#define PREDICTED(op)           PRED_##op:
-#define PREDICTED_WITH_ARG(op)  PRED_##op:
 #else
-#define PREDICT(op)             if (*next_instr == op) goto PRED_##op
-#define PREDICTED(op)           PRED_##op: next_instr++
-#define PREDICTED_WITH_ARG(op)  PRED_##op: oparg = PEEKARG(); next_instr += 3
+#define PREDICT(op) \
+    do{ \
+        _Py_CODEUNIT word = *next_instr; \
+        opcode = _Py_OPCODE(word); \
+        if (opcode == op){ \
+            oparg = _Py_OPARG(word); \
+            next_instr++; \
+            goto PRED_##op; \
+        } \
+    } while(0)
 #endif
+#define PREDICTED(op)           PRED_##op:
 
 
 /* Stack manipulation macros */
@@ -1100,7 +979,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
     }
 
 #define UNWIND_EXCEPT_HANDLER(b) \
-    { \
+    do { \
         PyObject *type, *value, *traceback; \
         assert(STACK_LEVEL() >= (b)->b_level + 3); \
         while (STACK_LEVEL() > (b)->b_level + 3) { \
@@ -1116,7 +995,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
         Py_XDECREF(type); \
         Py_XDECREF(value); \
         Py_XDECREF(traceback); \
-    }
+    } while(0)
 
 /* Start of code */
 
@@ -1160,21 +1039,24 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
         }
     }
 
+    if (PyDTrace_FUNCTION_ENTRY_ENABLED())
+        dtrace_function_entry(f);
+
     co = f->f_code;
     names = co->co_names;
     consts = co->co_consts;
     fastlocals = f->f_localsplus;
     freevars = f->f_localsplus + co->co_nlocals;
-    first_instr = (unsigned char*) PyBytes_AS_STRING(co->co_code);
-    /* An explanation is in order for the next line.
-
-       f->f_lasti now refers to the index of the last instruction
-       executed.  You might think this was obvious from the name, but
-       this wasn't always true before 2.3!  PyFrame_New now sets
-       f->f_lasti to -1 (i.e. the index *before* the first instruction)
-       and YIELD_VALUE doesn't fiddle with f_lasti any more.  So this
-       does work.  Promise.
-       YIELD_FROM sets f_lasti to itself, in order to repeated yield
+    assert(PyBytes_Check(co->co_code));
+    assert(PyBytes_GET_SIZE(co->co_code) <= INT_MAX);
+    assert(PyBytes_GET_SIZE(co->co_code) % sizeof(_Py_CODEUNIT) == 0);
+    assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(co->co_code), sizeof(_Py_CODEUNIT)));
+    first_instr = (_Py_CODEUNIT *) PyBytes_AS_STRING(co->co_code);
+    /*
+       f->f_lasti refers to the index of the last instruction,
+       unless it's -1 in which case next_instr should be first_instr.
+
+       YIELD_FROM sets f_lasti to itself, in order to repeatedly yield
        multiple values.
 
        When the PREDICT() macros are enabled, some opcode pairs follow in
@@ -1183,15 +1065,20 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
        were a single new opcode; accordingly,f->f_lasti will point to
        the first code in the pair (for instance, GET_ITER followed by
        FOR_ITER is effectively a single opcode and f->f_lasti will point
-       at to the beginning of the combined pair.)
+       to the beginning of the combined pair.)
     */
-    next_instr = first_instr + f->f_lasti + 1;
+    assert(f->f_lasti >= -1);
+    next_instr = first_instr;
+    if (f->f_lasti >= 0) {
+        assert(f->f_lasti % sizeof(_Py_CODEUNIT) == 0);
+        next_instr += f->f_lasti / sizeof(_Py_CODEUNIT) + 1;
+    }
     stack_pointer = f->f_stacktop;
     assert(stack_pointer != NULL);
     f->f_stacktop = NULL;       /* remains NULL unless yield suspends frame */
     f->f_executing = 1;
 
-    if (co->co_flags & (CO_GENERATOR | CO_COROUTINE)) {
+    if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
         if (!throwflag && f->f_exc_type != NULL && f->f_exc_type != Py_None) {
             /* We were in an except handler when we left,
                restore the exception state which was put aside
@@ -1219,23 +1106,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 #endif
 
     for (;;) {
-#ifdef WITH_TSC
-        if (inst1 == 0) {
-            /* Almost surely, the opcode executed a break
-               or a continue, preventing inst1 from being set
-               on the way out of the loop.
-            */
-            READ_TIMESTAMP(inst1);
-            loop1 = inst1;
-        }
-        dump_tsc(opcode, ticked, inst0, inst1, loop0, loop1,
-                 intr0, intr1);
-        ticked = 0;
-        inst1 = 0;
-        intr0 = 0;
-        intr1 = 0;
-        READ_TIMESTAMP(loop0);
-#endif
         assert(stack_pointer >= f->f_valuestack); /* else underflow */
         assert(STACK_LEVEL() <= co->co_stacksize);  /* else overflow */
         assert(!PyErr_Occurred());
@@ -1249,14 +1119,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
            Py_MakePendingCalls() above. */
 
         if (_Py_atomic_load_relaxed(&eval_breaker)) {
-            if (*next_instr == SETUP_FINALLY) {
+            if (_Py_OPCODE(*next_instr) == SETUP_FINALLY) {
                 /* Make the last opcode before
                    a try: finally: block uninterruptible. */
                 goto fast_next_opcode;
             }
-#ifdef WITH_TSC
-            ticked = 1;
-#endif
             if (_Py_atomic_load_relaxed(&pendingcalls_to_do)) {
                 if (Py_MakePendingCalls() < 0)
                     goto error;
@@ -1296,6 +1163,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
     fast_next_opcode:
         f->f_lasti = INSTR_OFFSET();
 
+        if (PyDTrace_LINE_ENABLED())
+            maybe_dtrace_line(f, &instr_lb, &instr_ub, &instr_prev);
+
         /* line-by-line tracing support */
 
         if (_Py_TracingPossible &&
@@ -1322,11 +1192,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 
         /* Extract opcode and argument */
 
-        opcode = NEXTOP();
-        oparg = 0;   /* allows oparg to be stored in a register because
-            it doesn't have to be remembered across a full loop */
-        if (HAS_ARG(opcode))
-            oparg = NEXTARG();
+        NEXTOPARG();
     dispatch_opcode:
 #ifdef DYNAMIC_EXECUTION_PROFILE
 #ifdef DXPAIRS
@@ -1351,9 +1217,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
         }
 #endif
 
-        /* Main switch on opcode */
-        READ_TIMESTAMP(inst0);
-
         switch (opcode) {
 
         /* BEWARE!
@@ -1377,6 +1240,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             FAST_DISPATCH();
         }
 
+        PREDICTED(LOAD_CONST);
         TARGET(LOAD_CONST) {
             PyObject *value = GETITEM(consts, oparg);
             Py_INCREF(value);
@@ -1384,7 +1248,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             FAST_DISPATCH();
         }
 
-        PREDICTED_WITH_ARG(STORE_FAST);
+        PREDICTED(STORE_FAST);
         TARGET(STORE_FAST) {
             PyObject *value = POP();
             SETLOCAL(oparg, value);
@@ -1863,6 +1727,61 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
+        TARGET(STORE_ANNOTATION) {
+            _Py_IDENTIFIER(__annotations__);
+            PyObject *ann_dict;
+            PyObject *ann = POP();
+            PyObject *name = GETITEM(names, oparg);
+            int err;
+            if (f->f_locals == NULL) {
+                PyErr_Format(PyExc_SystemError,
+                             "no locals found when storing annotation");
+                Py_DECREF(ann);
+                goto error;
+            }
+            /* first try to get __annotations__ from locals... */
+            if (PyDict_CheckExact(f->f_locals)) {
+                ann_dict = _PyDict_GetItemId(f->f_locals,
+                                             &PyId___annotations__);
+                if (ann_dict == NULL) {
+                    PyErr_SetString(PyExc_NameError,
+                                    "__annotations__ not found");
+                    Py_DECREF(ann);
+                    goto error;
+                }
+                Py_INCREF(ann_dict);
+            }
+            else {
+                PyObject *ann_str = _PyUnicode_FromId(&PyId___annotations__);
+                if (ann_str == NULL) {
+                    Py_DECREF(ann);
+                    goto error;
+                }
+                ann_dict = PyObject_GetItem(f->f_locals, ann_str);
+                if (ann_dict == NULL) {
+                    if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+                        PyErr_SetString(PyExc_NameError,
+                                        "__annotations__ not found");
+                    }
+                    Py_DECREF(ann);
+                    goto error;
+                }
+            }
+            /* ...if succeeded, __annotations__[name] = ann */
+            if (PyDict_CheckExact(ann_dict)) {
+                err = PyDict_SetItem(ann_dict, name, ann);
+            }
+            else {
+                err = PyObject_SetItem(ann_dict, name, ann);
+            }
+            Py_DECREF(ann_dict);
+            Py_DECREF(ann);
+            if (err != 0) {
+                goto error;
+            }
+            DISPATCH();
+        }
+
         TARGET(DELETE_SUBSCR) {
             PyObject *sub = TOP();
             PyObject *container = SECOND();
@@ -1992,7 +1911,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
                 Py_DECREF(iter);
 
                 if (PyErr_WarnFormat(
-                        PyExc_PendingDeprecationWarning, 1,
+                        PyExc_DeprecationWarning, 1,
                         "'%.100s' implements legacy __aiter__ protocol; "
                         "__aiter__ should return an asynchronous "
                         "iterator, not awaitable",
@@ -2006,6 +1925,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             }
 
             SET_TOP(awaitable);
+            PREDICT(LOAD_CONST);
             DISPATCH();
         }
 
@@ -2016,41 +1936,52 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             PyObject *aiter = TOP();
             PyTypeObject *type = Py_TYPE(aiter);
 
-            if (type->tp_as_async != NULL)
-                getter = type->tp_as_async->am_anext;
+            if (PyAsyncGen_CheckExact(aiter)) {
+                awaitable = type->tp_as_async->am_anext(aiter);
+                if (awaitable == NULL) {
+                    goto error;
+                }
+            } else {
+                if (type->tp_as_async != NULL){
+                    getter = type->tp_as_async->am_anext;
+                }
 
-            if (getter != NULL) {
-                next_iter = (*getter)(aiter);
-                if (next_iter == NULL) {
+                if (getter != NULL) {
+                    next_iter = (*getter)(aiter);
+                    if (next_iter == NULL) {
+                        goto error;
+                    }
+                }
+                else {
+                    PyErr_Format(
+                        PyExc_TypeError,
+                        "'async for' requires an iterator with "
+                        "__anext__ method, got %.100s",
+                        type->tp_name);
                     goto error;
                 }
-            }
-            else {
-                PyErr_Format(
-                    PyExc_TypeError,
-                    "'async for' requires an iterator with "
-                    "__anext__ method, got %.100s",
-                    type->tp_name);
-                goto error;
-            }
 
-            awaitable = _PyCoro_GetAwaitableIter(next_iter);
-            if (awaitable == NULL) {
-                PyErr_Format(
-                    PyExc_TypeError,
-                    "'async for' received an invalid object "
-                    "from __anext__: %.100s",
-                    Py_TYPE(next_iter)->tp_name);
+                awaitable = _PyCoro_GetAwaitableIter(next_iter);
+                if (awaitable == NULL) {
+                    PyErr_Format(
+                        PyExc_TypeError,
+                        "'async for' received an invalid object "
+                        "from __anext__: %.100s",
+                        Py_TYPE(next_iter)->tp_name);
 
-                Py_DECREF(next_iter);
-                goto error;
-            } else
-                Py_DECREF(next_iter);
+                    Py_DECREF(next_iter);
+                    goto error;
+                } else {
+                    Py_DECREF(next_iter);
+                }
+            }
 
             PUSH(awaitable);
+            PREDICT(LOAD_CONST);
             DISPATCH();
         }
 
+        PREDICTED(GET_AWAITABLE);
         TARGET(GET_AWAITABLE) {
             PyObject *iterable = TOP();
             PyObject *iter = _PyCoro_GetAwaitableIter(iterable);
@@ -2078,21 +2009,22 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
                 goto error;
             }
 
+            PREDICT(LOAD_CONST);
             DISPATCH();
         }
 
         TARGET(YIELD_FROM) {
             PyObject *v = POP();
-            PyObject *reciever = TOP();
+            PyObject *receiver = TOP();
             int err;
-            if (PyGen_CheckExact(reciever) || PyCoro_CheckExact(reciever)) {
-                retval = _PyGen_Send((PyGenObject *)reciever, v);
+            if (PyGen_CheckExact(receiver) || PyCoro_CheckExact(receiver)) {
+                retval = _PyGen_Send((PyGenObject *)receiver, v);
             } else {
                 _Py_IDENTIFIER(send);
                 if (v == Py_None)
-                    retval = Py_TYPE(reciever)->tp_iternext(reciever);
+                    retval = Py_TYPE(receiver)->tp_iternext(receiver);
                 else
-                    retval = _PyObject_CallMethodIdObjArgs(reciever, &PyId_send, v, NULL);
+                    retval = _PyObject_CallMethodIdObjArgs(receiver, &PyId_send, v, NULL);
             }
             Py_DECREF(v);
             if (retval == NULL) {
@@ -2103,7 +2035,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
                 err = _PyGen_FetchStopIterationValue(&val);
                 if (err < 0)
                     goto error;
-                Py_DECREF(reciever);
+                Py_DECREF(receiver);
                 SET_TOP(val);
                 DISPATCH();
             }
@@ -2111,12 +2043,24 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             f->f_stacktop = stack_pointer;
             why = WHY_YIELD;
             /* and repeat... */
-            f->f_lasti--;
+            assert(f->f_lasti >= (int)sizeof(_Py_CODEUNIT));
+            f->f_lasti -= sizeof(_Py_CODEUNIT);
             goto fast_yield;
         }
 
         TARGET(YIELD_VALUE) {
             retval = POP();
+
+            if (co->co_flags & CO_ASYNC_GENERATOR) {
+                PyObject *w = _PyAsyncGenValueWrapperNew(retval);
+                Py_DECREF(retval);
+                if (w == NULL) {
+                    retval = NULL;
+                    goto error;
+                }
+                retval = w;
+            }
+
             f->f_stacktop = stack_pointer;
             why = WHY_YIELD;
             goto fast_yield;
@@ -2133,6 +2077,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
+        PREDICTED(POP_BLOCK);
         TARGET(POP_BLOCK) {
             PyTryBlock *b = PyFrame_BlockPop(f);
             UNWIND_BLOCK(b);
@@ -2249,7 +2194,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
-        PREDICTED_WITH_ARG(UNPACK_SEQUENCE);
+        PREDICTED(UNPACK_SEQUENCE);
         TARGET(UNPACK_SEQUENCE) {
             PyObject *seq = POP(), *item, **items;
             if (PyTuple_CheckExact(seq) &&
@@ -2358,7 +2303,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             }
             else {
                 v = PyObject_GetItem(locals, name);
-                if (v == NULL && _PyErr_OCCURRED()) {
+                if (v == NULL) {
                     if (!PyErr_ExceptionMatches(PyExc_KeyError))
                         goto error;
                     PyErr_Clear();
@@ -2398,26 +2343,33 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             PyObject *name = GETITEM(names, oparg);
             PyObject *v;
             if (PyDict_CheckExact(f->f_globals)
-                && PyDict_CheckExact(f->f_builtins)) {
+                && PyDict_CheckExact(f->f_builtins))
+            {
                 v = _PyDict_LoadGlobal((PyDictObject *)f->f_globals,
                                        (PyDictObject *)f->f_builtins,
                                        name);
                 if (v == NULL) {
-                    if (!_PyErr_OCCURRED())
+                    if (!_PyErr_OCCURRED()) {
+                        /* _PyDict_LoadGlobal() returns NULL without raising
+                         * an exception if the key doesn't exist */
                         format_exc_check_arg(PyExc_NameError,
                                              NAME_ERROR_MSG, name);
+                    }
                     goto error;
                 }
                 Py_INCREF(v);
             }
             else {
                 /* Slow-path if globals or builtins is not a dict */
+
+                /* namespace 1: globals */
                 v = PyObject_GetItem(f->f_globals, name);
                 if (v == NULL) {
                     if (!PyErr_ExceptionMatches(PyExc_KeyError))
                         goto error;
                     PyErr_Clear();
 
+                    /* namespace 2: builtins */
                     v = PyObject_GetItem(f->f_builtins, name);
                     if (v == NULL) {
                         if (PyErr_ExceptionMatches(PyExc_KeyError))
@@ -2477,7 +2429,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             }
             else {
                 value = PyObject_GetItem(locals, name);
-                if (value == NULL && PyErr_Occurred()) {
+                if (value == NULL) {
                     if (!PyErr_ExceptionMatches(PyExc_KeyError))
                         goto error;
                     PyErr_Clear();
@@ -2511,8 +2463,27 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
         TARGET(STORE_DEREF) {
             PyObject *v = POP();
             PyObject *cell = freevars[oparg];
-            PyCell_Set(cell, v);
-            Py_DECREF(v);
+            PyObject *oldobj = PyCell_GET(cell);
+            PyCell_SET(cell, v);
+            Py_XDECREF(oldobj);
+            DISPATCH();
+        }
+
+        TARGET(BUILD_STRING) {
+            PyObject *str;
+            PyObject *empty = PyUnicode_New(0, 0);
+            if (empty == NULL) {
+                goto error;
+            }
+            str = _PyUnicode_JoinArray(empty, stack_pointer - oparg, oparg);
+            Py_DECREF(empty);
+            if (str == NULL)
+                goto error;
+            while (--oparg >= 0) {
+                PyObject *item = POP();
+                Py_DECREF(item);
+            }
+            PUSH(str);
             DISPATCH();
         }
 
@@ -2540,13 +2511,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
-        TARGET_WITH_IMPL(BUILD_TUPLE_UNPACK, _build_list_unpack)
-        TARGET(BUILD_LIST_UNPACK)
-        _build_list_unpack: {
-            int convert_to_tuple = opcode == BUILD_TUPLE_UNPACK;
-            int i;
+        TARGET(BUILD_TUPLE_UNPACK_WITH_CALL)
+        TARGET(BUILD_TUPLE_UNPACK)
+        TARGET(BUILD_LIST_UNPACK) {
+            int convert_to_tuple = opcode != BUILD_LIST_UNPACK;
+            Py_ssize_t i;
             PyObject *sum = PyList_New(0);
             PyObject *return_value;
+
             if (sum == NULL)
                 goto error;
 
@@ -2555,6 +2527,16 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 
                 none_val = _PyList_Extend((PyListObject *)sum, PEEK(i));
                 if (none_val == NULL) {
+                    if (opcode == BUILD_TUPLE_UNPACK_WITH_CALL &&
+                        PyErr_ExceptionMatches(PyExc_TypeError)) {
+                        PyObject *func = PEEK(1 + oparg);
+                        PyErr_Format(PyExc_TypeError,
+                                "%.200s%.200s argument after * "
+                                "must be an iterable, not %.200s",
+                                PyEval_GetFuncName(func),
+                                PyEval_GetFuncDesc(func),
+                                PEEK(i)->ob_type->tp_name);
+                    }
                     Py_DECREF(sum);
                     goto error;
                 }
@@ -2599,7 +2581,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
         }
 
         TARGET(BUILD_SET_UNPACK) {
-            int i;
+            Py_ssize_t i;
             PyObject *sum = PySet_New(NULL);
             if (sum == NULL)
                 goto error;
@@ -2618,7 +2600,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
         }
 
         TARGET(BUILD_MAP) {
-            int i;
+            Py_ssize_t i;
             PyObject *map = _PyDict_NewPresized((Py_ssize_t)oparg);
             if (map == NULL)
                 goto error;
@@ -2641,87 +2623,162 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
-        TARGET_WITH_IMPL(BUILD_MAP_UNPACK_WITH_CALL, _build_map_unpack)
-        TARGET(BUILD_MAP_UNPACK)
-        _build_map_unpack: {
-            int with_call = opcode == BUILD_MAP_UNPACK_WITH_CALL;
-            int num_maps;
-            int i;
-            PyObject *sum = PyDict_New();
-            if (sum == NULL)
+        TARGET(SETUP_ANNOTATIONS) {
+            _Py_IDENTIFIER(__annotations__);
+            int err;
+            PyObject *ann_dict;
+            if (f->f_locals == NULL) {
+                PyErr_Format(PyExc_SystemError,
+                             "no locals found when setting up annotations");
                 goto error;
-            if (with_call) {
-                num_maps = oparg & 0xff;
             }
-            else {
-                num_maps = oparg;
+            /* check if __annotations__ in locals()... */
+            if (PyDict_CheckExact(f->f_locals)) {
+                ann_dict = _PyDict_GetItemId(f->f_locals,
+                                             &PyId___annotations__);
+                if (ann_dict == NULL) {
+                    /* ...if not, create a new one */
+                    ann_dict = PyDict_New();
+                    if (ann_dict == NULL) {
+                        goto error;
+                    }
+                    err = _PyDict_SetItemId(f->f_locals,
+                                            &PyId___annotations__, ann_dict);
+                    Py_DECREF(ann_dict);
+                    if (err != 0) {
+                        goto error;
+                    }
+                }
             }
-
-            for (i = num_maps; i > 0; i--) {
-                PyObject *arg = PEEK(i);
-                if (with_call) {
-                    PyObject *intersection = _PyDictView_Intersect(sum, arg);
-
-                    if (intersection == NULL) {
-                        if (PyErr_ExceptionMatches(PyExc_AttributeError) ||
-                            !PyMapping_Check(arg)) {
-                            int function_location = (oparg>>8) & 0xff;
-                            PyObject *func = (
-                                    PEEK(function_location + num_maps));
-                            PyErr_Format(PyExc_TypeError,
-                                    "%.200s%.200s argument after ** "
-                                    "must be a mapping, not %.200s",
-                                    PyEval_GetFuncName(func),
-                                    PyEval_GetFuncDesc(func),
-                                    arg->ob_type->tp_name);
-                        }
-                        Py_DECREF(sum);
+            else {
+                /* do the same if locals() is not a dict */
+                PyObject *ann_str = _PyUnicode_FromId(&PyId___annotations__);
+                if (ann_str == NULL) {
+                    goto error;
+                }
+                ann_dict = PyObject_GetItem(f->f_locals, ann_str);
+                if (ann_dict == NULL) {
+                    if (!PyErr_ExceptionMatches(PyExc_KeyError)) {
                         goto error;
                     }
-
-                    if (PySet_GET_SIZE(intersection)) {
-                        Py_ssize_t idx = 0;
-                        PyObject *key;
-                        int function_location = (oparg>>8) & 0xff;
-                        PyObject *func = PEEK(function_location + num_maps);
-                        Py_hash_t hash;
-                        _PySet_NextEntry(intersection, &idx, &key, &hash);
-                        if (!PyUnicode_Check(key)) {
-                            PyErr_Format(PyExc_TypeError,
-                                    "%.200s%.200s keywords must be strings",
-                                    PyEval_GetFuncName(func),
-                                    PyEval_GetFuncDesc(func));
-                        } else {
-                            PyErr_Format(PyExc_TypeError,
-                                    "%.200s%.200s got multiple "
-                                    "values for keyword argument '%U'",
-                                    PyEval_GetFuncName(func),
-                                    PyEval_GetFuncDesc(func),
-                                    key);
-                        }
-                        Py_DECREF(intersection);
-                        Py_DECREF(sum);
+                    PyErr_Clear();
+                    ann_dict = PyDict_New();
+                    if (ann_dict == NULL) {
+                        goto error;
+                    }
+                    err = PyObject_SetItem(f->f_locals, ann_str, ann_dict);
+                    Py_DECREF(ann_dict);
+                    if (err != 0) {
                         goto error;
                     }
-                    Py_DECREF(intersection);
                 }
+                else {
+                    Py_DECREF(ann_dict);
+                }
+            }
+            DISPATCH();
+        }
 
+        TARGET(BUILD_CONST_KEY_MAP) {
+            Py_ssize_t i;
+            PyObject *map;
+            PyObject *keys = TOP();
+            if (!PyTuple_CheckExact(keys) ||
+                PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
+                PyErr_SetString(PyExc_SystemError,
+                                "bad BUILD_CONST_KEY_MAP keys argument");
+                goto error;
+            }
+            map = _PyDict_NewPresized((Py_ssize_t)oparg);
+            if (map == NULL) {
+                goto error;
+            }
+            for (i = oparg; i > 0; i--) {
+                int err;
+                PyObject *key = PyTuple_GET_ITEM(keys, oparg - i);
+                PyObject *value = PEEK(i + 1);
+                err = PyDict_SetItem(map, key, value);
+                if (err != 0) {
+                    Py_DECREF(map);
+                    goto error;
+                }
+            }
+
+            Py_DECREF(POP());
+            while (oparg--) {
+                Py_DECREF(POP());
+            }
+            PUSH(map);
+            DISPATCH();
+        }
+
+        TARGET(BUILD_MAP_UNPACK) {
+            Py_ssize_t i;
+            PyObject *sum = PyDict_New();
+            if (sum == NULL)
+                goto error;
+
+            for (i = oparg; i > 0; i--) {
+                PyObject *arg = PEEK(i);
                 if (PyDict_Update(sum, arg) < 0) {
                     if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
-                        if (with_call) {
-                            int function_location = (oparg>>8) & 0xff;
-                            PyObject *func = PEEK(function_location + num_maps);
-                            PyErr_Format(PyExc_TypeError,
-                                    "%.200s%.200s argument after ** "
-                                    "must be a mapping, not %.200s",
-                                    PyEval_GetFuncName(func),
-                                    PyEval_GetFuncDesc(func),
-                                    arg->ob_type->tp_name);
+                        PyErr_Format(PyExc_TypeError,
+                                "'%.200s' object is not a mapping",
+                                arg->ob_type->tp_name);
+                    }
+                    Py_DECREF(sum);
+                    goto error;
+                }
+            }
+
+            while (oparg--)
+                Py_DECREF(POP());
+            PUSH(sum);
+            DISPATCH();
+        }
+
+        TARGET(BUILD_MAP_UNPACK_WITH_CALL) {
+            Py_ssize_t i;
+            PyObject *sum = PyDict_New();
+            if (sum == NULL)
+                goto error;
+
+            for (i = oparg; i > 0; i--) {
+                PyObject *arg = PEEK(i);
+                if (_PyDict_MergeEx(sum, arg, 2) < 0) {
+                    PyObject *func = PEEK(2 + oparg);
+                    if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
+                        PyErr_Format(PyExc_TypeError,
+                                "%.200s%.200s argument after ** "
+                                "must be a mapping, not %.200s",
+                                PyEval_GetFuncName(func),
+                                PyEval_GetFuncDesc(func),
+                                arg->ob_type->tp_name);
+                    }
+                    else if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+                        PyObject *exc, *val, *tb;
+                        PyErr_Fetch(&exc, &val, &tb);
+                        if (val && PyTuple_Check(val) && PyTuple_GET_SIZE(val) == 1) {
+                            PyObject *key = PyTuple_GET_ITEM(val, 0);
+                            if (!PyUnicode_Check(key)) {
+                                PyErr_Format(PyExc_TypeError,
+                                        "%.200s%.200s keywords must be strings",
+                                        PyEval_GetFuncName(func),
+                                        PyEval_GetFuncDesc(func));
+                            } else {
+                                PyErr_Format(PyExc_TypeError,
+                                        "%.200s%.200s got multiple "
+                                        "values for keyword argument '%U'",
+                                        PyEval_GetFuncName(func),
+                                        PyEval_GetFuncDesc(func),
+                                        key);
+                            }
+                            Py_XDECREF(exc);
+                            Py_XDECREF(val);
+                            Py_XDECREF(tb);
                         }
                         else {
-                            PyErr_Format(PyExc_TypeError,
-                                    "'%.200s' object is not a mapping",
-                                    arg->ob_type->tp_name);
+                            PyErr_Restore(exc, val, tb);
                         }
                     }
                     Py_DECREF(sum);
@@ -2729,7 +2786,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
                 }
             }
 
-            while (num_maps--)
+            while (oparg--)
                 Py_DECREF(POP());
             PUSH(sum);
             DISPATCH();
@@ -2778,45 +2835,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
         }
 
         TARGET(IMPORT_NAME) {
-            _Py_IDENTIFIER(__import__);
             PyObject *name = GETITEM(names, oparg);
-            PyObject *func = _PyDict_GetItemId(f->f_builtins, &PyId___import__);
-            PyObject *from, *level, *args, *res;
-            if (func == NULL) {
-                PyErr_SetString(PyExc_ImportError,
-                                "__import__ not found");
-                goto error;
-            }
-            Py_INCREF(func);
-            from = POP();
-            level = TOP();
-            if (PyLong_AsLong(level) != -1 || PyErr_Occurred())
-                args = PyTuple_Pack(5,
-                            name,
-                            f->f_globals,
-                            f->f_locals == NULL ?
-                                  Py_None : f->f_locals,
-                            from,
-                            level);
-            else
-                args = PyTuple_Pack(4,
-                            name,
-                            f->f_globals,
-                            f->f_locals == NULL ?
-                                  Py_None : f->f_locals,
-                            from);
+            PyObject *fromlist = POP();
+            PyObject *level = TOP();
+            PyObject *res;
+            res = import_name(f, name, fromlist, level);
             Py_DECREF(level);
-            Py_DECREF(from);
-            if (args == NULL) {
-                Py_DECREF(func);
-                STACKADJ(-1);
-                goto error;
-            }
-            READ_TIMESTAMP(intr0);
-            res = PyEval_CallObject(func, args);
-            READ_TIMESTAMP(intr1);
-            Py_DECREF(args);
-            Py_DECREF(func);
+            Py_DECREF(fromlist);
             SET_TOP(res);
             if (res == NULL)
                 goto error;
@@ -2835,9 +2860,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
                     "no locals found during 'import *'");
                 goto error;
             }
-            READ_TIMESTAMP(intr0);
             err = import_all_from(locals, from);
-            READ_TIMESTAMP(intr1);
             PyFrame_LocalsToFast(f, 0);
             Py_DECREF(from);
             if (err != 0)
@@ -2849,9 +2872,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             PyObject *name = GETITEM(names, oparg);
             PyObject *from = TOP();
             PyObject *res;
-            READ_TIMESTAMP(intr0);
             res = import_from(from, name);
-            READ_TIMESTAMP(intr1);
             PUSH(res);
             if (res == NULL)
                 goto error;
@@ -2863,7 +2884,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             FAST_DISPATCH();
         }
 
-        PREDICTED_WITH_ARG(POP_JUMP_IF_FALSE);
+        PREDICTED(POP_JUMP_IF_FALSE);
         TARGET(POP_JUMP_IF_FALSE) {
             PyObject *cond = POP();
             int err;
@@ -2887,7 +2908,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
-        PREDICTED_WITH_ARG(POP_JUMP_IF_TRUE);
+        PREDICTED(POP_JUMP_IF_TRUE);
         TARGET(POP_JUMP_IF_TRUE) {
             PyObject *cond = POP();
             int err;
@@ -2964,7 +2985,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
-        PREDICTED_WITH_ARG(JUMP_ABSOLUTE);
+        PREDICTED(JUMP_ABSOLUTE);
         TARGET(JUMP_ABSOLUTE) {
             JUMPTO(oparg);
 #if FAST_LOOPS
@@ -2990,6 +3011,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             if (iter == NULL)
                 goto error;
             PREDICT(FOR_ITER);
+            PREDICT(CALL_FUNCTION);
             DISPATCH();
         }
 
@@ -3018,10 +3040,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
                 if (iter == NULL)
                     goto error;
             }
+            PREDICT(LOAD_CONST);
             DISPATCH();
         }
 
-        PREDICTED_WITH_ARG(FOR_ITER);
+        PREDICTED(FOR_ITER);
         TARGET(FOR_ITER) {
             /* before: [iter]; after: [iter, iter()] *or* [] */
             PyObject *iter = TOP();
@@ -3043,6 +3066,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             STACKADJ(-1);
             Py_DECREF(iter);
             JUMPBY(oparg);
+            PREDICT(POP_BLOCK);
             DISPATCH();
         }
 
@@ -3059,10 +3083,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             goto fast_block_end;
         }
 
-        TARGET_WITH_IMPL(SETUP_LOOP, _setup_finally)
-        TARGET_WITH_IMPL(SETUP_EXCEPT, _setup_finally)
-        TARGET(SETUP_FINALLY)
-        _setup_finally: {
+        TARGET(SETUP_LOOP)
+        TARGET(SETUP_EXCEPT)
+        TARGET(SETUP_FINALLY) {
             /* NOTE: If you add any new block-setup opcodes that
                are not try/except/finally handlers, you may need
                to update the PyGen_NeedsFinalizing() function.
@@ -3093,6 +3116,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             if (res == NULL)
                 goto error;
             PUSH(res);
+            PREDICT(GET_AWAITABLE);
             DISPATCH();
         }
 
@@ -3110,15 +3134,17 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             _Py_IDENTIFIER(__exit__);
             _Py_IDENTIFIER(__enter__);
             PyObject *mgr = TOP();
-            PyObject *exit = special_lookup(mgr, &PyId___exit__), *enter;
+            PyObject *enter = special_lookup(mgr, &PyId___enter__), *exit;
             PyObject *res;
-            if (exit == NULL)
+            if (enter == NULL)
                 goto error;
+            exit = special_lookup(mgr, &PyId___exit__);
+            if (exit == NULL) {
+                Py_DECREF(enter);
+                goto error;
+            }
             SET_TOP(exit);
-            enter = special_lookup(mgr, &PyId___enter__);
             Py_DECREF(mgr);
-            if (enter == NULL)
-                goto error;
             res = PyObject_CallFunctionObjArgs(enter, NULL);
             Py_DECREF(enter);
             if (res == NULL)
@@ -3241,178 +3267,135 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
+        PREDICTED(CALL_FUNCTION);
         TARGET(CALL_FUNCTION) {
             PyObject **sp, *res;
             PCALL(PCALL_ALL);
             sp = stack_pointer;
-#ifdef WITH_TSC
-            res = call_function(&sp, oparg, &intr0, &intr1);
-#else
-            res = call_function(&sp, oparg);
-#endif
+            res = call_function(&sp, oparg, NULL);
             stack_pointer = sp;
             PUSH(res);
-            if (res == NULL)
+            if (res == NULL) {
                 goto error;
+            }
             DISPATCH();
         }
 
-        TARGET_WITH_IMPL(CALL_FUNCTION_VAR, _call_function_var_kw)
-        TARGET_WITH_IMPL(CALL_FUNCTION_KW, _call_function_var_kw)
-        TARGET(CALL_FUNCTION_VAR_KW)
-        _call_function_var_kw: {
-            int na = oparg & 0xff;
-            int nk = (oparg>>8) & 0xff;
-            int flags = (opcode - CALL_FUNCTION) & 3;
-            int n = na + 2 * nk;
-            PyObject **pfunc, *func, **sp, *res;
+        TARGET(CALL_FUNCTION_KW) {
+            PyObject **sp, *res, *names;
+
+            names = POP();
+            assert(PyTuple_CheckExact(names) && PyTuple_GET_SIZE(names) <= oparg);
             PCALL(PCALL_ALL);
-            if (flags & CALL_FLAG_VAR)
-                n++;
-            if (flags & CALL_FLAG_KW)
-                n++;
-            pfunc = stack_pointer - n - 1;
-            func = *pfunc;
-
-            if (PyMethod_Check(func)
-                && PyMethod_GET_SELF(func) != NULL) {
-                PyObject *self = PyMethod_GET_SELF(func);
-                Py_INCREF(self);
-                func = PyMethod_GET_FUNCTION(func);
-                Py_INCREF(func);
-                Py_SETREF(*pfunc, self);
-                na++;
-                /* n++; */
-            } else
-                Py_INCREF(func);
             sp = stack_pointer;
-            READ_TIMESTAMP(intr0);
-            res = ext_do_call(func, &sp, flags, na, nk);
-            READ_TIMESTAMP(intr1);
+            res = call_function(&sp, oparg, names);
             stack_pointer = sp;
-            Py_DECREF(func);
-
-            while (stack_pointer > pfunc) {
-                PyObject *o = POP();
-                Py_DECREF(o);
-            }
             PUSH(res);
-            if (res == NULL)
-                goto error;
-            DISPATCH();
-        }
+            Py_DECREF(names);
 
-        TARGET_WITH_IMPL(MAKE_CLOSURE, _make_function)
-        TARGET(MAKE_FUNCTION)
-        _make_function: {
-            int posdefaults = oparg & 0xff;
-            int kwdefaults = (oparg>>8) & 0xff;
-            int num_annotations = (oparg >> 16) & 0x7fff;
-
-            PyObject *qualname = POP(); /* qualname */
-            PyObject *code = POP(); /* code object */
-            PyObject *func = PyFunction_NewWithQualName(code, f->f_globals, qualname);
-            Py_DECREF(code);
-            Py_DECREF(qualname);
-
-            if (func == NULL)
+            if (res == NULL) {
                 goto error;
-
-            if (opcode == MAKE_CLOSURE) {
-                PyObject *closure = POP();
-                if (PyFunction_SetClosure(func, closure) != 0) {
-                    /* Can't happen unless bytecode is corrupt. */
-                    Py_DECREF(func);
-                    Py_DECREF(closure);
-                    goto error;
-                }
-                Py_DECREF(closure);
             }
+            DISPATCH();
+        }
 
-            if (num_annotations > 0) {
-                Py_ssize_t name_ix;
-                PyObject *names = POP(); /* names of args with annotations */
-                PyObject *anns = PyDict_New();
-                if (anns == NULL) {
-                    Py_DECREF(func);
-                    Py_DECREF(names);
-                    goto error;
-                }
-                name_ix = PyTuple_Size(names);
-                assert(num_annotations == name_ix+1);
-                while (name_ix > 0) {
-                    PyObject *name, *value;
-                    int err;
-                    --name_ix;
-                    name = PyTuple_GET_ITEM(names, name_ix);
-                    value = POP();
-                    err = PyDict_SetItem(anns, name, value);
-                    Py_DECREF(value);
-                    if (err != 0) {
-                        Py_DECREF(anns);
-                        Py_DECREF(func);
-                        Py_DECREF(names);
+        TARGET(CALL_FUNCTION_EX) {
+            PyObject *func, *callargs, *kwargs = NULL, *result;
+            PCALL(PCALL_ALL);
+            if (oparg & 0x01) {
+                kwargs = POP();
+                if (!PyDict_CheckExact(kwargs)) {
+                    PyObject *d = PyDict_New();
+                    if (d == NULL)
                         goto error;
-                    }
-                }
-                Py_DECREF(names);
-
-                if (PyFunction_SetAnnotations(func, anns) != 0) {
-                    /* Can't happen unless
-                       PyFunction_SetAnnotations changes. */
-                    Py_DECREF(anns);
-                    Py_DECREF(func);
-                    goto error;
-                }
-                Py_DECREF(anns);
-            }
-
-            /* XXX Maybe this should be a separate opcode? */
-            if (kwdefaults > 0) {
-                PyObject *defs = PyDict_New();
-                if (defs == NULL) {
-                    Py_DECREF(func);
-                    goto error;
-                }
-                while (--kwdefaults >= 0) {
-                    PyObject *v = POP(); /* default value */
-                    PyObject *key = POP(); /* kw only arg name */
-                    int err = PyDict_SetItem(defs, key, v);
-                    Py_DECREF(v);
-                    Py_DECREF(key);
-                    if (err != 0) {
-                        Py_DECREF(defs);
-                        Py_DECREF(func);
+                    if (PyDict_Update(d, kwargs) != 0) {
+                        Py_DECREF(d);
+                        /* PyDict_Update raises attribute
+                         * error (percolated from an attempt
+                         * to get 'keys' attribute) instead of
+                         * a type error if its second argument
+                         * is not a mapping.
+                         */
+                        if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
+                            func = SECOND();
+                            PyErr_Format(PyExc_TypeError,
+                                         "%.200s%.200s argument after ** "
+                                         "must be a mapping, not %.200s",
+                                         PyEval_GetFuncName(func),
+                                         PyEval_GetFuncDesc(func),
+                                         kwargs->ob_type->tp_name);
+                        }
+                        Py_DECREF(kwargs);
                         goto error;
                     }
+                    Py_DECREF(kwargs);
+                    kwargs = d;
                 }
-                if (PyFunction_SetKwDefaults(func, defs) != 0) {
-                    /* Can't happen unless
-                       PyFunction_SetKwDefaults changes. */
-                    Py_DECREF(func);
-                    Py_DECREF(defs);
-                    goto error;
-                }
-                Py_DECREF(defs);
+                assert(PyDict_CheckExact(kwargs));
             }
-            if (posdefaults > 0) {
-                PyObject *defs = PyTuple_New(posdefaults);
-                if (defs == NULL) {
-                    Py_DECREF(func);
+            callargs = POP();
+            func = TOP();
+            if (!PyTuple_CheckExact(callargs)) {
+                if (Py_TYPE(callargs)->tp_iter == NULL &&
+                        !PySequence_Check(callargs)) {
+                    PyErr_Format(PyExc_TypeError,
+                                 "%.200s%.200s argument after * "
+                                 "must be an iterable, not %.200s",
+                                 PyEval_GetFuncName(func),
+                                 PyEval_GetFuncDesc(func),
+                                 callargs->ob_type->tp_name);
+                    Py_DECREF(callargs);
                     goto error;
                 }
-                while (--posdefaults >= 0)
-                    PyTuple_SET_ITEM(defs, posdefaults, POP());
-                if (PyFunction_SetDefaults(func, defs) != 0) {
-                    /* Can't happen unless
-                       PyFunction_SetDefaults changes. */
-                    Py_DECREF(defs);
-                    Py_DECREF(func);
+                Py_SETREF(callargs, PySequence_Tuple(callargs));
+                if (callargs == NULL) {
                     goto error;
                 }
-                Py_DECREF(defs);
             }
-            PUSH(func);
+            assert(PyTuple_CheckExact(callargs));
+
+            result = do_call_core(func, callargs, kwargs);
+            Py_DECREF(func);
+            Py_DECREF(callargs);
+            Py_XDECREF(kwargs);
+
+            SET_TOP(result);
+            if (result == NULL) {
+                goto error;
+            }
+            DISPATCH();
+        }
+
+        TARGET(MAKE_FUNCTION) {
+            PyObject *qualname = POP();
+            PyObject *codeobj = POP();
+            PyFunctionObject *func = (PyFunctionObject *)
+                PyFunction_NewWithQualName(codeobj, f->f_globals, qualname);
+
+            Py_DECREF(codeobj);
+            Py_DECREF(qualname);
+            if (func == NULL) {
+                goto error;
+            }
+
+            if (oparg & 0x08) {
+                assert(PyTuple_CheckExact(TOP()));
+                func ->func_closure = POP();
+            }
+            if (oparg & 0x04) {
+                assert(PyDict_CheckExact(TOP()));
+                func->func_annotations = POP();
+            }
+            if (oparg & 0x02) {
+                assert(PyDict_CheckExact(TOP()));
+                func->func_kwdefaults = POP();
+            }
+            if (oparg & 0x01) {
+                assert(PyTuple_CheckExact(TOP()));
+                func->func_defaults = POP();
+            }
+
+            PUSH((PyObject *)func);
             DISPATCH();
         }
 
@@ -3434,9 +3417,68 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
+        TARGET(FORMAT_VALUE) {
+            /* Handles f-string value formatting. */
+            PyObject *result;
+            PyObject *fmt_spec;
+            PyObject *value;
+            PyObject *(*conv_fn)(PyObject *);
+            int which_conversion = oparg & FVC_MASK;
+            int have_fmt_spec = (oparg & FVS_MASK) == FVS_HAVE_SPEC;
+
+            fmt_spec = have_fmt_spec ? POP() : NULL;
+            value = POP();
+
+            /* See if any conversion is specified. */
+            switch (which_conversion) {
+            case FVC_STR:   conv_fn = PyObject_Str;   break;
+            case FVC_REPR:  conv_fn = PyObject_Repr;  break;
+            case FVC_ASCII: conv_fn = PyObject_ASCII; break;
+
+            /* Must be 0 (meaning no conversion), since only four
+               values are allowed by (oparg & FVC_MASK). */
+            default:        conv_fn = NULL;           break;
+            }
+
+            /* If there's a conversion function, call it and replace
+               value with that result. Otherwise, just use value,
+               without conversion. */
+            if (conv_fn != NULL) {
+                result = conv_fn(value);
+                Py_DECREF(value);
+                if (result == NULL) {
+                    Py_XDECREF(fmt_spec);
+                    goto error;
+                }
+                value = result;
+            }
+
+            /* If value is a unicode object, and there's no fmt_spec,
+               then we know the result of format(value) is value
+               itself. In that case, skip calling format(). I plan to
+               move this optimization in to PyObject_Format()
+               itself. */
+            if (PyUnicode_CheckExact(value) && fmt_spec == NULL) {
+                /* Do nothing, just transfer ownership to result. */
+                result = value;
+            } else {
+                /* Actually call format(). */
+                result = PyObject_Format(value, fmt_spec);
+                Py_DECREF(value);
+                Py_XDECREF(fmt_spec);
+                if (result == NULL) {
+                    goto error;
+                }
+            }
+
+            PUSH(result);
+            DISPATCH();
+        }
+
         TARGET(EXTENDED_ARG) {
-            opcode = NEXTOP();
-            oparg = oparg<<16 | NEXTARG();
+            int oldoparg = oparg;
+            NEXTOPARG();
+            oparg |= oldoparg << 8;
             goto dispatch_opcode;
         }
 
@@ -3463,7 +3505,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
         assert(0);
 
 error:
-        READ_TIMESTAMP(inst1);
 
         assert(why == WHY_NOT);
         why = WHY_EXCEPTION;
@@ -3567,7 +3608,6 @@ fast_block_end:
 
         if (why != WHY_NOT)
             break;
-        READ_TIMESTAMP(loop1);
 
         assert(!PyErr_Occurred());
 
@@ -3586,7 +3626,7 @@ fast_block_end:
     assert((retval != NULL) ^ (PyErr_Occurred() != NULL));
 
 fast_yield:
-    if (co->co_flags & (CO_GENERATOR | CO_COROUTINE)) {
+    if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
 
         /* The purpose of this block is to put aside the generator's exception
            state and restore that of the calling frame. If the current
@@ -3597,9 +3637,11 @@ fast_yield:
            state came into existence in this frame. (An uncaught exception
            would have why == WHY_EXCEPTION, and we wouldn't be here). */
         int i;
-        for (i = 0; i < f->f_iblock; i++)
-            if (f->f_blockstack[i].b_type == EXCEPT_HANDLER)
+        for (i = 0; i < f->f_iblock; i++) {
+            if (f->f_blockstack[i].b_type == EXCEPT_HANDLER) {
                 break;
+            }
+        }
         if (i == f->f_iblock)
             /* We did not create this exception. */
             restore_and_clear_exc_state(tstate, f);
@@ -3640,6 +3682,8 @@ fast_yield:
 
     /* pop frame */
 exit_eval_frame:
+    if (PyDTrace_FUNCTION_RETURN_ENABLED())
+        dtrace_function_return(f);
     Py_LeaveRecursiveCall();
     f->f_executing = 0;
     tstate->frame = f->f_back;
@@ -3710,12 +3754,12 @@ format_missing(const char *kind, PyCodeObject *co, PyObject *names)
 }
 
 static void
-missing_arguments(PyCodeObject *co, int missing, int defcount,
+missing_arguments(PyCodeObject *co, Py_ssize_t missing, Py_ssize_t defcount,
                   PyObject **fastlocals)
 {
-    int i, j = 0;
-    int start, end;
-    int positional = defcount != -1;
+    Py_ssize_t i, j = 0;
+    Py_ssize_t start, end;
+    int positional = (defcount != -1);
     const char *kind = positional ? "positional" : "keyword-only";
     PyObject *missing_names;
 
@@ -3748,33 +3792,39 @@ missing_arguments(PyCodeObject *co, int missing, int defcount,
 }
 
 static void
-too_many_positional(PyCodeObject *co, int given, int defcount, PyObject **fastlocals)
+too_many_positional(PyCodeObject *co, Py_ssize_t given, Py_ssize_t defcount,
+                    PyObject **fastlocals)
 {
     int plural;
-    int kwonly_given = 0;
-    int i;
+    Py_ssize_t kwonly_given = 0;
+    Py_ssize_t i;
     PyObject *sig, *kwonly_sig;
+    Py_ssize_t co_argcount = co->co_argcount;
 
     assert((co->co_flags & CO_VARARGS) == 0);
     /* Count missing keyword-only args. */
-    for (i = co->co_argcount; i < co->co_argcount + co->co_kwonlyargcount; i++)
-        if (GETLOCAL(i) != NULL)
+    for (i = co_argcount; i < co_argcount + co->co_kwonlyargcount; i++) {
+        if (GETLOCAL(i) != NULL) {
             kwonly_given++;
+        }
+    }
     if (defcount) {
-        int atleast = co->co_argcount - defcount;
+        Py_ssize_t atleast = co_argcount - defcount;
         plural = 1;
-        sig = PyUnicode_FromFormat("from %d to %d", atleast, co->co_argcount);
+        sig = PyUnicode_FromFormat("from %zd to %zd", atleast, co_argcount);
     }
     else {
-        plural = co->co_argcount != 1;
-        sig = PyUnicode_FromFormat("%d", co->co_argcount);
+        plural = (co_argcount != 1);
+        sig = PyUnicode_FromFormat("%zd", co_argcount);
     }
     if (sig == NULL)
         return;
     if (kwonly_given) {
-        const char *format = " positional argument%s (and %d keyword-only argument%s)";
-        kwonly_sig = PyUnicode_FromFormat(format, given != 1 ? "s" : "", kwonly_given,
-                                              kwonly_given != 1 ? "s" : "");
+        const char *format = " positional argument%s (and %zd keyword-only argument%s)";
+        kwonly_sig = PyUnicode_FromFormat(format,
+                                          given != 1 ? "s" : "",
+                                          kwonly_given,
+                                          kwonly_given != 1 ? "s" : "");
         if (kwonly_sig == NULL) {
             Py_DECREF(sig);
             return;
@@ -3786,7 +3836,7 @@ too_many_positional(PyCodeObject *co, int given, int defcount, PyObject **fastlo
         assert(kwonly_sig != NULL);
     }
     PyErr_Format(PyExc_TypeError,
-                 "%U() takes %U positional argument%s but %d%U %s given",
+                 "%U() takes %U positional argument%s but %zd%U %s given",
                  co->co_name,
                  sig,
                  plural ? "s" : "",
@@ -3803,20 +3853,22 @@ too_many_positional(PyCodeObject *co, int given, int defcount, PyObject **fastlo
 
 static PyObject *
 _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals,
-           PyObject **args, int argcount, PyObject **kws, int kwcount,
-           PyObject **defs, int defcount, PyObject *kwdefs, PyObject *closure,
+           PyObject **args, Py_ssize_t argcount,
+           PyObject **kwnames, PyObject **kwargs,
+           Py_ssize_t kwcount, int kwstep,
+           PyObject **defs, Py_ssize_t defcount,
+           PyObject *kwdefs, PyObject *closure,
            PyObject *name, PyObject *qualname)
 {
     PyCodeObject* co = (PyCodeObject*)_co;
     PyFrameObject *f;
     PyObject *retval = NULL;
     PyObject **fastlocals, **freevars;
-    PyThreadState *tstate = PyThreadState_GET();
+    PyThreadState *tstate;
     PyObject *x, *u;
-    int total_args = co->co_argcount + co->co_kwonlyargcount;
-    int i;
-    int n = argcount;
-    PyObject *kwdict = NULL;
+    const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount;
+    Py_ssize_t i, n;
+    PyObject *kwdict;
 
     if (globals == NULL) {
         PyErr_SetString(PyExc_SystemError,
@@ -3824,36 +3876,50 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals,
         return NULL;
     }
 
+    /* Create the frame */
+    tstate = PyThreadState_GET();
     assert(tstate != NULL);
-    assert(globals != NULL);
     f = PyFrame_New(tstate, co, globals, locals);
-    if (f == NULL)
+    if (f == NULL) {
         return NULL;
-
+    }
     fastlocals = f->f_localsplus;
     freevars = f->f_localsplus + co->co_nlocals;
 
-    /* Parse arguments. */
+    /* Create a dictionary for keyword parameters (**kwags) */
     if (co->co_flags & CO_VARKEYWORDS) {
         kwdict = PyDict_New();
         if (kwdict == NULL)
             goto fail;
         i = total_args;
-        if (co->co_flags & CO_VARARGS)
+        if (co->co_flags & CO_VARARGS) {
             i++;
+        }
         SETLOCAL(i, kwdict);
     }
-    if (argcount > co->co_argcount)
+    else {
+        kwdict = NULL;
+    }
+
+    /* Copy positional arguments into local variables */
+    if (argcount > co->co_argcount) {
         n = co->co_argcount;
+    }
+    else {
+        n = argcount;
+    }
     for (i = 0; i < n; i++) {
         x = args[i];
         Py_INCREF(x);
         SETLOCAL(i, x);
     }
+
+    /* Pack other positional arguments into the *args argument */
     if (co->co_flags & CO_VARARGS) {
         u = PyTuple_New(argcount - n);
-        if (u == NULL)
+        if (u == NULL) {
             goto fail;
+        }
         SETLOCAL(total_args, u);
         for (i = n; i < argcount; i++) {
             x = args[i];
@@ -3861,69 +3927,82 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals,
             PyTuple_SET_ITEM(u, i-n, x);
         }
     }
-    for (i = 0; i < kwcount; i++) {
+
+    /* Handle keyword arguments passed as two strided arrays */
+    kwcount *= kwstep;
+    for (i = 0; i < kwcount; i += kwstep) {
         PyObject **co_varnames;
-        PyObject *keyword = kws[2*i];
-        PyObject *value = kws[2*i + 1];
-        int j;
+        PyObject *keyword = kwnames[i];
+        PyObject *value = kwargs[i];
+        Py_ssize_t j;
+
         if (keyword == NULL || !PyUnicode_Check(keyword)) {
             PyErr_Format(PyExc_TypeError,
                          "%U() keywords must be strings",
                          co->co_name);
             goto fail;
         }
+
         /* Speed hack: do raw pointer compares. As names are
            normally interned this should almost always hit. */
         co_varnames = ((PyTupleObject *)(co->co_varnames))->ob_item;
         for (j = 0; j < total_args; j++) {
-            PyObject *nm = co_varnames[j];
-            if (nm == keyword)
+            PyObject *name = co_varnames[j];
+            if (name == keyword) {
                 goto kw_found;
+            }
         }
+
         /* Slow fallback, just in case */
         for (j = 0; j < total_args; j++) {
-            PyObject *nm = co_varnames[j];
-            int cmp = PyObject_RichCompareBool(
-                keyword, nm, Py_EQ);
-            if (cmp > 0)
+            PyObject *name = co_varnames[j];
+            int cmp = PyObject_RichCompareBool( keyword, name, Py_EQ);
+            if (cmp > 0) {
                 goto kw_found;
-            else if (cmp < 0)
+            }
+            else if (cmp < 0) {
                 goto fail;
+            }
         }
+
         if (j >= total_args && kwdict == NULL) {
             PyErr_Format(PyExc_TypeError,
-                         "%U() got an unexpected "
-                         "keyword argument '%S'",
-                         co->co_name,
-                         keyword);
+                         "%U() got an unexpected keyword argument '%S'",
+                         co->co_name, keyword);
             goto fail;
         }
+
         if (PyDict_SetItem(kwdict, keyword, value) == -1) {
             goto fail;
         }
         continue;
+
       kw_found:
         if (GETLOCAL(j) != NULL) {
             PyErr_Format(PyExc_TypeError,
-                         "%U() got multiple "
-                         "values for argument '%S'",
-                         co->co_name,
-                         keyword);
+                         "%U() got multiple values for argument '%S'",
+                         co->co_name, keyword);
             goto fail;
         }
         Py_INCREF(value);
         SETLOCAL(j, value);
     }
+
+    /* Check the number of positional arguments */
     if (argcount > co->co_argcount && !(co->co_flags & CO_VARARGS)) {
         too_many_positional(co, argcount, defcount, fastlocals);
         goto fail;
     }
+
+    /* Add missing positional arguments (copy default values from defs) */
     if (argcount < co->co_argcount) {
-        int m = co->co_argcount - defcount;
-        int missing = 0;
-        for (i = argcount; i < m; i++)
-            if (GETLOCAL(i) == NULL)
+        Py_ssize_t m = co->co_argcount - defcount;
+        Py_ssize_t missing = 0;
+        for (i = argcount; i < m; i++) {
+            if (GETLOCAL(i) == NULL) {
                 missing++;
+            }
+        }
         if (missing) {
             missing_arguments(co, missing, defcount, fastlocals);
             goto fail;
@@ -3940,8 +4019,10 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals,
             }
         }
     }
+
+    /* Add missing keyword arguments (copy default values from kwdefs) */
     if (co->co_kwonlyargcount > 0) {
-        int missing = 0;
+        Py_ssize_t missing = 0;
         for (i = co->co_argcount; i < total_args; i++) {
             PyObject *name;
             if (GETLOCAL(i) != NULL)
@@ -3982,13 +4063,16 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals,
             goto fail;
         SETLOCAL(co->co_nlocals + i, c);
     }
+
+    /* Copy closure variables to free variables */
     for (i = 0; i < PyTuple_GET_SIZE(co->co_freevars); ++i) {
         PyObject *o = PyTuple_GET_ITEM(closure, i);
         Py_INCREF(o);
         freevars[PyTuple_GET_SIZE(co->co_cellvars) + i] = o;
     }
 
-    if (co->co_flags & (CO_GENERATOR | CO_COROUTINE)) {
+    /* Handle generator/coroutine/asynchronous generator */
+    if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
         PyObject *gen;
         PyObject *coro_wrapper = tstate->coroutine_wrapper;
         int is_coro = co->co_flags & CO_COROUTINE;
@@ -4013,6 +4097,8 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals,
          * and return that as the value. */
         if (is_coro) {
             gen = PyCoro_New(f, name, qualname);
+        } else if (co->co_flags & CO_ASYNC_GENERATOR) {
+            gen = PyAsyncGen_New(f, name, qualname);
         } else {
             gen = PyGen_NewWithQualName(f, name, qualname);
         }
@@ -4052,8 +4138,10 @@ PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals,
            PyObject **defs, int defcount, PyObject *kwdefs, PyObject *closure)
 {
     return _PyEval_EvalCodeWithName(_co, globals, locals,
-                                    args, argcount, kws, kwcount,
-                                    defs, defcount, kwdefs, closure,
+                                    args, argcount,
+                                    kws, kws + 1, kwcount, 2,
+                                    defs, defcount,
+                                    kwdefs, closure,
                                     NULL, NULL);
 }
 
@@ -4312,7 +4400,7 @@ Error:
 
 #ifdef LLTRACE
 static int
-prtrace(PyObject *v, char *str)
+prtrace(PyObject *v, const char *str)
 {
     printf("%s ", str);
     if (PyObject_Print(v, stdout, 0) != 0)
@@ -4489,6 +4577,38 @@ _PyEval_GetCoroutineWrapper(void)
     return tstate->coroutine_wrapper;
 }
 
+void
+_PyEval_SetAsyncGenFirstiter(PyObject *firstiter)
+{
+    PyThreadState *tstate = PyThreadState_GET();
+
+    Py_XINCREF(firstiter);
+    Py_XSETREF(tstate->async_gen_firstiter, firstiter);
+}
+
+PyObject *
+_PyEval_GetAsyncGenFirstiter(void)
+{
+    PyThreadState *tstate = PyThreadState_GET();
+    return tstate->async_gen_firstiter;
+}
+
+void
+_PyEval_SetAsyncGenFinalizer(PyObject *finalizer)
+{
+    PyThreadState *tstate = PyThreadState_GET();
+
+    Py_XINCREF(finalizer);
+    Py_XSETREF(tstate->async_gen_finalizer, finalizer);
+}
+
+PyObject *
+_PyEval_GetAsyncGenFinalizer(void)
+{
+    PyThreadState *tstate = PyThreadState_GET();
+    return tstate->async_gen_finalizer;
+}
+
 PyObject *
 PyEval_GetBuiltins(void)
 {
@@ -4561,10 +4681,8 @@ PyEval_MergeCompilerFlags(PyCompilerFlags *cf)
    The arg must be a tuple or NULL.  The kw must be a dict or NULL. */
 
 PyObject *
-PyEval_CallObjectWithKeywords(PyObject *func, PyObject *arg, PyObject *kw)
+PyEval_CallObjectWithKeywords(PyObject *func, PyObject *args, PyObject *kwargs)
 {
-    PyObject *result;
-
 #ifdef Py_DEBUG
     /* PyEval_CallObjectWithKeywords() must not be called with an exception
        set. It raises a new exception if parameters are invalid or if
@@ -4572,30 +4690,23 @@ PyEval_CallObjectWithKeywords(PyObject *func, PyObject *arg, PyObject *kw)
     assert(!PyErr_Occurred());
 #endif
 
-    if (arg == NULL) {
-        arg = PyTuple_New(0);
-        if (arg == NULL)
-            return NULL;
+    if (args == NULL) {
+        return _PyObject_FastCallDict(func, NULL, 0, kwargs);
     }
-    else if (!PyTuple_Check(arg)) {
+
+    if (!PyTuple_Check(args)) {
         PyErr_SetString(PyExc_TypeError,
                         "argument list must be a tuple");
         return NULL;
     }
-    else
-        Py_INCREF(arg);
 
-    if (kw != NULL && !PyDict_Check(kw)) {
+    if (kwargs != NULL && !PyDict_Check(kwargs)) {
         PyErr_SetString(PyExc_TypeError,
                         "keyword list must be a dictionary");
-        Py_DECREF(arg);
         return NULL;
     }
 
-    result = PyObject_Call(func, arg, kw);
-    Py_DECREF(arg);
-
-    return result;
+    return PyObject_Call(func, args, kwargs);
 }
 
 const char *
@@ -4604,7 +4715,7 @@ PyEval_GetFuncName(PyObject *func)
     if (PyMethod_Check(func))
         return PyEval_GetFuncName(PyMethod_GET_FUNCTION(func));
     else if (PyFunction_Check(func))
-        return _PyUnicode_AsString(((PyFunctionObject*)func)->func_name);
+        return PyUnicode_AsUTF8(((PyFunctionObject*)func)->func_name);
     else if (PyCFunction_Check(func))
         return ((PyCFunctionObject*)func)->m_ml->ml_name;
     else
@@ -4624,21 +4735,6 @@ PyEval_GetFuncDesc(PyObject *func)
         return " object";
 }
 
-static void
-err_args(PyObject *func, int flags, int nargs)
-{
-    if (flags & METH_NOARGS)
-        PyErr_Format(PyExc_TypeError,
-                     "%.200s() takes no arguments (%d given)",
-                     ((PyCFunctionObject *)func)->m_ml->ml_name,
-                     nargs);
-    else
-        PyErr_Format(PyExc_TypeError,
-                     "%.200s() takes exactly one argument (%d given)",
-                     ((PyCFunctionObject *)func)->m_ml->ml_name,
-                     nargs);
-}
-
 #define C_TRACE(x, call) \
 if (tstate->use_tracing && tstate->c_profilefunc) { \
     if (call_trace(tstate->c_profilefunc, tstate->c_profileobj, \
@@ -4671,60 +4767,25 @@ if (tstate->use_tracing && tstate->c_profilefunc) { \
     }
 
 static PyObject *
-call_function(PyObject ***pp_stack, int oparg
-#ifdef WITH_TSC
-                , uint64* pintr0, uint64* pintr1
-#endif
-                )
+call_function(PyObject ***pp_stack, Py_ssize_t oparg, PyObject *kwnames)
 {
-    int na = oparg & 0xff;
-    int nk = (oparg>>8) & 0xff;
-    int n = na + 2 * nk;
-    PyObject **pfunc = (*pp_stack) - n - 1;
+    PyObject **pfunc = (*pp_stack) - oparg - 1;
     PyObject *func = *pfunc;
     PyObject *x, *w;
+    Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames);
+    Py_ssize_t nargs = oparg - nkwargs;
+    PyObject **stack;
 
     /* Always dispatch PyCFunction first, because these are
        presumed to be the most frequent callable object.
     */
-    if (PyCFunction_Check(func) && nk == 0) {
-        int flags = PyCFunction_GET_FLAGS(func);
+    if (PyCFunction_Check(func)) {
         PyThreadState *tstate = PyThreadState_GET();
 
         PCALL(PCALL_CFUNCTION);
-        if (flags & (METH_NOARGS | METH_O)) {
-            PyCFunction meth = PyCFunction_GET_FUNCTION(func);
-            PyObject *self = PyCFunction_GET_SELF(func);
-            if (flags & METH_NOARGS && na == 0) {
-                C_TRACE(x, (*meth)(self,NULL));
 
-                x = _Py_CheckFunctionResult(func, x, NULL);
-            }
-            else if (flags & METH_O && na == 1) {
-                PyObject *arg = EXT_POP(*pp_stack);
-                C_TRACE(x, (*meth)(self,arg));
-                Py_DECREF(arg);
-
-                x = _Py_CheckFunctionResult(func, x, NULL);
-            }
-            else {
-                err_args(func, flags, na);
-                x = NULL;
-            }
-        }
-        else {
-            PyObject *callargs;
-            callargs = load_args(pp_stack, na);
-            if (callargs != NULL) {
-                READ_TIMESTAMP(*pintr0);
-                C_TRACE(x, PyCFunction_Call(func,callargs,NULL));
-                READ_TIMESTAMP(*pintr1);
-                Py_XDECREF(callargs);
-            }
-            else {
-                x = NULL;
-            }
-        }
+        stack = (*pp_stack) - nargs - nkwargs;
+        C_TRACE(x, _PyCFunction_FastCallKeywords(func, stack, nargs, kwnames));
     }
     else {
         if (PyMethod_Check(func) && PyMethod_GET_SELF(func) != NULL) {
@@ -4736,21 +4797,26 @@ call_function(PyObject ***pp_stack, int oparg
             func = PyMethod_GET_FUNCTION(func);
             Py_INCREF(func);
             Py_SETREF(*pfunc, self);
-            na++;
-            n++;
-        } else
+            nargs++;
+        }
+        else {
             Py_INCREF(func);
-        READ_TIMESTAMP(*pintr0);
-        if (PyFunction_Check(func))
-            x = fast_function(func, pp_stack, n, na, nk);
-        else
-            x = do_call(func, pp_stack, na, nk);
-        READ_TIMESTAMP(*pintr1);
-        Py_DECREF(func);
+        }
+
+        stack = (*pp_stack) - nargs - nkwargs;
 
-        assert((x != NULL) ^ (PyErr_Occurred() != NULL));
+        if (PyFunction_Check(func)) {
+            x = fast_function(func, stack, nargs, kwnames);
+        }
+        else {
+            x = _PyObject_FastCallKeywords(func, stack, nargs, kwnames);
+        }
+
+        Py_DECREF(func);
     }
 
+    assert((x != NULL) ^ (PyErr_Occurred() != NULL));
+
     /* Clear the stack of the function object.  Also removes
        the arguments in case they weren't consumed already
        (fast_function() and err_args() leave them on the stack).
@@ -4761,7 +4827,6 @@ call_function(PyObject ***pp_stack, int oparg
         PCALL(PCALL_POP);
     }
 
-    assert((x != NULL) ^ (PyErr_Occurred() != NULL));
     return x;
 }
 
@@ -4774,257 +4839,197 @@ call_function(PyObject ***pp_stack, int oparg
    done before evaluating the frame.
 */
 
+static PyObject*
+_PyFunction_FastCall(PyCodeObject *co, PyObject **args, Py_ssize_t nargs,
+                     PyObject *globals)
+{
+    PyFrameObject *f;
+    PyThreadState *tstate = PyThreadState_GET();
+    PyObject **fastlocals;
+    Py_ssize_t i;
+    PyObject *result;
+
+    PCALL(PCALL_FASTER_FUNCTION);
+    assert(globals != NULL);
+    /* XXX Perhaps we should create a specialized
+       PyFrame_New() that doesn't take locals, but does
+       take builtins without sanity checking them.
+       */
+    assert(tstate != NULL);
+    f = PyFrame_New(tstate, co, globals, NULL);
+    if (f == NULL) {
+        return NULL;
+    }
+
+    fastlocals = f->f_localsplus;
+
+    for (i = 0; i < nargs; i++) {
+        Py_INCREF(*args);
+        fastlocals[i] = *args++;
+    }
+    result = PyEval_EvalFrameEx(f,0);
+
+    ++tstate->recursion_depth;
+    Py_DECREF(f);
+    --tstate->recursion_depth;
+
+    return result;
+}
+
 static PyObject *
-fast_function(PyObject *func, PyObject ***pp_stack, int n, int na, int nk)
+fast_function(PyObject *func, PyObject **stack,
+              Py_ssize_t nargs, PyObject *kwnames)
 {
     PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func);
     PyObject *globals = PyFunction_GET_GLOBALS(func);
     PyObject *argdefs = PyFunction_GET_DEFAULTS(func);
-    PyObject *kwdefs = PyFunction_GET_KW_DEFAULTS(func);
-    PyObject *name = ((PyFunctionObject *)func) -> func_name;
-    PyObject *qualname = ((PyFunctionObject *)func) -> func_qualname;
-    PyObject **d = NULL;
-    int nd = 0;
+    PyObject *kwdefs, *closure, *name, *qualname;
+    PyObject **d;
+    Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames);
+    Py_ssize_t nd;
+
+    assert(PyFunction_Check(func));
+    assert(nargs >= 0);
+    assert(kwnames == NULL || PyTuple_CheckExact(kwnames));
+    assert((nargs == 0 && nkwargs == 0) || stack != NULL);
+    /* kwnames must only contains str strings, no subclass, and all keys must
+       be unique */
 
     PCALL(PCALL_FUNCTION);
     PCALL(PCALL_FAST_FUNCTION);
-    if (argdefs == NULL && co->co_argcount == n &&
-        co->co_kwonlyargcount == 0 && nk==0 &&
-        co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) {
-        PyFrameObject *f;
-        PyObject *retval = NULL;
-        PyThreadState *tstate = PyThreadState_GET();
-        PyObject **fastlocals, **stack;
-        int i;
-
-        PCALL(PCALL_FASTER_FUNCTION);
-        assert(globals != NULL);
-        /* XXX Perhaps we should create a specialized
-           PyFrame_New() that doesn't take locals, but does
-           take builtins without sanity checking them.
-        */
-        assert(tstate != NULL);
-        f = PyFrame_New(tstate, co, globals, NULL);
-        if (f == NULL)
-            return NULL;
 
-        fastlocals = f->f_localsplus;
-        stack = (*pp_stack) - n;
-
-        for (i = 0; i < n; i++) {
-            Py_INCREF(*stack);
-            fastlocals[i] = *stack++;
+    if (co->co_kwonlyargcount == 0 && nkwargs == 0 &&
+        co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE))
+    {
+        if (argdefs == NULL && co->co_argcount == nargs) {
+            return _PyFunction_FastCall(co, stack, nargs, globals);
+        }
+        else if (nargs == 0 && argdefs != NULL
+                 && co->co_argcount == Py_SIZE(argdefs)) {
+            /* function called with no arguments, but all parameters have
+               a default value: use default values as arguments .*/
+            stack = &PyTuple_GET_ITEM(argdefs, 0);
+            return _PyFunction_FastCall(co, stack, Py_SIZE(argdefs), globals);
         }
-        retval = PyEval_EvalFrameEx(f,0);
-        ++tstate->recursion_depth;
-        Py_DECREF(f);
-        --tstate->recursion_depth;
-        return retval;
     }
+
+    kwdefs = PyFunction_GET_KW_DEFAULTS(func);
+    closure = PyFunction_GET_CLOSURE(func);
+    name = ((PyFunctionObject *)func) -> func_name;
+    qualname = ((PyFunctionObject *)func) -> func_qualname;
+
     if (argdefs != NULL) {
         d = &PyTuple_GET_ITEM(argdefs, 0);
         nd = Py_SIZE(argdefs);
     }
-    return _PyEval_EvalCodeWithName((PyObject*)co, globals,
-                                    (PyObject *)NULL, (*pp_stack)-n, na,
-                                    (*pp_stack)-2*nk, nk, d, nd, kwdefs,
-                                    PyFunction_GET_CLOSURE(func),
-                                    name, qualname);
+    else {
+        d = NULL;
+        nd = 0;
+    }
+    return _PyEval_EvalCodeWithName((PyObject*)co, globals, (PyObject *)NULL,
+                                    stack, nargs,
+                                    nkwargs ? &PyTuple_GET_ITEM(kwnames, 0) : NULL,
+                                    stack + nargs,
+                                    nkwargs, 1,
+                                    d, (int)nd, kwdefs,
+                                    closure, name, qualname);
 }
 
-static PyObject *
-update_keyword_args(PyObject *orig_kwdict, int nk, PyObject ***pp_stack,
-                    PyObject *func)
+PyObject *
+_PyFunction_FastCallKeywords(PyObject *func, PyObject **stack,
+                             Py_ssize_t nargs, PyObject *kwnames)
 {
-    PyObject *kwdict = NULL;
-    if (orig_kwdict == NULL)
-        kwdict = PyDict_New();
-    else {
-        kwdict = PyDict_Copy(orig_kwdict);
-        Py_DECREF(orig_kwdict);
-    }
-    if (kwdict == NULL)
-        return NULL;
-    while (--nk >= 0) {
-        int err;
-        PyObject *value = EXT_POP(*pp_stack);
-        PyObject *key = EXT_POP(*pp_stack);
-        if (PyDict_GetItem(kwdict, key) != NULL) {
-            PyErr_Format(PyExc_TypeError,
-                         "%.200s%s got multiple values "
-                         "for keyword argument '%U'",
-                         PyEval_GetFuncName(func),
-                         PyEval_GetFuncDesc(func),
-                         key);
-            Py_DECREF(key);
-            Py_DECREF(value);
-            Py_DECREF(kwdict);
-            return NULL;
-        }
-        err = PyDict_SetItem(kwdict, key, value);
-        Py_DECREF(key);
-        Py_DECREF(value);
-        if (err) {
-            Py_DECREF(kwdict);
-            return NULL;
-        }
-    }
-    return kwdict;
+    return fast_function(func, stack, nargs, kwnames);
 }
 
-static PyObject *
-update_star_args(int nstack, int nstar, PyObject *stararg,
-                 PyObject ***pp_stack)
+PyObject *
+_PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs,
+                         PyObject *kwargs)
 {
-    PyObject *callargs, *w;
+    PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func);
+    PyObject *globals = PyFunction_GET_GLOBALS(func);
+    PyObject *argdefs = PyFunction_GET_DEFAULTS(func);
+    PyObject *kwdefs, *closure, *name, *qualname;
+    PyObject *kwtuple, **k;
+    PyObject **d;
+    Py_ssize_t nd, nk;
+    PyObject *result;
 
-    callargs = PyTuple_New(nstack + nstar);
-    if (callargs == NULL) {
-        return NULL;
-    }
-    if (nstar) {
-        int i;
-        for (i = 0; i < nstar; i++) {
-            PyObject *a = PyTuple_GET_ITEM(stararg, i);
-            Py_INCREF(a);
-            PyTuple_SET_ITEM(callargs, nstack + i, a);
+    assert(func != NULL);
+    assert(nargs >= 0);
+    assert(nargs == 0 || args != NULL);
+    assert(kwargs == NULL || PyDict_Check(kwargs));
+
+    PCALL(PCALL_FUNCTION);
+    PCALL(PCALL_FAST_FUNCTION);
+
+    if (co->co_kwonlyargcount == 0 &&
+        (kwargs == NULL || PyDict_Size(kwargs) == 0) &&
+        co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE))
+    {
+        /* Fast paths */
+        if (argdefs == NULL && co->co_argcount == nargs) {
+            return _PyFunction_FastCall(co, args, nargs, globals);
+        }
+        else if (nargs == 0 && argdefs != NULL
+                 && co->co_argcount == Py_SIZE(argdefs)) {
+            /* function called with no arguments, but all parameters have
+               a default value: use default values as arguments .*/
+            args = &PyTuple_GET_ITEM(argdefs, 0);
+            return _PyFunction_FastCall(co, args, Py_SIZE(argdefs), globals);
         }
     }
-    while (--nstack >= 0) {
-        w = EXT_POP(*pp_stack);
-        PyTuple_SET_ITEM(callargs, nstack, w);
-    }
-    return callargs;
-}
 
-static PyObject *
-load_args(PyObject ***pp_stack, int na)
-{
-    PyObject *args = PyTuple_New(na);
-    PyObject *w;
+    if (kwargs != NULL) {
+        Py_ssize_t pos, i;
+        nk = PyDict_Size(kwargs);
 
-    if (args == NULL)
-        return NULL;
-    while (--na >= 0) {
-        w = EXT_POP(*pp_stack);
-        PyTuple_SET_ITEM(args, na, w);
+        kwtuple = PyTuple_New(2 * nk);
+        if (kwtuple == NULL) {
+            return NULL;
+        }
+
+        k = &PyTuple_GET_ITEM(kwtuple, 0);
+        pos = i = 0;
+        while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) {
+            Py_INCREF(k[i]);
+            Py_INCREF(k[i+1]);
+            i += 2;
+        }
+        nk = i / 2;
+    }
+    else {
+        kwtuple = NULL;
+        k = NULL;
+        nk = 0;
     }
-    return args;
-}
 
-static PyObject *
-do_call(PyObject *func, PyObject ***pp_stack, int na, int nk)
-{
-    PyObject *callargs = NULL;
-    PyObject *kwdict = NULL;
-    PyObject *result = NULL;
+    kwdefs = PyFunction_GET_KW_DEFAULTS(func);
+    closure = PyFunction_GET_CLOSURE(func);
+    name = ((PyFunctionObject *)func) -> func_name;
+    qualname = ((PyFunctionObject *)func) -> func_qualname;
 
-    if (nk > 0) {
-        kwdict = update_keyword_args(NULL, nk, pp_stack, func);
-        if (kwdict == NULL)
-            goto call_fail;
+    if (argdefs != NULL) {
+        d = &PyTuple_GET_ITEM(argdefs, 0);
+        nd = Py_SIZE(argdefs);
     }
-    callargs = load_args(pp_stack, na);
-    if (callargs == NULL)
-        goto call_fail;
-#ifdef CALL_PROFILE
-    /* At this point, we have to look at the type of func to
-       update the call stats properly.  Do it here so as to avoid
-       exposing the call stats machinery outside ceval.c
-    */
-    if (PyFunction_Check(func))
-        PCALL(PCALL_FUNCTION);
-    else if (PyMethod_Check(func))
-        PCALL(PCALL_METHOD);
-    else if (PyType_Check(func))
-        PCALL(PCALL_TYPE);
-    else if (PyCFunction_Check(func))
-        PCALL(PCALL_CFUNCTION);
-    else
-        PCALL(PCALL_OTHER);
-#endif
-    if (PyCFunction_Check(func)) {
-        PyThreadState *tstate = PyThreadState_GET();
-        C_TRACE(result, PyCFunction_Call(func, callargs, kwdict));
+    else {
+        d = NULL;
+        nd = 0;
     }
-    else
-        result = PyObject_Call(func, callargs, kwdict);
-call_fail:
-    Py_XDECREF(callargs);
-    Py_XDECREF(kwdict);
+
+    result = _PyEval_EvalCodeWithName((PyObject*)co, globals, (PyObject *)NULL,
+                                      args, nargs,
+                                      k, k + 1, nk, 2,
+                                      d, nd, kwdefs,
+                                      closure, name, qualname);
+    Py_XDECREF(kwtuple);
     return result;
 }
 
 static PyObject *
-ext_do_call(PyObject *func, PyObject ***pp_stack, int flags, int na, int nk)
+do_call_core(PyObject *func, PyObject *callargs, PyObject *kwdict)
 {
-    int nstar = 0;
-    PyObject *callargs = NULL;
-    PyObject *stararg = NULL;
-    PyObject *kwdict = NULL;
-    PyObject *result = NULL;
-
-    if (flags & CALL_FLAG_KW) {
-        kwdict = EXT_POP(*pp_stack);
-        if (!PyDict_Check(kwdict)) {
-            PyObject *d;
-            d = PyDict_New();
-            if (d == NULL)
-                goto ext_call_fail;
-            if (PyDict_Update(d, kwdict) != 0) {
-                Py_DECREF(d);
-                /* PyDict_Update raises attribute
-                 * error (percolated from an attempt
-                 * to get 'keys' attribute) instead of
-                 * a type error if its second argument
-                 * is not a mapping.
-                 */
-                if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
-                    PyErr_Format(PyExc_TypeError,
-                                 "%.200s%.200s argument after ** "
-                                 "must be a mapping, not %.200s",
-                                 PyEval_GetFuncName(func),
-                                 PyEval_GetFuncDesc(func),
-                                 kwdict->ob_type->tp_name);
-                }
-                goto ext_call_fail;
-            }
-            Py_DECREF(kwdict);
-            kwdict = d;
-        }
-    }
-    if (nk > 0) {
-        kwdict = update_keyword_args(kwdict, nk, pp_stack, func);
-        if (kwdict == NULL)
-            goto ext_call_fail;
-    }
-
-    if (flags & CALL_FLAG_VAR) {
-        stararg = EXT_POP(*pp_stack);
-        if (!PyTuple_Check(stararg)) {
-            PyObject *t = NULL;
-            if (Py_TYPE(stararg)->tp_iter == NULL &&
-                    !PySequence_Check(stararg)) {
-                PyErr_Format(PyExc_TypeError,
-                             "%.200s%.200s argument after * "
-                             "must be an iterable, not %.200s",
-                             PyEval_GetFuncName(func),
-                             PyEval_GetFuncDesc(func),
-                             stararg->ob_type->tp_name);
-                goto ext_call_fail;
-            }
-            t = PySequence_Tuple(stararg);
-            if (t == NULL) {
-                goto ext_call_fail;
-            }
-            Py_DECREF(stararg);
-            stararg = t;
-        }
-        nstar = PyTuple_GET_SIZE(stararg);
-    }
-    callargs = update_star_args(na, nstar, stararg, pp_stack);
-    if (callargs == NULL)
-        goto ext_call_fail;
 #ifdef CALL_PROFILE
     /* At this point, we have to look at the type of func to
        update the call stats properly.  Do it here so as to avoid
@@ -5041,17 +5046,16 @@ ext_do_call(PyObject *func, PyObject ***pp_stack, int flags, int na, int nk)
     else
         PCALL(PCALL_OTHER);
 #endif
+
     if (PyCFunction_Check(func)) {
+        PyObject *result;
         PyThreadState *tstate = PyThreadState_GET();
         C_TRACE(result, PyCFunction_Call(func, callargs, kwdict));
+        return result;
+    }
+    else {
+        return PyObject_Call(func, callargs, kwdict);
     }
-    else
-        result = PyObject_Call(func, callargs, kwdict);
-ext_call_fail:
-    Py_XDECREF(callargs);
-    Py_XDECREF(kwdict);
-    Py_XDECREF(stararg);
-    return result;
 }
 
 /* Extract a slice index from a PyLong or an object with the
@@ -5141,6 +5145,46 @@ cmp_outcome(int op, PyObject *v, PyObject *w)
 }
 
 static PyObject *
+import_name(PyFrameObject *f, PyObject *name, PyObject *fromlist, PyObject *level)
+{
+    _Py_IDENTIFIER(__import__);
+    PyObject *import_func, *res;
+    PyObject* stack[5];
+
+    import_func = _PyDict_GetItemId(f->f_builtins, &PyId___import__);
+    if (import_func == NULL) {
+        PyErr_SetString(PyExc_ImportError, "__import__ not found");
+        return NULL;
+    }
+
+    /* Fast path for not overloaded __import__. */
+    if (import_func == PyThreadState_GET()->interp->import_func) {
+        int ilevel = _PyLong_AsInt(level);
+        if (ilevel == -1 && PyErr_Occurred()) {
+            return NULL;
+        }
+        res = PyImport_ImportModuleLevelObject(
+                        name,
+                        f->f_globals,
+                        f->f_locals == NULL ? Py_None : f->f_locals,
+                        fromlist,
+                        ilevel);
+        return res;
+    }
+
+    Py_INCREF(import_func);
+
+    stack[0] = name;
+    stack[1] = f->f_globals;
+    stack[2] = f->f_locals == NULL ? Py_None : f->f_locals;
+    stack[3] = fromlist;
+    stack[4] = level;
+    res = _PyObject_FastCall(import_func, stack, 5);
+    Py_DECREF(import_func);
+    return res;
+}
+
+static PyObject *
 import_from(PyObject *v, PyObject *name)
 {
     PyObject *x;
@@ -5245,7 +5289,7 @@ format_exc_check_arg(PyObject *exc, const char *format_str, PyObject *obj)
     if (!obj)
         return;
 
-    obj_str = _PyUnicode_AsString(obj);
+    obj_str = PyUnicode_AsUTF8(obj);
     if (!obj_str)
         return;
 
@@ -5276,7 +5320,7 @@ format_exc_unbound(PyCodeObject *co, int oparg)
 
 static PyObject *
 unicode_concatenate(PyObject *v, PyObject *w,
-                    PyFrameObject *f, unsigned char *next_instr)
+                    PyFrameObject *f, const _Py_CODEUNIT *next_instr)
 {
     PyObject *res;
     if (Py_REFCNT(v) == 2) {
@@ -5286,10 +5330,11 @@ unicode_concatenate(PyObject *v, PyObject *w,
          * 'variable'.  We try to delete the variable now to reduce
          * the refcnt to 1.
          */
-        switch (*next_instr) {
+        int opcode, oparg;
+        NEXTOPARG();
+        switch (opcode) {
         case STORE_FAST:
         {
-            int oparg = PEEKARG();
             PyObject **fastlocals = f->f_localsplus;
             if (GETLOCAL(oparg) == v)
                 SETLOCAL(oparg, NULL);
@@ -5299,7 +5344,7 @@ unicode_concatenate(PyObject *v, PyObject *w,
         {
             PyObject **freevars = (f->f_localsplus +
                                    f->f_code->co_nlocals);
-            PyObject *c = freevars[PEEKARG()];
+            PyObject *c = freevars[oparg];
             if (PyCell_GET(c) == v)
                 PyCell_Set(c, NULL);
             break;
@@ -5307,7 +5352,7 @@ unicode_concatenate(PyObject *v, PyObject *w,
         case STORE_NAME:
         {
             PyObject *names = f->f_code->co_names;
-            PyObject *name = GETITEM(names, PEEKARG());
+            PyObject *name = GETITEM(names, oparg);
             PyObject *locals = f->f_locals;
             if (PyDict_CheckExact(locals) &&
                 PyDict_GetItem(locals, name) == v) {
@@ -5367,3 +5412,79 @@ _Py_GetDXProfile(PyObject *self, PyObject *args)
 }
 
 #endif
+
+Py_ssize_t
+_PyEval_RequestCodeExtraIndex(freefunc free)
+{
+    PyThreadState *tstate = PyThreadState_Get();
+    Py_ssize_t new_index;
+
+    if (tstate->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) {
+        return -1;
+    }
+    new_index = tstate->co_extra_user_count++;
+    tstate->co_extra_freefuncs[new_index] = free;
+    return new_index;
+}
+
+static void
+dtrace_function_entry(PyFrameObject *f)
+{
+    char* filename;
+    char* funcname;
+    int lineno;
+
+    filename = PyUnicode_AsUTF8(f->f_code->co_filename);
+    funcname = PyUnicode_AsUTF8(f->f_code->co_name);
+    lineno = PyCode_Addr2Line(f->f_code, f->f_lasti);
+
+    PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno);
+}
+
+static void
+dtrace_function_return(PyFrameObject *f)
+{
+    char* filename;
+    char* funcname;
+    int lineno;
+
+    filename = PyUnicode_AsUTF8(f->f_code->co_filename);
+    funcname = PyUnicode_AsUTF8(f->f_code->co_name);
+    lineno = PyCode_Addr2Line(f->f_code, f->f_lasti);
+
+    PyDTrace_FUNCTION_RETURN(filename, funcname, lineno);
+}
+
+/* DTrace equivalent of maybe_call_line_trace. */
+static void
+maybe_dtrace_line(PyFrameObject *frame,
+                  int *instr_lb, int *instr_ub, int *instr_prev)
+{
+    int line = frame->f_lineno;
+    char *co_filename, *co_name;
+
+    /* If the last instruction executed isn't in the current
+       instruction window, reset the window.
+    */
+    if (frame->f_lasti < *instr_lb || frame->f_lasti >= *instr_ub) {
+        PyAddrPair bounds;
+        line = _PyCode_CheckLineNumber(frame->f_code, frame->f_lasti,
+                                       &bounds);
+        *instr_lb = bounds.ap_lower;
+        *instr_ub = bounds.ap_upper;
+    }
+    /* If the last instruction falls at the start of a line or if
+       it represents a jump backwards, update the frame's line
+       number and call the trace function. */
+    if (frame->f_lasti == *instr_lb || frame->f_lasti < *instr_prev) {
+        frame->f_lineno = line;
+        co_filename = PyUnicode_AsUTF8(frame->f_code->co_filename);
+        if (!co_filename)
+            co_filename = "?";
+        co_name = PyUnicode_AsUTF8(frame->f_code->co_name);
+        if (!co_name)
+            co_name = "?";
+        PyDTrace_LINE(co_filename, co_name, line);
+    }
+    *instr_prev = frame->f_lasti;
+}