Issue #7111: Python can now be run without a stdin, stdout or stderr stream.

It was already the case with Python 2.  However, the corresponding
sys module entries are now set to None (instead of an unusable file object).

1 files changed, 222 insertions, 96 deletions

diff --git a/Python/compile.c b/Python/compile.c
index ba593a0b99..849f48785d 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -90,6 +90,13 @@ struct fblockinfo {
     basicblock *fb_block;
 };
 
+enum {
+    COMPILER_SCOPE_MODULE,
+    COMPILER_SCOPE_CLASS,
+    COMPILER_SCOPE_FUNCTION,
+    COMPILER_SCOPE_COMPREHENSION,
+};
+
 /* The following items change on entry and exit of code blocks.
    They must be saved and restored when returning to a block.
 */
@@ -97,6 +104,9 @@ struct compiler_unit {
     PySTEntryObject *u_ste;
 
     PyObject *u_name;
+    PyObject *u_qualname;  /* dot-separated qualified name (lazy) */
+    int u_scope_type;
+
     /* The following fields are dicts that map objects to
        the index of them in co_XXX.      The index is used as
        the argument for opcodes that refer to those collections.
@@ -135,6 +145,7 @@ managed by compiler_enter_scope() and compiler_exit_scope().
 
 struct compiler {
     const char *c_filename;
+    PyObject *c_filename_obj;
     struct symtable *c_st;
     PyFutureFeatures *c_future; /* pointer to module's __future__ */
     PyCompilerFlags *c_flags;
@@ -148,7 +159,7 @@ struct compiler {
     PyArena *c_arena;            /* pointer to memory allocation arena */
 };
 
-static int compiler_enter_scope(struct compiler *, identifier, void *, int);
+static int compiler_enter_scope(struct compiler *, identifier, int, void *, int);
 static void compiler_free(struct compiler *);
 static basicblock *compiler_new_block(struct compiler *);
 static int compiler_next_instr(struct compiler *, basicblock *);
@@ -178,12 +189,13 @@ static int compiler_in_loop(struct compiler *);
 static int inplace_binop(struct compiler *, operator_ty);
 static int expr_constant(struct compiler *, expr_ty);
 
-static int compiler_with(struct compiler *, stmt_ty);
+static int compiler_with(struct compiler *, stmt_ty, int);
 static int compiler_call_helper(struct compiler *c, int n,
                                 asdl_seq *args,
                                 asdl_seq *keywords,
                                 expr_ty starargs,
                                 expr_ty kwargs);
+static int compiler_try_except(struct compiler *, stmt_ty);
 
 static PyCodeObject *assemble(struct compiler *, int addNone);
 static PyObject *__doc__;
@@ -195,16 +207,17 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident)
 {
     /* Name mangling: __private becomes _classname__private.
        This is independent from how the name is used. */
-    const Py_UNICODE *p, *name = PyUnicode_AS_UNICODE(ident);
-    Py_UNICODE *buffer;
-    size_t nlen, plen;
+    PyObject *result;
+    size_t nlen, plen, ipriv;
+    Py_UCS4 maxchar;
     if (privateobj == NULL || !PyUnicode_Check(privateobj) ||
-        name == NULL || name[0] != '_' || name[1] != '_') {
+        PyUnicode_READ_CHAR(ident, 0) != '_' ||
+        PyUnicode_READ_CHAR(ident, 1) != '_') {
         Py_INCREF(ident);
         return ident;
     }
-    p = PyUnicode_AS_UNICODE(privateobj);
-    nlen = Py_UNICODE_strlen(name);
+    nlen = PyUnicode_GET_LENGTH(ident);
+    plen = PyUnicode_GET_LENGTH(privateobj);
     /* Don't mangle __id__ or names with dots.
 
        The only time a name with a dot can occur is when
@@ -214,32 +227,43 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident)
        TODO(jhylton): Decide whether we want to support
        mangling of the module name, e.g. __M.X.
     */
-    if ((name[nlen-1] == '_' && name[nlen-2] == '_')
-        || Py_UNICODE_strchr(name, '.')) {
+    if ((PyUnicode_READ_CHAR(ident, nlen-1) == '_' &&
+         PyUnicode_READ_CHAR(ident, nlen-2) == '_') ||
+        PyUnicode_FindChar(ident, '.', 0, nlen, 1) != -1) {
         Py_INCREF(ident);
         return ident; /* Don't mangle __whatever__ */
     }
     /* Strip leading underscores from class name */
-    while (*p == '_')
-        p++;
-    if (*p == 0) {
+    ipriv = 0;
+    while (PyUnicode_READ_CHAR(privateobj, ipriv) == '_')
+        ipriv++;
+    if (ipriv == plen) {
         Py_INCREF(ident);
         return ident; /* Don't mangle if class is just underscores */
     }
-    plen = Py_UNICODE_strlen(p);
+    plen -= ipriv;
 
     assert(1 <= PY_SSIZE_T_MAX - nlen);
     assert(1 + nlen <= PY_SSIZE_T_MAX - plen);
 
-    ident = PyUnicode_FromStringAndSize(NULL, 1 + nlen + plen);
-    if (!ident)
+    maxchar = PyUnicode_MAX_CHAR_VALUE(ident);
+    if (PyUnicode_MAX_CHAR_VALUE(privateobj) > maxchar)
+        maxchar = PyUnicode_MAX_CHAR_VALUE(privateobj);
+
+    result = PyUnicode_New(1 + nlen + plen, maxchar);
+    if (!result)
         return 0;
-    /* ident = "_" + p[:plen] + name # i.e. 1+plen+nlen bytes */
-    buffer = PyUnicode_AS_UNICODE(ident);
-    buffer[0] = '_';
-    Py_UNICODE_strncpy(buffer+1, p, plen);
-    Py_UNICODE_strcpy(buffer+1+plen, name);
-    return ident;
+    /* ident = "_" + priv[ipriv:] + ident # i.e. 1+plen+nlen bytes */
+    PyUnicode_WRITE(PyUnicode_KIND(result), PyUnicode_DATA(result), 0, '_');
+    if (PyUnicode_CopyCharacters(result, 1, privateobj, ipriv, plen) < 0) {
+        Py_DECREF(result);
+        return NULL;
+    }
+    if (PyUnicode_CopyCharacters(result, plen+1, ident, 0, nlen) < 0) {
+        Py_DECREF(result);
+        return NULL;
+    }
+    return result;
 }
 
 static int
@@ -272,6 +296,9 @@ PyAST_CompileEx(mod_ty mod, const char *filename, PyCompilerFlags *flags,
     if (!compiler_init(&c))
         return NULL;
     c.c_filename = filename;
+    c.c_filename_obj = PyUnicode_DecodeFSDefault(filename);
+    if (!c.c_filename_obj)
+        goto finally;
     c.c_arena = arena;
     c.c_future = PyFuture_FromAST(mod, filename);
     if (c.c_future == NULL)
@@ -324,6 +351,8 @@ compiler_free(struct compiler *c)
         PySymtable_Free(c->c_st);
     if (c->c_future)
         PyObject_Free(c->c_future);
+    if (c->c_filename_obj)
+        Py_DECREF(c->c_filename_obj);
     Py_DECREF(c->c_stack);
 }
 
@@ -438,6 +467,7 @@ compiler_unit_free(struct compiler_unit *u)
     }
     Py_CLEAR(u->u_ste);
     Py_CLEAR(u->u_name);
+    Py_CLEAR(u->u_qualname);
     Py_CLEAR(u->u_consts);
     Py_CLEAR(u->u_names);
     Py_CLEAR(u->u_varnames);
@@ -448,8 +478,8 @@ compiler_unit_free(struct compiler_unit *u)
 }
 
 static int
-compiler_enter_scope(struct compiler *c, identifier name, void *key,
-                     int lineno)
+compiler_enter_scope(struct compiler *c, identifier name,
+                     int scope_type, void *key, int lineno)
 {
     struct compiler_unit *u;
 
@@ -460,6 +490,7 @@ compiler_enter_scope(struct compiler *c, identifier name, void *key,
         return 0;
     }
     memset(u, 0, sizeof(struct compiler_unit));
+    u->u_scope_type = scope_type;
     u->u_argcount = 0;
     u->u_kwonlyargcount = 0;
     u->u_ste = PySymtable_Lookup(c->c_st, key);
@@ -547,6 +578,59 @@ compiler_exit_scope(struct compiler *c)
 
 }
 
+static PyObject *
+compiler_scope_qualname(struct compiler *c)
+{
+    Py_ssize_t stack_size, i;
+    _Py_static_string(dot, ".");
+    _Py_static_string(locals, "<locals>");
+    struct compiler_unit *u;
+    PyObject *capsule, *name, *seq, *dot_str, *locals_str;
+
+    u = c->u;
+    if (u->u_qualname != NULL) {
+        Py_INCREF(u->u_qualname);
+        return u->u_qualname;
+    }
+
+    seq = PyList_New(0);
+    if (seq == NULL)
+        return NULL;
+
+    stack_size = PyList_GET_SIZE(c->c_stack);
+    for (i = 0; i < stack_size; i++) {
+        capsule = PyList_GET_ITEM(c->c_stack, i);
+        u = (struct compiler_unit *)PyCapsule_GetPointer(capsule, COMPILER_CAPSULE_NAME_COMPILER_UNIT);
+        assert(u);
+        if (u->u_scope_type == COMPILER_SCOPE_MODULE)
+            continue;
+        if (PyList_Append(seq, u->u_name))
+            goto _error;
+        if (u->u_scope_type == COMPILER_SCOPE_FUNCTION) {
+            locals_str = _PyUnicode_FromId(&locals);
+            if (locals_str == NULL)
+                goto _error;
+            if (PyList_Append(seq, locals_str))
+                goto _error;
+        }
+    }
+    u = c->u;
+    if (PyList_Append(seq, u->u_name))
+        goto _error;
+    dot_str = _PyUnicode_FromId(&dot);
+    if (dot_str == NULL)
+        goto _error;
+    name = PyUnicode_Join(dot_str, seq);
+    Py_DECREF(seq);
+    u->u_qualname = name;
+    Py_XINCREF(name);
+    return name;
+
+_error:
+    Py_XDECREF(seq);
+    return NULL;
+}
+
 /* Allocate a new block and return a pointer to it.
    Returns NULL on error.
 */
@@ -843,9 +927,9 @@ opcode_stack_effect(int opcode, int oparg)
         case CALL_FUNCTION_VAR_KW:
             return -NARGS(oparg)-2;
         case MAKE_FUNCTION:
-            return -NARGS(oparg) - ((oparg >> 16) & 0xffff);
+            return -1 -NARGS(oparg) - ((oparg >> 16) & 0xffff);
         case MAKE_CLOSURE:
-            return -1 - NARGS(oparg) - ((oparg >> 16) & 0xffff);
+            return -2 - NARGS(oparg) - ((oparg >> 16) & 0xffff);
 #undef NARGS
         case BUILD_SLICE:
             if (oparg == 3)
@@ -1175,7 +1259,7 @@ compiler_mod(struct compiler *c, mod_ty mod)
             return NULL;
     }
     /* Use 0 for firstlineno initially, will fixup in assemble(). */
-    if (!compiler_enter_scope(c, module, mod, 0))
+    if (!compiler_enter_scope(c, module, COMPILER_SCOPE_MODULE, mod, 0))
         return NULL;
     switch (mod->kind) {
     case Module_kind:
@@ -1251,11 +1335,15 @@ compiler_lookup_arg(PyObject *dict, PyObject *name)
 }
 
 static int
-compiler_make_closure(struct compiler *c, PyCodeObject *co, int args)
+compiler_make_closure(struct compiler *c, PyCodeObject *co, int args, PyObject *qualname)
 {
     int i, free = PyCode_GetNumFree(co);
+    if (qualname == NULL)
+        qualname = co->co_name;
+
     if (free == 0) {
         ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts);
+        ADDOP_O(c, LOAD_CONST, qualname, consts);
         ADDOP_I(c, MAKE_FUNCTION, args);
         return 1;
     }
@@ -1292,6 +1380,7 @@ compiler_make_closure(struct compiler *c, PyCodeObject *co, int args)
     }
     ADDOP_I(c, BUILD_TUPLE, free);
     ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts);
+    ADDOP_O(c, LOAD_CONST, qualname, consts);
     ADDOP_I(c, MAKE_CLOSURE, args);
     return 1;
 }
@@ -1433,7 +1522,7 @@ static int
 compiler_function(struct compiler *c, stmt_ty s)
 {
     PyCodeObject *co;
-    PyObject *first_const = Py_None;
+    PyObject *qualname, *first_const = Py_None;
     arguments_ty args = s->v.FunctionDef.args;
     expr_ty returns = s->v.FunctionDef.returns;
     asdl_seq* decos = s->v.FunctionDef.decorator_list;
@@ -1459,7 +1548,8 @@ compiler_function(struct compiler *c, stmt_ty s)
         return 0;
     assert((num_annotations & 0xFFFF) == num_annotations);
 
-    if (!compiler_enter_scope(c, s->v.FunctionDef.name, (void *)s,
+    if (!compiler_enter_scope(c, s->v.FunctionDef.name,
+                              COMPILER_SCOPE_FUNCTION, (void *)s,
                               s->lineno))
         return 0;
 
@@ -1481,14 +1571,19 @@ compiler_function(struct compiler *c, stmt_ty s)
         VISIT_IN_SCOPE(c, stmt, st);
     }
     co = assemble(c, 1);
+    qualname = compiler_scope_qualname(c);
     compiler_exit_scope(c);
-    if (co == NULL)
+    if (qualname == NULL || co == NULL) {
+        Py_XDECREF(qualname);
+        Py_XDECREF(co);
         return 0;
+    }
 
     arglength = asdl_seq_LEN(args->defaults);
     arglength |= kw_default_count << 8;
     arglength |= num_annotations << 16;
-    compiler_make_closure(c, co, arglength);
+    compiler_make_closure(c, co, arglength, qualname);
+    Py_DECREF(qualname);
     Py_DECREF(co);
 
     /* decorators */
@@ -1523,7 +1618,8 @@ compiler_class(struct compiler *c, stmt_ty s)
     */
 
     /* 1. compile the class body into a code object */
-    if (!compiler_enter_scope(c, s->v.ClassDef.name, (void *)s, s->lineno))
+    if (!compiler_enter_scope(c, s->v.ClassDef.name,
+                              COMPILER_SCOPE_CLASS, (void *)s, s->lineno))
         return 0;
     /* this block represents what we do in the new scope */
     {
@@ -1553,13 +1649,28 @@ compiler_class(struct compiler *c, stmt_ty s)
             return 0;
         }
         Py_DECREF(str);
+        /* store the __qualname__ */
+        str = compiler_scope_qualname(c);
+        if (!str) {
+            compiler_exit_scope(c);
+            return 0;
+        }
+        ADDOP_O(c, LOAD_CONST, str, consts);
+        Py_DECREF(str);
+        str = PyUnicode_InternFromString("__qualname__");
+        if (!str || !compiler_nameop(c, str, Store)) {
+            Py_XDECREF(str);
+            compiler_exit_scope(c);
+            return 0;
+        }
+        Py_DECREF(str);
         /* compile the body proper */
         if (!compiler_body(c, s->v.ClassDef.body)) {
             compiler_exit_scope(c);
             return 0;
         }
         /* return the (empty) __class__ cell */
-        str = PyUnicode_InternFromString("__class__");
+        str = PyUnicode_InternFromString("@__class__");
         if (str == NULL) {
             compiler_exit_scope(c);
             return 0;
@@ -1589,7 +1700,7 @@ compiler_class(struct compiler *c, stmt_ty s)
     ADDOP(c, LOAD_BUILD_CLASS);
 
     /* 3. load a function (or closure) made from the code object */
-    compiler_make_closure(c, co, 0);
+    compiler_make_closure(c, co, 0, NULL);
     Py_DECREF(co);
 
     /* 4. load class name */
@@ -1640,6 +1751,7 @@ static int
 compiler_lambda(struct compiler *c, expr_ty e)
 {
     PyCodeObject *co;
+    PyObject *qualname;
     static identifier name;
     int kw_default_count = 0, arglength;
     arguments_ty args = e->v.Lambda.args;
@@ -1659,7 +1771,8 @@ compiler_lambda(struct compiler *c, expr_ty e)
     }
     if (args->defaults)
         VISIT_SEQ(c, expr, args->defaults);
-    if (!compiler_enter_scope(c, name, (void *)e, e->lineno))
+    if (!compiler_enter_scope(c, name, COMPILER_SCOPE_FUNCTION,
+                              (void *)e, e->lineno))
         return 0;
 
     /* Make None the first constant, so the lambda can't have a
@@ -1677,13 +1790,15 @@ compiler_lambda(struct compiler *c, expr_ty e)
         ADDOP_IN_SCOPE(c, RETURN_VALUE);
     }
     co = assemble(c, 1);
+    qualname = compiler_scope_qualname(c);
     compiler_exit_scope(c);
-    if (co == NULL)
+    if (qualname == NULL || co == NULL)
         return 0;
 
     arglength = asdl_seq_LEN(args->defaults);
     arglength |= kw_default_count << 8;
-    compiler_make_closure(c, co, arglength);
+    compiler_make_closure(c, co, arglength, qualname);
+    Py_DECREF(qualname);
     Py_DECREF(co);
 
     return 1;
@@ -1892,7 +2007,13 @@ compiler_try_finally(struct compiler *c, stmt_ty s)
     compiler_use_next_block(c, body);
     if (!compiler_push_fblock(c, FINALLY_TRY, body))
         return 0;
-    VISIT_SEQ(c, stmt, s->v.TryFinally.body);
+    if (s->v.Try.handlers && asdl_seq_LEN(s->v.Try.handlers)) {
+        if (!compiler_try_except(c, s))
+            return 0;
+    }
+    else {
+        VISIT_SEQ(c, stmt, s->v.Try.body);
+    }
     ADDOP(c, POP_BLOCK);
     compiler_pop_fblock(c, FINALLY_TRY, body);
 
@@ -1900,7 +2021,7 @@ compiler_try_finally(struct compiler *c, stmt_ty s)
     compiler_use_next_block(c, end);
     if (!compiler_push_fblock(c, FINALLY_END, end))
         return 0;
-    VISIT_SEQ(c, stmt, s->v.TryFinally.finalbody);
+    VISIT_SEQ(c, stmt, s->v.Try.finalbody);
     ADDOP(c, END_FINALLY);
     compiler_pop_fblock(c, FINALLY_END, end);
 
@@ -1954,15 +2075,15 @@ compiler_try_except(struct compiler *c, stmt_ty s)
     compiler_use_next_block(c, body);
     if (!compiler_push_fblock(c, EXCEPT, body))
         return 0;
-    VISIT_SEQ(c, stmt, s->v.TryExcept.body);
+    VISIT_SEQ(c, stmt, s->v.Try.body);
     ADDOP(c, POP_BLOCK);
     compiler_pop_fblock(c, EXCEPT, body);
     ADDOP_JREL(c, JUMP_FORWARD, orelse);
-    n = asdl_seq_LEN(s->v.TryExcept.handlers);
+    n = asdl_seq_LEN(s->v.Try.handlers);
     compiler_use_next_block(c, except);
     for (i = 0; i < n; i++) {
         excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET(
-            s->v.TryExcept.handlers, i);
+            s->v.Try.handlers, i);
         if (!handler->v.ExceptHandler.type && i < n-1)
             return compiler_error(c, "default 'except:' must be last");
         c->u->u_lineno_set = 0;
@@ -2049,12 +2170,21 @@ compiler_try_except(struct compiler *c, stmt_ty s)
     }
     ADDOP(c, END_FINALLY);
     compiler_use_next_block(c, orelse);
-    VISIT_SEQ(c, stmt, s->v.TryExcept.orelse);
+    VISIT_SEQ(c, stmt, s->v.Try.orelse);
     compiler_use_next_block(c, end);
     return 1;
 }
 
 static int
+compiler_try(struct compiler *c, stmt_ty s) {
+    if (s->v.Try.finalbody && asdl_seq_LEN(s->v.Try.finalbody))
+        return compiler_try_finally(c, s);
+    else
+        return compiler_try_except(c, s);
+}
+
+
+static int
 compiler_import_as(struct compiler *c, identifier name, identifier asname)
 {
     /* The IMPORT_NAME opcode was already generated.  This function
@@ -2063,22 +2193,27 @@ compiler_import_as(struct compiler *c, identifier name, identifier asname)
        If there is a dot in name, we need to split it and emit a
        LOAD_ATTR for each name.
     */
-    const Py_UNICODE *src = PyUnicode_AS_UNICODE(name);
-    const Py_UNICODE *dot = Py_UNICODE_strchr(src, '.');
-    if (dot) {
+    Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0,
+                                        PyUnicode_GET_LENGTH(name), 1);
+    if (dot == -2)
+        return -1;
+    if (dot != -1) {
         /* Consume the base module name to get the first attribute */
-        src = dot + 1;
-        while (dot) {
-            /* NB src is only defined when dot != NULL */
+        Py_ssize_t pos = dot + 1;
+        while (dot != -1) {
             PyObject *attr;
-            dot = Py_UNICODE_strchr(src, '.');
-            attr = PyUnicode_FromUnicode(src,
-                                dot ? dot - src : Py_UNICODE_strlen(src));
+            dot = PyUnicode_FindChar(name, '.', pos,
+                                     PyUnicode_GET_LENGTH(name), 1);
+            if (dot == -2)
+                return -1;
+            attr = PyUnicode_Substring(name, pos,
+                                       (dot != -1) ? dot :
+                                       PyUnicode_GET_LENGTH(name));
             if (!attr)
                 return -1;
             ADDOP_O(c, LOAD_ATTR, attr, names);
             Py_DECREF(attr);
-            src = dot + 1;
+            pos = dot + 1;
         }
     }
     return compiler_nameop(c, asname, Store);
@@ -2117,13 +2252,12 @@ compiler_import(struct compiler *c, stmt_ty s)
         }
         else {
             identifier tmp = alias->name;
-            const Py_UNICODE *base = PyUnicode_AS_UNICODE(alias->name);
-            Py_UNICODE *dot = Py_UNICODE_strchr(base, '.');
-            if (dot)
-                tmp = PyUnicode_FromUnicode(base,
-                                            dot - base);
+            Py_ssize_t dot = PyUnicode_FindChar(
+                alias->name, '.', 0, PyUnicode_GET_LENGTH(alias->name), 1);
+            if (dot != -1)
+                tmp = PyUnicode_Substring(alias->name, 0, dot);
             r = compiler_nameop(c, tmp, Store);
-            if (dot) {
+            if (dot != -1) {
                 Py_DECREF(tmp);
             }
             if (!r)
@@ -2186,7 +2320,7 @@ compiler_from_import(struct compiler *c, stmt_ty s)
         alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i);
         identifier store_name;
 
-        if (i == 0 && *PyUnicode_AS_UNICODE(alias->name) == '*') {
+        if (i == 0 && PyUnicode_READ_CHAR(alias->name, 0) == '*') {
             assert(n == 1);
             ADDOP(c, IMPORT_STAR);
             return 1;
@@ -2301,10 +2435,8 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s)
         }
         ADDOP_I(c, RAISE_VARARGS, n);
         break;
-    case TryExcept_kind:
-        return compiler_try_except(c, s);
-    case TryFinally_kind:
-        return compiler_try_finally(c, s);
+    case Try_kind:
+        return compiler_try(c, s);
     case Assert_kind:
         return compiler_assert(c, s);
     case Import_kind:
@@ -2335,7 +2467,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s)
     case Continue_kind:
         return compiler_continue(c);
     case With_kind:
-        return compiler_with(c, s);
+        return compiler_with(c, s, 0);
     }
     return 1;
 }
@@ -2502,7 +2634,7 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx)
     }
 
     /* XXX Leave assert here, but handle __doc__ and the like better */
-    assert(scope || PyUnicode_AS_UNICODE(name)[0] == '_');
+    assert(scope || PyUnicode_READ_CHAR(name, 0) == '_');
 
     switch (optype) {
     case OP_DEREF:
@@ -2880,11 +3012,13 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name,
 {
     PyCodeObject *co = NULL;
     expr_ty outermost_iter;
+    PyObject *qualname = NULL;
 
     outermost_iter = ((comprehension_ty)
                       asdl_seq_GET(generators, 0))->iter;
 
-    if (!compiler_enter_scope(c, name, (void *)e, e->lineno))
+    if (!compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION,
+                              (void *)e, e->lineno))
         goto error;
 
     if (type != COMP_GENEXP) {
@@ -2917,12 +3051,14 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name,
     }
 
     co = assemble(c, 1);
+    qualname = compiler_scope_qualname(c);
     compiler_exit_scope(c);
-    if (co == NULL)
+    if (qualname == NULL || co == NULL)
         goto error;
 
-    if (!compiler_make_closure(c, co, 0))
+    if (!compiler_make_closure(c, co, 0, qualname))
         goto error;
+    Py_DECREF(qualname);
     Py_DECREF(co);
 
     VISIT(c, expr, outermost_iter);
@@ -2932,6 +3068,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name,
 error_in_scope:
     compiler_exit_scope(c);
 error:
+    Py_XDECREF(qualname);
     Py_XDECREF(co);
     return 0;
 }
@@ -3025,8 +3162,7 @@ expr_constant(struct compiler *c, expr_ty e)
         return PyObject_IsTrue(e->v.Str.s);
     case Name_kind:
         /* optimize away names that can't be reassigned */
-        id = PyBytes_AS_STRING(
-            _PyUnicode_AsDefaultEncodedString(e->v.Name.id, NULL));
+        id = PyUnicode_AsUTF8(e->v.Name.id);
         if (strcmp(id, "True") == 0) return 1;
         if (strcmp(id, "False") == 0) return 0;
         if (strcmp(id, "None") == 0) return 0;
@@ -3062,9 +3198,10 @@ expr_constant(struct compiler *c, expr_ty e)
        exit(*exc)
  */
 static int
-compiler_with(struct compiler *c, stmt_ty s)
+compiler_with(struct compiler *c, stmt_ty s, int pos)
 {
     basicblock *block, *finally;
+    withitem_ty item = asdl_seq_GET(s->v.With.items, pos);
 
     assert(s->kind == With_kind);
 
@@ -3074,7 +3211,7 @@ compiler_with(struct compiler *c, stmt_ty s)
         return 0;
 
     /* Evaluate EXPR */
-    VISIT(c, expr, s->v.With.context_expr);
+    VISIT(c, expr, item->context_expr);
     ADDOP_JREL(c, SETUP_WITH, finally);
 
     /* SETUP_WITH pushes a finally block. */
@@ -3083,16 +3220,20 @@ compiler_with(struct compiler *c, stmt_ty s)
         return 0;
     }
 
-    if (s->v.With.optional_vars) {
-        VISIT(c, expr, s->v.With.optional_vars);
+    if (item->optional_vars) {
+        VISIT(c, expr, item->optional_vars);
     }
     else {
     /* Discard result from context.__enter__() */
         ADDOP(c, POP_TOP);
     }
 
-    /* BLOCK code */
-    VISIT_SEQ(c, stmt, s->v.With.body);
+    pos++;
+    if (pos == asdl_seq_LEN(s->v.With.items))
+        /* BLOCK code */
+        VISIT_SEQ(c, stmt, s->v.With.body)
+    else if (!compiler_with(c, s, pos))
+            return 0;
 
     /* End of try block; start the finally block */
     ADDOP(c, POP_BLOCK);
@@ -3361,7 +3502,7 @@ compiler_in_loop(struct compiler *c) {
 static int
 compiler_error(struct compiler *c, const char *errstr)
 {
-    PyObject *loc, *filename;
+    PyObject *loc;
     PyObject *u = NULL, *v = NULL;
 
     loc = PyErr_ProgramText(c->c_filename, c->u->u_lineno);
@@ -3369,16 +3510,7 @@ compiler_error(struct compiler *c, const char *errstr)
         Py_INCREF(Py_None);
         loc = Py_None;
     }
-    if (c->c_filename != NULL) {
-        filename = PyUnicode_DecodeFSDefault(c->c_filename);
-        if (!filename)
-            goto exit;
-    }
-    else {
-        Py_INCREF(Py_None);
-        filename = Py_None;
-    }
-    u = Py_BuildValue("(NiiO)", filename, c->u->u_lineno,
+    u = Py_BuildValue("(OiiO)", c->c_filename_obj, c->u->u_lineno,
                       c->u->u_col_offset, loc);
     if (!u)
         goto exit;
@@ -3927,7 +4059,6 @@ makecode(struct compiler *c, struct assembler *a)
     PyObject *consts = NULL;
     PyObject *names = NULL;
     PyObject *varnames = NULL;
-    PyObject *filename = NULL;
     PyObject *name = NULL;
     PyObject *freevars = NULL;
     PyObject *cellvars = NULL;
@@ -3951,10 +4082,6 @@ makecode(struct compiler *c, struct assembler *a)
     freevars = dict_keys_inorder(c->u->u_freevars, PyTuple_Size(cellvars));
     if (!freevars)
         goto error;
-    filename = PyUnicode_DecodeFSDefault(c->c_filename);
-    if (!filename)
-        goto error;
-
     nlocals = PyDict_Size(c->u->u_varnames);
     flags = compute_code_flags(c);
     if (flags < 0)
@@ -3974,14 +4101,13 @@ makecode(struct compiler *c, struct assembler *a)
                     nlocals, stackdepth(c), flags,
                     bytecode, consts, names, varnames,
                     freevars, cellvars,
-                    filename, c->u->u_name,
+                    c->c_filename_obj, c->u->u_name,
                     c->u->u_firstlineno,
                     a->a_lnotab);
  error:
     Py_XDECREF(consts);
     Py_XDECREF(names);
     Py_XDECREF(varnames);
-    Py_XDECREF(filename);
     Py_XDECREF(name);
     Py_XDECREF(freevars);
     Py_XDECREF(cellvars);