Issue #13612: handle unknown encodings without a buffer overflow.

This affects pyexpat and _elementtree. PyExpat_CAPI now exposes a new
function - DefaultUnknownEncodingHandler.

Based on a patch by Serhiy Storchaka.

21 files changed, 152 insertions, 81 deletions

diff --git a/Include/Python-ast.h b/Include/Python-ast.h
index 7ad6cb370b..49edcdcebf 100644
--- a/Include/Python-ast.h
+++ b/Include/Python-ast.h
@@ -182,8 +182,9 @@ enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
                   SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
                   Yield_kind=12, YieldFrom_kind=13, Compare_kind=14,
                   Call_kind=15, Num_kind=16, Str_kind=17, Bytes_kind=18,
-                  Ellipsis_kind=19, Attribute_kind=20, Subscript_kind=21,
-                  Starred_kind=22, Name_kind=23, List_kind=24, Tuple_kind=25};
+                  NameConstant_kind=19, Ellipsis_kind=20, Attribute_kind=21,
+                  Subscript_kind=22, Starred_kind=23, Name_kind=24,
+                  List_kind=25, Tuple_kind=26};
 struct _expr {
         enum _expr_kind kind;
         union {
@@ -279,6 +280,10 @@ struct _expr {
                 } Bytes;
                 
                 struct {
+                        singleton value;
+                } NameConstant;
+                
+                struct {
                         expr_ty value;
                         identifier attr;
                         expr_context_ty ctx;
@@ -359,18 +364,18 @@ struct _excepthandler {
 
 struct _arguments {
         asdl_seq *args;
-        identifier vararg;
-        expr_ty varargannotation;
+        arg_ty vararg;
         asdl_seq *kwonlyargs;
-        identifier kwarg;
-        expr_ty kwargannotation;
-        asdl_seq *defaults;
         asdl_seq *kw_defaults;
+        arg_ty kwarg;
+        asdl_seq *defaults;
 };
 
 struct _arg {
         identifier arg;
         expr_ty annotation;
+        int lineno;
+        int col_offset;
 };
 
 struct _keyword {
@@ -509,6 +514,9 @@ expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena);
 expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena);
 #define Bytes(a0, a1, a2, a3) _Py_Bytes(a0, a1, a2, a3)
 expr_ty _Py_Bytes(bytes s, int lineno, int col_offset, PyArena *arena);
+#define NameConstant(a0, a1, a2, a3) _Py_NameConstant(a0, a1, a2, a3)
+expr_ty _Py_NameConstant(singleton value, int lineno, int col_offset, PyArena
+                         *arena);
 #define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2)
 expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena);
 #define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5)
@@ -542,11 +550,10 @@ comprehension_ty _Py_comprehension(expr_ty target, expr_ty iter, asdl_seq *
 excepthandler_ty _Py_ExceptHandler(expr_ty type, identifier name, asdl_seq *
                                    body, int lineno, int col_offset, PyArena
                                    *arena);
-#define arguments(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_arguments(a0, a1, a2, a3, a4, a5, a6, a7, a8)
-arguments_ty _Py_arguments(asdl_seq * args, identifier vararg, expr_ty
-                           varargannotation, asdl_seq * kwonlyargs, identifier
-                           kwarg, expr_ty kwargannotation, asdl_seq * defaults,
-                           asdl_seq * kw_defaults, PyArena *arena);
+#define arguments(a0, a1, a2, a3, a4, a5, a6) _Py_arguments(a0, a1, a2, a3, a4, a5, a6)
+arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq *
+                           kwonlyargs, asdl_seq * kw_defaults, arg_ty kwarg,
+                           asdl_seq * defaults, PyArena *arena);
 #define arg(a0, a1, a2) _Py_arg(a0, a1, a2)
 arg_ty _Py_arg(identifier arg, expr_ty annotation, PyArena *arena);
 #define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2)
diff --git a/Include/abstract.h b/Include/abstract.h
index 44b5af72c9..6f1691399a 100644
--- a/Include/abstract.h
+++ b/Include/abstract.h
@@ -339,11 +339,10 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
 
      PyAPI_FUNC(PyObject *) PyObject_CallMethodObjArgs(PyObject *o,
                                                        PyObject *method, ...);
-     PyAPI_FUNC(PyObject *) _PyObject_CallMethodObjIdArgs(PyObject *o,
+     PyAPI_FUNC(PyObject *) _PyObject_CallMethodIdObjArgs(PyObject *o,
                                                struct _Py_Identifier *method,
                                                ...);
 
-
        /*
      Call the method named m of object o with a variable number of
      C arguments.  The C arguments are provided as PyObject *
@@ -404,8 +403,9 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
 #define PyObject_Length PyObject_Size
 
 #ifndef Py_LIMITED_API
-     PyAPI_FUNC(Py_ssize_t) _PyObject_LengthHint(PyObject *o, Py_ssize_t);
+     PyAPI_FUNC(int) _PyObject_HasLen(PyObject *o);
 #endif
+PyAPI_FUNC(Py_ssize_t) PyObject_LengthHint(PyObject *o, Py_ssize_t);
 
        /*
      Guess the size of object o using len(o) or o.__length_hint__().
diff --git a/Include/asdl.h b/Include/asdl.h
index 6bf618f3e9..0789ad870b 100644
--- a/Include/asdl.h
+++ b/Include/asdl.h
@@ -5,6 +5,7 @@ typedef PyObject * identifier;
 typedef PyObject * string;
 typedef PyObject * bytes;
 typedef PyObject * object;
+typedef PyObject * singleton;
 
 /* It would be nice if the code generated by asdl_c.py was completely
    independent of Python, but it is a goal the requires too much work
diff --git a/Include/dictobject.h b/Include/dictobject.h
index d89aac8932..16bd1d6cb1 100644
--- a/Include/dictobject.h
+++ b/Include/dictobject.h
@@ -53,6 +53,8 @@ PyAPI_FUNC(PyObject *) PyDict_GetItem(PyObject *mp, PyObject *key);
 PyAPI_FUNC(PyObject *) PyDict_GetItemWithError(PyObject *mp, PyObject *key);
 PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp,
                                                   struct _Py_Identifier *key);
+PyAPI_FUNC(PyObject *) PyDict_SetDefault(
+    PyObject *mp, PyObject *key, PyObject *defaultobj);
 PyAPI_FUNC(int) PyDict_SetItem(PyObject *mp, PyObject *key, PyObject *item);
 PyAPI_FUNC(int) PyDict_DelItem(PyObject *mp, PyObject *key);
 PyAPI_FUNC(void) PyDict_Clear(PyObject *mp);
diff --git a/Include/grammar.h b/Include/grammar.h
index 8426da30d9..862f6a8e6e 100644
--- a/Include/grammar.h
+++ b/Include/grammar.h
@@ -76,7 +76,7 @@ dfa *PyGrammar_FindDFA(grammar *g, int type);
 
 int addlabel(labellist *ll, int type, char *str);
 int findlabel(labellist *ll, int type, char *str);
-char *PyGrammar_LabelRepr(label *lb);
+const char *PyGrammar_LabelRepr(label *lb);
 void translatelabels(grammar *g);
 
 void addfirstsets(grammar *g);
diff --git a/Include/import.h b/Include/import.h
index fdc27336c3..90049e051b 100644
--- a/Include/import.h
+++ b/Include/import.h
@@ -113,15 +113,15 @@ PyAPI_FUNC(int) PyImport_AppendInittab(
 
 #ifndef Py_LIMITED_API
 struct _frozen {
-    char *name;                 /* ASCII encoded string */
-    unsigned char *code;
+    const char *name;                 /* ASCII encoded string */
+    const unsigned char *code;
     int size;
 };
 
 /* Embedding apps may change this pointer to point to their favorite
    collection of frozen modules: */
 
-PyAPI_DATA(struct _frozen *) PyImport_FrozenModules;
+PyAPI_DATA(const struct _frozen *) PyImport_FrozenModules;
 #endif
 
 #ifdef __cplusplus
diff --git a/Include/marshal.h b/Include/marshal.h
index e96d062b18..0c69655cf2 100644
--- a/Include/marshal.h
+++ b/Include/marshal.h
@@ -7,7 +7,7 @@
 extern "C" {
 #endif
 
-#define Py_MARSHAL_VERSION 2
+#define Py_MARSHAL_VERSION 3
 
 PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int);
 PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int);
diff --git a/Include/modsupport.h b/Include/modsupport.h
index ecf1dccc81..ab725f612a 100644
--- a/Include/modsupport.h
+++ b/Include/modsupport.h
@@ -26,7 +26,7 @@ PyAPI_FUNC(PyObject *) _Py_VaBuildValue_SizeT(const char *, va_list);
 /* Due to a glitch in 3.2, the _SizeT versions weren't exported from the DLL. */
 #if !defined(PY_SSIZE_T_CLEAN) || !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
 PyAPI_FUNC(int) PyArg_Parse(PyObject *, const char *, ...);
-PyAPI_FUNC(int) PyArg_ParseTuple(PyObject *, const char *, ...) Py_FORMAT_PARSETUPLE(PyArg_ParseTuple, 2, 3);
+PyAPI_FUNC(int) PyArg_ParseTuple(PyObject *, const char *, ...);
 PyAPI_FUNC(int) PyArg_ParseTupleAndKeywords(PyObject *, PyObject *,
                                                   const char *, char **, ...);
 PyAPI_FUNC(int) PyArg_ValidateKeywordArguments(PyObject *);
diff --git a/Include/object.h b/Include/object.h
index 387cadb4e4..ad58450fb9 100644
--- a/Include/object.h
+++ b/Include/object.h
@@ -362,7 +362,7 @@ typedef struct _typeobject {
     PyBufferProcs *tp_as_buffer;
 
     /* Flags to define presence of optional/expanded features */
-    long tp_flags;
+    unsigned long tp_flags;
 
     const char *tp_doc; /* Documentation string */
 
@@ -428,7 +428,7 @@ typedef struct{
     const char* name;
     int basicsize;
     int itemsize;
-    int flags;
+    unsigned int flags;
     PyType_Slot *slots; /* terminated by slot==0. */
 } PyType_Spec;
 
@@ -470,7 +470,7 @@ PyAPI_DATA(PyTypeObject) PyType_Type; /* built-in 'type' */
 PyAPI_DATA(PyTypeObject) PyBaseObject_Type; /* built-in 'object' */
 PyAPI_DATA(PyTypeObject) PySuper_Type; /* built-in 'super' */
 
-PyAPI_FUNC(long) PyType_GetFlags(PyTypeObject*);
+PyAPI_FUNC(unsigned long) PyType_GetFlags(PyTypeObject*);
 
 #define PyType_Check(op) \
     PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS)
@@ -603,44 +603,43 @@ given type object has a specified feature.
 */
 
 /* Set if the type object is dynamically allocated */
-#define Py_TPFLAGS_HEAPTYPE (1L<<9)
+#define Py_TPFLAGS_HEAPTYPE (1UL << 9)
 
 /* Set if the type allows subclassing */
-#define Py_TPFLAGS_BASETYPE (1L<<10)
+#define Py_TPFLAGS_BASETYPE (1UL << 10)
 
 /* Set if the type is 'ready' -- fully initialized */
-#define Py_TPFLAGS_READY (1L<<12)
+#define Py_TPFLAGS_READY (1UL << 12)
 
 /* Set while the type is being 'readied', to prevent recursive ready calls */
-#define Py_TPFLAGS_READYING (1L<<13)
+#define Py_TPFLAGS_READYING (1UL << 13)
 
 /* Objects support garbage collection (see objimp.h) */
-#define Py_TPFLAGS_HAVE_GC (1L<<14)
+#define Py_TPFLAGS_HAVE_GC (1UL << 14)
 
 /* These two bits are preserved for Stackless Python, next after this is 17 */
 #ifdef STACKLESS
-#define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION (3L<<15)
+#define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION (3UL << 15)
 #else
 #define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION 0
 #endif
 
 /* Objects support type attribute cache */
-#define Py_TPFLAGS_HAVE_VERSION_TAG   (1L<<18)
-#define Py_TPFLAGS_VALID_VERSION_TAG  (1L<<19)
+#define Py_TPFLAGS_HAVE_VERSION_TAG   (1UL << 18)
+#define Py_TPFLAGS_VALID_VERSION_TAG  (1UL << 19)
 
 /* Type is abstract and cannot be instantiated */
-#define Py_TPFLAGS_IS_ABSTRACT (1L<<20)
+#define Py_TPFLAGS_IS_ABSTRACT (1UL << 20)
 
 /* These flags are used to determine if a type is a subclass. */
-#define Py_TPFLAGS_INT_SUBCLASS         (1L<<23)
-#define Py_TPFLAGS_LONG_SUBCLASS        (1L<<24)
-#define Py_TPFLAGS_LIST_SUBCLASS        (1L<<25)
-#define Py_TPFLAGS_TUPLE_SUBCLASS       (1L<<26)
-#define Py_TPFLAGS_BYTES_SUBCLASS       (1L<<27)
-#define Py_TPFLAGS_UNICODE_SUBCLASS     (1L<<28)
-#define Py_TPFLAGS_DICT_SUBCLASS        (1L<<29)
-#define Py_TPFLAGS_BASE_EXC_SUBCLASS    (1L<<30)
-#define Py_TPFLAGS_TYPE_SUBCLASS        (1L<<31)
+#define Py_TPFLAGS_LONG_SUBCLASS        (1UL << 24)
+#define Py_TPFLAGS_LIST_SUBCLASS        (1UL << 25)
+#define Py_TPFLAGS_TUPLE_SUBCLASS       (1UL << 26)
+#define Py_TPFLAGS_BYTES_SUBCLASS       (1UL << 27)
+#define Py_TPFLAGS_UNICODE_SUBCLASS     (1UL << 28)
+#define Py_TPFLAGS_DICT_SUBCLASS        (1UL << 29)
+#define Py_TPFLAGS_BASE_EXC_SUBCLASS    (1UL << 30)
+#define Py_TPFLAGS_TYPE_SUBCLASS        (1UL << 31)
 
 #define Py_TPFLAGS_DEFAULT  ( \
                  Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | \
diff --git a/Include/objimpl.h b/Include/objimpl.h
index 3d5f509404..8f36360bbf 100644
--- a/Include/objimpl.h
+++ b/Include/objimpl.h
@@ -98,6 +98,8 @@ PyAPI_FUNC(void *) PyObject_Malloc(size_t);
 PyAPI_FUNC(void *) PyObject_Realloc(void *, size_t);
 PyAPI_FUNC(void) PyObject_Free(void *);
 
+/* This function returns the number of allocated memory blocks, regardless of size */
+PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void);
 
 /* Macros */
 #ifdef WITH_PYMALLOC
@@ -230,6 +232,10 @@ PyAPI_FUNC(PyVarObject *) _PyObject_NewVar(PyTypeObject *, Py_ssize_t);
 /* C equivalent of gc.collect(). */
 PyAPI_FUNC(Py_ssize_t) PyGC_Collect(void);
 
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(Py_ssize_t) _PyGC_CollectNoFail(void);
+#endif
+
 /* Test if a type has a GC head */
 #define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC)
 
@@ -249,7 +255,7 @@ typedef union _gc_head {
         union _gc_head *gc_prev;
         Py_ssize_t gc_refs;
     } gc;
-    long double dummy;  /* force worst-case alignment */
+    double dummy;  /* force worst-case alignment */
 } PyGC_Head;
 
 extern PyGC_Head *_PyGC_generation0;
diff --git a/Include/opcode.h b/Include/opcode.h
index a90184d0d7..0936f2d9f4 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -49,7 +49,6 @@ extern "C" {
 #define BINARY_OR       66
 #define INPLACE_POWER   67
 #define GET_ITER        68
-#define STORE_LOCALS    69
 #define PRINT_EXPR      70
 #define LOAD_BUILD_CLASS 71
 #define YIELD_FROM      72
@@ -140,6 +139,7 @@ extern "C" {
 #define SET_ADD         146
 #define MAP_ADD         147
 
+#define LOAD_CLASSDEREF   148
 
 /* EXCEPT_HANDLER is a special, implicit block type which is created when
    entering an except handler. It is not an opcode but we define it here
diff --git a/Include/osdefs.h b/Include/osdefs.h
index 05c0c8ea4a..0c2e34b879 100644
--- a/Include/osdefs.h
+++ b/Include/osdefs.h
@@ -9,16 +9,10 @@ extern "C" {
 
 /* Mod by chrish: QNX has WATCOM, but isn't DOS */
 #if !defined(__QNX__)
-#if defined(MS_WINDOWS) || defined(__BORLANDC__) || defined(__WATCOMC__) || defined(__DJGPP__) || defined(PYOS_OS2)
-#if defined(PYOS_OS2) && defined(PYCC_GCC)
-#define MAXPATHLEN 260
-#define SEP L'/'
-#define ALTSEP L'\\'
-#else
+#if defined(MS_WINDOWS) || defined(__BORLANDC__) || defined(__WATCOMC__) || defined(__DJGPP__)
 #define SEP L'\\'
 #define ALTSEP L'/'
 #define MAXPATHLEN 256
-#endif
 #define DELIM L';'
 #endif
 #endif
diff --git a/Include/patchlevel.h b/Include/patchlevel.h
index bfd67a5389..a759045682 100644
--- a/Include/patchlevel.h
+++ b/Include/patchlevel.h
@@ -17,13 +17,13 @@
 /* Version parsed out into numeric values */
 /*--start constants--*/
 #define PY_MAJOR_VERSION	3
-#define PY_MINOR_VERSION	3
-#define PY_MICRO_VERSION	2
-#define PY_RELEASE_LEVEL	PY_RELEASE_LEVEL_FINAL
+#define PY_MINOR_VERSION	4
+#define PY_MICRO_VERSION	0
+#define PY_RELEASE_LEVEL	PY_RELEASE_LEVEL_ALPHA
 #define PY_RELEASE_SERIAL	0
 
 /* Version as a string */
-#define PY_VERSION      	"3.3.2+"
+#define PY_VERSION      	"3.4.0a0"
 /*--end constants--*/
 
 /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2.
diff --git a/Include/pyport.h b/Include/pyport.h
index e4e3601d7b..96e8f1a5f8 100644
--- a/Include/pyport.h
+++ b/Include/pyport.h
@@ -392,9 +392,6 @@ typedef size_t Py_uhash_t;
 #endif
 
 #ifdef HAVE_SYS_STAT_H
-#if defined(PYOS_OS2) && defined(PYCC_GCC)
-#include <sys/types.h>
-#endif
 #include <sys/stat.h>
 #elif defined(HAVE_STAT_H)
 #include <stat.h>
@@ -835,15 +832,6 @@ extern pid_t forkpty(int *, char *, struct termios *, struct winsize *);
 #endif
 
 /*
- * Add PyArg_ParseTuple format where available.
- */
-#ifdef HAVE_ATTRIBUTE_FORMAT_PARSETUPLE
-#define Py_FORMAT_PARSETUPLE(func,p1,p2) __attribute__((format(func,p1,p2)))
-#else
-#define Py_FORMAT_PARSETUPLE(func,p1,p2)
-#endif
-
-/*
  * Specify alignment on compilers that support it.
  */
 #if defined(__GNUC__) && __GNUC__ >= 3
@@ -881,4 +869,18 @@ extern pid_t forkpty(int *, char *, struct termios *, struct winsize *);
 #endif
 #endif
 
+/*
+ * Convenient macros to deal with endianness of the platform. WORDS_BIGENDIAN is
+ * detected by configure and defined in pyconfig.h. The code in pyconfig.h
+ * also takes care of Apple's universal builds.
+ */
+
+#ifdef WORDS_BIGENDIAN
+#define PY_BIG_ENDIAN 1
+#define PY_LITTLE_ENDIAN 0
+#else
+#define PY_BIG_ENDIAN 0
+#define PY_LITTLE_ENDIAN 1
+#endif
+
 #endif /* Py_PYPORT_H */
diff --git a/Include/pystate.h b/Include/pystate.h
index 2017b0276f..cd1d7765be 100644
--- a/Include/pystate.h
+++ b/Include/pystate.h
@@ -69,6 +69,7 @@ typedef struct _ts PyThreadState;
 typedef struct _ts {
     /* See Python/ceval.c for comments explaining most fields */
 
+    struct _ts *prev;
     struct _ts *next;
     PyInterpreterState *interp;
 
@@ -139,6 +140,7 @@ PyAPI_FUNC(PyThreadState *) _PyThreadState_Prealloc(PyInterpreterState *);
 PyAPI_FUNC(void) _PyThreadState_Init(PyThreadState *);
 PyAPI_FUNC(void) PyThreadState_Clear(PyThreadState *);
 PyAPI_FUNC(void) PyThreadState_Delete(PyThreadState *);
+PyAPI_FUNC(void) _PyThreadState_DeleteExcept(PyThreadState *tstate);
 #ifdef WITH_THREAD
 PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void);
 PyAPI_FUNC(void) _PyGILState_Reinit(void);
@@ -212,6 +214,11 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE);
 */
 PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void);
 
+/* Helper/diagnostic function - return 1 if the current thread
+ * currently holds the GIL, 0 otherwise
+ */
+PyAPI_FUNC(int) PyGILState_Check(void);
+
 #endif   /* #ifdef WITH_THREAD */
 
 /* The implementation of sys._current_frames()  Returns a dict mapping
diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index e8a582d50a..66766dd0bb 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -217,6 +217,7 @@ PyAPI_FUNC(void) PyBytes_Fini(void);
 PyAPI_FUNC(void) PyByteArray_Fini(void);
 PyAPI_FUNC(void) PyFloat_Fini(void);
 PyAPI_FUNC(void) PyOS_FiniInterrupts(void);
+PyAPI_FUNC(void) _PyGC_DumpShutdownStats(void);
 PyAPI_FUNC(void) _PyGC_Fini(void);
 PyAPI_FUNC(void) PySlice_Fini(void);
 PyAPI_FUNC(void) _PyType_Fini(void);
diff --git a/Include/sliceobject.h b/Include/sliceobject.h
index 8bec17909d..f7ee90c3cc 100644
--- a/Include/sliceobject.h
+++ b/Include/sliceobject.h
@@ -34,6 +34,9 @@ PyAPI_FUNC(PyObject *) PySlice_New(PyObject* start, PyObject* stop,
                                   PyObject* step);
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(PyObject *) _PySlice_FromIndices(Py_ssize_t start, Py_ssize_t stop);
+PyAPI_FUNC(int) _PySlice_GetLongIndices(PySliceObject *self, PyObject *length,
+                                 PyObject **start_ptr, PyObject **stop_ptr,
+                                 PyObject **step_ptr);
 #endif
 PyAPI_FUNC(int) PySlice_GetIndices(PyObject *r, Py_ssize_t length,
                                   Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step);
diff --git a/Include/symtable.h b/Include/symtable.h
index 6ed3a2b06c..44535d0d95 100644
--- a/Include/symtable.h
+++ b/Include/symtable.h
@@ -41,6 +41,7 @@ typedef struct _symtable_entry {
     PyObject *ste_name;      /* string: name of current block */
     PyObject *ste_varnames;  /* list of function parameters */
     PyObject *ste_children;  /* list of child blocks */
+    PyObject *ste_directives;/* locations of global and nonlocal statements */
     _Py_block_ty ste_type;   /* module, class, or function */
     int ste_unoptimized;     /* false if namespace is optimized */
     int ste_nested;      /* true if block is nested */
@@ -52,6 +53,9 @@ typedef struct _symtable_entry {
     unsigned ste_varkeywords : 1; /* true if block has varkeywords */
     unsigned ste_returns_value : 1;  /* true if namespace uses return with
                                         an argument */
+    unsigned ste_needs_class_closure : 1; /* for class scopes, true if a
+                                             closure over __class__
+                                             should be created */
     int ste_lineno;          /* first line of block */
     int ste_col_offset;      /* offset of first line of block */
     int ste_opt_lineno;      /* lineno of last exec or import * */
diff --git a/Include/token.h b/Include/token.h
index f7f650456e..905022b8d2 100644
--- a/Include/token.h
+++ b/Include/token.h
@@ -75,7 +75,7 @@ extern "C" {
 #define ISEOF(x)		((x) == ENDMARKER)
 
 
-PyAPI_DATA(char *) _PyParser_TokenNames[]; /* Token names */
+PyAPI_DATA(const char *) _PyParser_TokenNames[]; /* Token names */
 PyAPI_FUNC(int) PyToken_OneChar(int);
 PyAPI_FUNC(int) PyToken_TwoChars(int, int);
 PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int);
diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h
index a8f5b5df36..2abc9b469a 100644
--- a/Include/unicodeobject.h
+++ b/Include/unicodeobject.h
@@ -180,9 +180,9 @@ typedef unsigned char Py_UCS1;
     } while (0)
 
 /* macros to work with surrogates */
-#define Py_UNICODE_IS_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDFFF)
-#define Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDBFF)
-#define Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= ch && ch <= 0xDFFF)
+#define Py_UNICODE_IS_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDFFF)
+#define Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDBFF)
+#define Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= (ch) && (ch) <= 0xDFFF)
 /* Join two surrogate characters and return a single Py_UCS4 value. */
 #define Py_UNICODE_JOIN_SURROGATES(high, low)  \
     (((((Py_UCS4)(high) & 0x03FF) << 10) |      \
@@ -898,22 +898,28 @@ typedef struct {
     Py_UCS4 maxchar;
     Py_ssize_t size;
     Py_ssize_t pos;
-    /* minimum length of the buffer when overallocation is enabled,
-       see _PyUnicodeWriter_Init() */
+
+    /* minimum number of allocated characters (default: 0) */
     Py_ssize_t min_length;
+
+    /* minimum character (default: 127, ASCII) */
+    Py_UCS4 min_char;
+
+    /* If non-zero, overallocate the buffer by 25% (default: 0). */
     unsigned char overallocate;
+
     /* If readonly is 1, buffer is a shared string (cannot be modified)
        and size is set to 0. */
     unsigned char readonly;
 } _PyUnicodeWriter ;
 
 /* Initialize a Unicode writer.
-
-   If min_length is greater than zero, _PyUnicodeWriter_Prepare()
-   overallocates the buffer and min_length is the minimum length in characters
-   of the buffer. */
+ *
+ * By default, the minimum buffer size is 0 character and overallocation is
+ * disabled. Set min_length, min_char and overallocate attributes to control
+ * the allocation of the buffer. */
 PyAPI_FUNC(void)
-_PyUnicodeWriter_Init(_PyUnicodeWriter *writer, Py_ssize_t min_length);
+_PyUnicodeWriter_Init(_PyUnicodeWriter *writer);
 
 /* Prepare the buffer to write 'length' characters
    with the specified maximum character.
@@ -933,12 +939,44 @@ PyAPI_FUNC(int)
 _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer,
                                  Py_ssize_t length, Py_UCS4 maxchar);
 
+/* Append a Unicode character.
+   Return 0 on success, raise an exception and return -1 on error. */
+PyAPI_FUNC(int)
+_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer,
+    Py_UCS4 ch
+    );
+
+/* Append a Unicode string.
+   Return 0 on success, raise an exception and return -1 on error. */
+PyAPI_FUNC(int)
+_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer,
+    PyObject *str               /* Unicode string */
+    );
+
+/* Append a substring of a Unicode string.
+   Return 0 on success, raise an exception and return -1 on error. */
 PyAPI_FUNC(int)
-_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str);
+_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer,
+    PyObject *str,              /* Unicode string */
+    Py_ssize_t start,
+    Py_ssize_t end
+    );
 
+/* Append a latin1-encoded byte string.
+   Return 0 on success, raise an exception and return -1 on error. */
+PyAPI_FUNC(int)
+_PyUnicodeWriter_WriteCstr(_PyUnicodeWriter *writer,
+    const char *str,            /* latin1-encoded byte string */
+    Py_ssize_t len              /* length in bytes */
+    );
+
+/* Get the value of the writer as an Unicode string. Clear the
+   buffer of the writer. Raise an exception and return NULL
+   on error. */
 PyAPI_FUNC(PyObject *)
 _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer);
 
+/* Deallocate memory of a writer (clear its internal buffer). */
 PyAPI_FUNC(void)
 _PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer);
 #endif
@@ -1726,7 +1764,7 @@ PyAPI_FUNC(PyObject*) PyUnicode_DecodeLocale(
 /* Encode a Unicode object to the current locale encoding. The encoder is
    strict is *surrogateescape* is equal to zero, otherwise the
    "surrogateescape" error handler is used. Return a bytes object. The string
-   cannot contain embedded null characters.. */
+   cannot contain embedded null characters. */
 
 PyAPI_FUNC(PyObject*) PyUnicode_EncodeLocale(
     PyObject *unicode,
@@ -1950,7 +1988,8 @@ PyAPI_FUNC(PyObject *) PyUnicode_Replace(
     );
 
 /* Compare two strings and return -1, 0, 1 for less than, equal,
-   greater than resp. */
+   greater than resp.
+   Raise an exception and return -1 on error. */
 
 PyAPI_FUNC(int) PyUnicode_Compare(
     PyObject *left,             /* Left string */
diff --git a/Include/warnings.h b/Include/warnings.h
index b7db681cc4..5a7dfb1bb2 100644
--- a/Include/warnings.h
+++ b/Include/warnings.h
@@ -25,6 +25,12 @@ PyAPI_FUNC(int) PyErr_WarnExplicit(
     const char *module,         /* UTF-8 encoded string */
     PyObject *registry);
 
+PyAPI_FUNC(int)
+PyErr_WarnExplicitFormat(PyObject *category,
+                         const char *filename, int lineno,
+                         const char *module, PyObject *registry,
+                         const char *format, ...);
+
 /* DEPRECATED: Use PyErr_WarnEx() instead. */
 #ifndef Py_LIMITED_API
 #define PyErr_Warn(category, msg) PyErr_WarnEx(category, msg, 1)