From e409cb0905a799d6de9b91b82b0bb5648b12aaec Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@gmail.com>
Date: Wed, 14 May 2014 17:24:35 +0200
Subject: Issue #21490: Add new C macros: Py_ABS() and Py_STRINGIFY()

Keep _Py_STRINGIZE() in PC/pyconfig.h to not introduce a dependency between
pyconfig.h and pymacros.h.
---
 Python/marshal.c | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index dc5411c1ff..ca64be3948 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -13,8 +13,6 @@
 #include "code.h"
 #include "marshal.h"
 
-#define ABS(x) ((x) < 0 ? -(x) : (x))
-
 /* High water mark to determine when the marshalled object is dangerously deep
  * and risks coring the interpreter.  When the object stack gets this deep,
  * raise an exception instead of continuing.
@@ -192,7 +190,7 @@ w_PyLong(const PyLongObject *ob, char flag, WFILE *p)
     }
 
     /* set l to number of base PyLong_MARSHAL_BASE digits */
-    n = ABS(Py_SIZE(ob));
+    n = Py_ABS(Py_SIZE(ob));
     l = (n-1) * PyLong_MARSHAL_RATIO;
     d = ob->ob_digit[n-1];
     assert(d != 0); /* a PyLong is always normalized */
@@ -727,8 +725,8 @@ r_PyLong(RFILE *p)
         return NULL;
     }
 
-    size = 1 + (ABS(n) - 1) / PyLong_MARSHAL_RATIO;
-    shorts_in_top_digit = 1 + (ABS(n) - 1) % PyLong_MARSHAL_RATIO;
+    size = 1 + (Py_ABS(n) - 1) / PyLong_MARSHAL_RATIO;
+    shorts_in_top_digit = 1 + (Py_ABS(n) - 1) % PyLong_MARSHAL_RATIO;
     ob = _PyLong_New(size);
     if (ob == NULL)
         return NULL;
-- 
cgit v1.2.1


From 43e4c1002ed31cf8c749211fa184fc7130f2cb75 Mon Sep 17 00:00:00 2001
From: Steve Dower <steve.dower@microsoft.com>
Date: Sat, 1 Nov 2014 15:15:16 -0700
Subject: #22734 marshal needs a lower stack depth for debug builds on Windows

---
 Python/marshal.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index ca64be3948..837aa4812c 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -19,7 +19,7 @@
  * On Windows debug builds, reduce this value.
  */
 #if defined(MS_WINDOWS) && defined(_DEBUG)
-#define MAX_MARSHAL_STACK_DEPTH 1500
+#define MAX_MARSHAL_STACK_DEPTH 1000
 #else
 #define MAX_MARSHAL_STACK_DEPTH 2000
 #endif
-- 
cgit v1.2.1


From aa8124145bc571bdb27999b7ba0a711229284090 Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka <storchaka@gmail.com>
Date: Wed, 11 Feb 2015 15:53:31 +0200
Subject: Issue #20416: marshal.dumps() with protocols 3 and 4 is now 40-50%
 faster on average.

---
 Python/marshal.c | 95 ++++++++++++++++++++++++++++++++------------------------
 1 file changed, 55 insertions(+), 40 deletions(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index 5acf0deeb4..46b87c2458 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -12,6 +12,7 @@
 #include "longintrepr.h"
 #include "code.h"
 #include "marshal.h"
+#include "../Modules/hashtable.h"
 
 /* High water mark to determine when the marshalled object is dangerously deep
  * and risks coring the interpreter.  When the object stack gets this deep,
@@ -73,6 +74,7 @@ typedef struct {
     char *buf;
     Py_ssize_t buf_size;
     PyObject *refs; /* dict on marshal, list on unmarshal */
+    _Py_hashtable_t *hashtable;
     int version;
 } WFILE;
 
@@ -223,46 +225,38 @@ w_PyLong(const PyLongObject *ob, char flag, WFILE *p)
 static int
 w_ref(PyObject *v, char *flag, WFILE *p)
 {
-    PyObject *id;
-    PyObject *idx;
+    _Py_hashtable_entry_t *entry;
+    int w;
 
-    if (p->version < 3 || p->refs == NULL)
+    if (p->version < 3 || p->hashtable == NULL)
         return 0; /* not writing object references */
 
     /* if it has only one reference, it definitely isn't shared */
     if (Py_REFCNT(v) == 1)
         return 0;
 
-    id = PyLong_FromVoidPtr((void*)v);
-    if (id == NULL)
-        goto err;
-    idx = PyDict_GetItem(p->refs, id);
-    if (idx != NULL) {
+    entry = _Py_hashtable_get_entry(p->hashtable, v);
+    if (entry != NULL) {
         /* write the reference index to the stream */
-        long w = PyLong_AsLong(idx);
-        Py_DECREF(id);
-        if (w == -1 && PyErr_Occurred()) {
-            goto err;
-        }
+        _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, &w, sizeof(w), entry);
         /* we don't store "long" indices in the dict */
         assert(0 <= w && w <= 0x7fffffff);
         w_byte(TYPE_REF, p);
         w_long(w, p);
         return 1;
     } else {
-        int ok;
-        Py_ssize_t s = PyDict_Size(p->refs);
+        size_t s = p->hashtable->entries;
         /* we don't support long indices */
         if (s >= 0x7fffffff) {
             PyErr_SetString(PyExc_ValueError, "too many objects");
             goto err;
         }
-        idx = PyLong_FromSsize_t(s);
-        ok = idx && PyDict_SetItem(p->refs, id, idx) == 0;
-        Py_DECREF(id);
-        Py_XDECREF(idx);
-        if (!ok)
+        w = s;
+        Py_INCREF(v);
+        if (_Py_HASHTABLE_SET(p->hashtable, v, w) < 0) {
+            Py_DECREF(v);
             goto err;
+        }
         *flag |= FLAG_REF;
         return 0;
     }
@@ -545,15 +539,44 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
     }
 }
 
+static int
+w_init_refs(WFILE *wf, int version)
+{
+    if (version >= 3) {
+        wf->hashtable = _Py_hashtable_new(sizeof(int), _Py_hashtable_hash_ptr,
+                                          _Py_hashtable_compare_direct);
+        if (wf->hashtable == NULL) {
+            PyErr_NoMemory();
+            return -1;
+        }
+    }
+    return 0;
+}
+
+static int
+w_decref_entry(_Py_hashtable_entry_t *entry, void *Py_UNUSED(data))
+{
+    Py_XDECREF(entry->key);
+    return 0;
+}
+
+static void
+w_clear_refs(WFILE *wf)
+{
+    if (wf->hashtable != NULL) {
+        _Py_hashtable_foreach(wf->hashtable, w_decref_entry, NULL);
+        _Py_hashtable_destroy(wf->hashtable);
+    }
+}
+
 /* version currently has no effect for writing ints. */
 void
 PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
 {
     WFILE wf;
+    memset(&wf, 0, sizeof(wf));
     wf.fp = fp;
     wf.error = WFERR_OK;
-    wf.depth = 0;
-    wf.refs = NULL;
     wf.version = version;
     w_long(x, &wf);
 }
@@ -562,17 +585,14 @@ void
 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
 {
     WFILE wf;
+    memset(&wf, 0, sizeof(wf));
     wf.fp = fp;
     wf.error = WFERR_OK;
-    wf.depth = 0;
-    if (version >= 3) {
-        if ((wf.refs = PyDict_New()) == NULL)
-            return; /* caller mush check PyErr_Occurred() */
-    } else
-        wf.refs = NULL;
     wf.version = version;
+    if (w_init_refs(&wf, version))
+        return; /* caller mush check PyErr_Occurred() */
     w_object(x, &wf);
-    Py_XDECREF(wf.refs);
+    w_clear_refs(&wf);
 }
 
 typedef WFILE RFILE; /* Same struct with different invariants */
@@ -1509,25 +1529,20 @@ PyMarshal_WriteObjectToString(PyObject *x, int version)
 {
     WFILE wf;
 
-    wf.fp = NULL;
-    wf.readable = NULL;
+    memset(&wf, 0, sizeof(wf));
     wf.str = PyBytes_FromStringAndSize((char *)NULL, 50);
     if (wf.str == NULL)
         return NULL;
     wf.ptr = PyBytes_AS_STRING((PyBytesObject *)wf.str);
     wf.end = wf.ptr + PyBytes_Size(wf.str);
     wf.error = WFERR_OK;
-    wf.depth = 0;
     wf.version = version;
-    if (version >= 3) {
-        if ((wf.refs = PyDict_New()) == NULL) {
-            Py_DECREF(wf.str);
-            return NULL;
-        }
-    } else
-        wf.refs = NULL;
+    if (w_init_refs(&wf, version)) {
+        Py_DECREF(wf.str);
+        return NULL;
+    }
     w_object(x, &wf);
-    Py_XDECREF(wf.refs);
+    w_clear_refs(&wf);
     if (wf.str != NULL) {
         char *base = PyBytes_AS_STRING((PyBytesObject *)wf.str);
         if (wf.ptr - base > PY_SSIZE_T_MAX) {
-- 
cgit v1.2.1


From 096860e18bc5a8805850a4f82188206f15901ff9 Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka <storchaka@gmail.com>
Date: Wed, 11 Feb 2015 15:54:54 +0200
Subject: Issue #23344: marshal.dumps() is now 20-25% faster on average.

---
 Python/marshal.c | 83 ++++++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 62 insertions(+), 21 deletions(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index 46b87c2458..51d35c2295 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -78,42 +78,75 @@ typedef struct {
     int version;
 } WFILE;
 
-#define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
-                      else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
-                           else w_more((c), p)
+#define w_byte(c, p) do {                               \
+        if ((p)->ptr != (p)->end || w_reserve((p), 1))  \
+            *(p)->ptr++ = (c);                          \
+    } while(0)
 
 static void
-w_more(char c, WFILE *p)
+w_flush(WFILE *p)
+{
+    assert(p->fp != NULL);
+    fwrite(p->buf, 1, p->ptr - p->buf, p->fp);
+    p->ptr = p->buf;
+}
+
+static int
+w_reserve(WFILE *p, Py_ssize_t needed)
 {
-    Py_ssize_t size, newsize;
-    if (p->str == NULL)
-        return; /* An error already occurred */
+    Py_ssize_t pos, size, delta;
+    if (p->ptr == NULL)
+        return 0; /* An error already occurred */
+    if (p->fp != NULL) {
+        w_flush(p);
+        return needed <= p->end - p->ptr;
+    }
+    assert(p->str != NULL);
+    pos = p->ptr - p->buf;
     size = PyBytes_Size(p->str);
-    newsize = size + size + 1024;
-    if (newsize > 32*1024*1024) {
-        newsize = size + (size >> 3);           /* 12.5% overallocation */
+    if (size > 16*1024*1024)
+        delta = (size >> 3);            /* 12.5% overallocation */
+    else
+        delta = size + 1024;
+    delta = Py_MAX(delta, needed);
+    if (delta > PY_SSIZE_T_MAX - size) {
+        p->error = WFERR_NOMEMORY;
+        return 0;
     }
-    if (_PyBytes_Resize(&p->str, newsize) != 0) {
-        p->ptr = p->end = NULL;
+    size += delta;
+    if (_PyBytes_Resize(&p->str, size) != 0) {
+        p->ptr = p->buf = p->end = NULL;
+        return 0;
     }
     else {
-        p->ptr = PyBytes_AS_STRING((PyBytesObject *)p->str) + size;
-        p->end =
-            PyBytes_AS_STRING((PyBytesObject *)p->str) + newsize;
-        *p->ptr++ = c;
+        p->buf = PyBytes_AS_STRING(p->str);
+        p->ptr = p->buf + pos;
+        p->end = p->buf + size;
+        return 1;
     }
 }
 
 static void
 w_string(const char *s, Py_ssize_t n, WFILE *p)
 {
+    Py_ssize_t m;
+    if (!n || p->ptr == NULL)
+        return;
+    m = p->end - p->ptr;
     if (p->fp != NULL) {
-        fwrite(s, 1, n, p->fp);
+        if (n <= m) {
+            Py_MEMCPY(p->ptr, s, n);
+            p->ptr += n;
+        }
+        else {
+            w_flush(p);
+            fwrite(s, 1, n, p->fp);
+        }
     }
     else {
-        while (--n >= 0) {
-            w_byte(*s, p);
-            s++;
+        if (n <= m || w_reserve(p, n - m)) {
+            Py_MEMCPY(p->ptr, s, n);
+            p->ptr += n;
         }
     }
 }
@@ -573,26 +606,34 @@ w_clear_refs(WFILE *wf)
 void
 PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
 {
+    char buf[4];
     WFILE wf;
     memset(&wf, 0, sizeof(wf));
     wf.fp = fp;
+    wf.ptr = wf.buf = buf;
+    wf.end = wf.ptr + sizeof(buf);
     wf.error = WFERR_OK;
     wf.version = version;
     w_long(x, &wf);
+    w_flush(&wf);
 }
 
 void
 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
 {
+    char buf[BUFSIZ];
     WFILE wf;
     memset(&wf, 0, sizeof(wf));
     wf.fp = fp;
+    wf.ptr = wf.buf = buf;
+    wf.end = wf.ptr + sizeof(buf);
     wf.error = WFERR_OK;
     wf.version = version;
     if (w_init_refs(&wf, version))
         return; /* caller mush check PyErr_Occurred() */
     w_object(x, &wf);
     w_clear_refs(&wf);
+    w_flush(&wf);
 }
 
 typedef WFILE RFILE; /* Same struct with different invariants */
@@ -1533,7 +1574,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version)
     wf.str = PyBytes_FromStringAndSize((char *)NULL, 50);
     if (wf.str == NULL)
         return NULL;
-    wf.ptr = PyBytes_AS_STRING((PyBytesObject *)wf.str);
+    wf.ptr = wf.buf = PyBytes_AS_STRING((PyBytesObject *)wf.str);
     wf.end = wf.ptr + PyBytes_Size(wf.str);
     wf.error = WFERR_OK;
     wf.version = version;
-- 
cgit v1.2.1


From 110a5da3db3f8efa352cd0701bb104452a1544aa Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka <storchaka@gmail.com>
Date: Wed, 11 Feb 2015 16:18:09 +0200
Subject: Splitted the WFILE structure to WFILE and RFILE.

---
 Python/marshal.c | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index 51d35c2295..c1ce2fe8d6 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -65,15 +65,10 @@ typedef struct {
     FILE *fp;
     int error;  /* see WFERR_* values */
     int depth;
-    /* If fp == NULL, the following are valid: */
-    PyObject *readable;    /* Stream-like object being read from */
     PyObject *str;
-    PyObject *current_filename;
     char *ptr;
     char *end;
     char *buf;
-    Py_ssize_t buf_size;
-    PyObject *refs; /* dict on marshal, list on unmarshal */
     _Py_hashtable_t *hashtable;
     int version;
 } WFILE;
@@ -636,7 +631,17 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
     w_flush(&wf);
 }
 
-typedef WFILE RFILE; /* Same struct with different invariants */
+typedef struct {
+    FILE *fp;
+    int depth;
+    PyObject *readable;  /* Stream-like object being read from */
+    PyObject *current_filename;
+    char *ptr;
+    char *end;
+    char *buf;
+    Py_ssize_t buf_size;
+    PyObject *refs;  /* a list */
+} RFILE;
 
 static char *
 r_string(Py_ssize_t n, RFILE *p)
-- 
cgit v1.2.1


From d53d28906ce286782125ad3ebfc3a26faeb7c08f Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka <storchaka@gmail.com>
Date: Mon, 16 Feb 2015 20:52:17 +0200
Subject: Issue #23450: Fixed possible integer overflows.

---
 Python/marshal.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index c1ce2fe8d6..d26a997534 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -279,7 +279,7 @@ w_ref(PyObject *v, char *flag, WFILE *p)
             PyErr_SetString(PyExc_ValueError, "too many objects");
             goto err;
         }
-        w = s;
+        w = (int)s;
         Py_INCREF(v);
         if (_Py_HASHTABLE_SET(p->hashtable, v, w) < 0) {
             Py_DECREF(v);
-- 
cgit v1.2.1


From 4b5c8edd294f02afef7407d9fd5236789bd4bc34 Mon Sep 17 00:00:00 2001
From: Steve Dower <steve.dower@microsoft.com>
Date: Sat, 21 Feb 2015 08:44:05 -0800
Subject: Issue #23152: Implement _Py_fstat() to support files larger than 2 GB
 on Windows. fstat() may fail with EOVERFLOW on files larger than 2 GB because
 the file size type is an signed 32-bit integer.

---
 Python/marshal.c | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index d26a997534..93d9d1d477 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -1481,16 +1481,20 @@ PyMarshal_ReadLongFromFile(FILE *fp)
     return res;
 }
 
-#ifdef HAVE_FSTAT
-/* Return size of file in bytes; < 0 if unknown. */
+#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
+/* Return size of file in bytes; < 0 if unknown or INT_MAX if too big */
 static off_t
 getfilesize(FILE *fp)
 {
-    struct stat st;
-    if (fstat(fileno(fp), &st) != 0)
+    struct _Py_stat_struct st;
+    if (_Py_fstat(fileno(fp), &st) != 0)
         return -1;
+#if SIZEOF_OFF_T == 4
+    else if (st.st_size >= INT_MAX)
+        return (off_t)INT_MAX;
+#endif
     else
-        return st.st_size;
+        return (off_t)st.st_size;
 }
 #endif
 
@@ -1505,7 +1509,7 @@ PyMarshal_ReadLastObjectFromFile(FILE *fp)
 {
 /* REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc. */
 #define REASONABLE_FILE_LIMIT (1L << 18)
-#ifdef HAVE_FSTAT
+#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
     off_t filesize;
     filesize = getfilesize(fp);
     if (filesize > 0 && filesize <= REASONABLE_FILE_LIMIT) {
-- 
cgit v1.2.1


From eaf93d5dbe0a159d1d0908fab4f627934abbe6b1 Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@gmail.com>
Date: Tue, 24 Mar 2015 10:27:50 +0100
Subject: Issue #23753: Python doesn't support anymore platforms without stat()
 or fstat(), these functions are always required.

Remove HAVE_STAT and HAVE_FSTAT defines, and stop supporting DONT_HAVE_STAT and
DONT_HAVE_FSTAT.
---
 Python/marshal.c | 4 ----
 1 file changed, 4 deletions(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index 93d9d1d477..3472882fb5 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -1481,7 +1481,6 @@ PyMarshal_ReadLongFromFile(FILE *fp)
     return res;
 }
 
-#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
 /* Return size of file in bytes; < 0 if unknown or INT_MAX if too big */
 static off_t
 getfilesize(FILE *fp)
@@ -1496,7 +1495,6 @@ getfilesize(FILE *fp)
     else
         return (off_t)st.st_size;
 }
-#endif
 
 /* If we can get the size of the file up-front, and it's reasonably small,
  * read it in one gulp and delegate to ...FromString() instead.  Much quicker
@@ -1509,7 +1507,6 @@ PyMarshal_ReadLastObjectFromFile(FILE *fp)
 {
 /* REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc. */
 #define REASONABLE_FILE_LIMIT (1L << 18)
-#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
     off_t filesize;
     filesize = getfilesize(fp);
     if (filesize > 0 && filesize <= REASONABLE_FILE_LIMIT) {
@@ -1522,7 +1519,6 @@ PyMarshal_ReadLastObjectFromFile(FILE *fp)
         }
 
     }
-#endif
     /* We don't have fstat, or we do but the file is larger than
      * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
      */
-- 
cgit v1.2.1


From 0600ebc4d9a0fbb0f16689fb2c1d6a09482594cc Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@gmail.com>
Date: Mon, 30 Mar 2015 10:09:31 +0200
Subject: Issue #23752: _Py_fstat() is now responsible to raise the Python
 exception

Add _Py_fstat_noraise() function when a Python exception is not welcome.
---
 Python/marshal.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index 3472882fb5..f89cd04b65 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -1486,7 +1486,7 @@ static off_t
 getfilesize(FILE *fp)
 {
     struct _Py_stat_struct st;
-    if (_Py_fstat(fileno(fp), &st) != 0)
+    if (_Py_fstat_noraise(fileno(fp), &st) != 0)
         return -1;
 #if SIZEOF_OFF_T == 4
     else if (st.st_size >= INT_MAX)
-- 
cgit v1.2.1


From 1bfa33ccd1e1c7b69b7258aca538e27fb94cc582 Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka <storchaka@gmail.com>
Date: Fri, 18 Dec 2015 13:05:04 +0200
Subject: Issue #25899: Converted non-ASCII characters in docstrings and
 manpage to ASCII replacements.  Original patch by Chris Angelico.

---
 Python/marshal.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index f89cd04b65..5b8de99192 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -1644,7 +1644,7 @@ The file must be an open file object such as sys.stdout or returned by\n\
 open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
 \n\
 If the value has (or contains an object that has) an unsupported type, a\n\
-ValueError exception is raised — but garbage data will also be written\n\
+ValueError exception is raised - but garbage data will also be written\n\
 to the file. The object will not be properly read back by load()\n\
 \n\
 The version argument indicates the data format that dump should use.");
@@ -1695,7 +1695,7 @@ PyDoc_STRVAR(load_doc,
 "load(file)\n\
 \n\
 Read one value from the open file and return it. If no valid value is\n\
-read (e.g. because the data has a different Python version’s\n\
+read (e.g. because the data has a different Python version's\n\
 incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
 The file must be an open file object opened in binary mode ('rb' or\n\
 'r+b').\n\
-- 
cgit v1.2.1


From 5c633fc25fc6f9beee3ccb9ebc06633d1c9a1bc3 Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka <storchaka@gmail.com>
Date: Fri, 25 Dec 2015 20:01:53 +0200
Subject: Issue #25923: Added more const qualifiers to signatures of static and
 private functions.

---
 Python/marshal.c | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index 5b8de99192..589eb8065f 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -643,7 +643,7 @@ typedef struct {
     PyObject *refs;  /* a list */
 } RFILE;
 
-static char *
+static const char *
 r_string(Py_ssize_t n, RFILE *p)
 {
     Py_ssize_t read = -1;
@@ -729,7 +729,7 @@ r_byte(RFILE *p)
         c = getc(p->fp);
     }
     else {
-        char *ptr = r_string(1, p);
+        const char *ptr = r_string(1, p);
         if (ptr != NULL)
             c = *(unsigned char *) ptr;
     }
@@ -740,9 +740,9 @@ static int
 r_short(RFILE *p)
 {
     short x = -1;
-    unsigned char *buffer;
+    const unsigned char *buffer;
 
-    buffer = (unsigned char *) r_string(2, p);
+    buffer = (const unsigned char *) r_string(2, p);
     if (buffer != NULL) {
         x = buffer[0];
         x |= buffer[1] << 8;
@@ -756,9 +756,9 @@ static long
 r_long(RFILE *p)
 {
     long x = -1;
-    unsigned char *buffer;
+    const unsigned char *buffer;
 
-    buffer = (unsigned char *) r_string(4, p);
+    buffer = (const unsigned char *) r_string(4, p);
     if (buffer != NULL) {
         x = buffer[0];
         x |= (long)buffer[1] << 8;
@@ -978,7 +978,8 @@ r_object(RFILE *p)
 
     case TYPE_FLOAT:
         {
-            char buf[256], *ptr;
+            char buf[256];
+            const char *ptr;
             double dx;
             n = r_byte(p);
             if (n == EOF) {
@@ -1001,9 +1002,9 @@ r_object(RFILE *p)
 
     case TYPE_BINARY_FLOAT:
         {
-            unsigned char *buf;
+            const unsigned char *buf;
             double x;
-            buf = (unsigned char *) r_string(8, p);
+            buf = (const unsigned char *) r_string(8, p);
             if (buf == NULL)
                 break;
             x = _PyFloat_Unpack8(buf, 1);
@@ -1016,7 +1017,8 @@ r_object(RFILE *p)
 
     case TYPE_COMPLEX:
         {
-            char buf[256], *ptr;
+            char buf[256];
+            const char *ptr;
             Py_complex c;
             n = r_byte(p);
             if (n == EOF) {
@@ -1053,15 +1055,15 @@ r_object(RFILE *p)
 
     case TYPE_BINARY_COMPLEX:
         {
-            unsigned char *buf;
+            const unsigned char *buf;
             Py_complex c;
-            buf = (unsigned char *) r_string(8, p);
+            buf = (const unsigned char *) r_string(8, p);
             if (buf == NULL)
                 break;
             c.real = _PyFloat_Unpack8(buf, 1);
             if (c.real == -1.0 && PyErr_Occurred())
                 break;
-            buf = (unsigned char *) r_string(8, p);
+            buf = (const unsigned char *) r_string(8, p);
             if (buf == NULL)
                 break;
             c.imag = _PyFloat_Unpack8(buf, 1);
@@ -1074,7 +1076,7 @@ r_object(RFILE *p)
 
     case TYPE_STRING:
         {
-            char *ptr;
+            const char *ptr;
             n = r_long(p);
             if (PyErr_Occurred())
                 break;
@@ -1119,7 +1121,7 @@ r_object(RFILE *p)
         }
     _read_ascii:
         {
-            char *ptr;
+            const char *ptr;
             ptr = r_string(n, p);
             if (ptr == NULL)
                 break;
@@ -1137,7 +1139,7 @@ r_object(RFILE *p)
         is_interned = 1;
     case TYPE_UNICODE:
         {
-        char *buffer;
+        const char *buffer;
 
         n = r_long(p);
         if (PyErr_Occurred())
-- 
cgit v1.2.1


From bfc057fcb43946cd16d923e044e79185fb82d74c Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@gmail.com>
Date: Sat, 23 Jan 2016 14:15:48 +0100
Subject: Issue #26146: marshal.loads() now uses the empty frozenset singleton

---
 Python/marshal.c | 69 ++++++++++++++++++++++++++++++++------------------------
 1 file changed, 40 insertions(+), 29 deletions(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index 589eb8065f..7a4b9d29b4 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -1266,41 +1266,52 @@ r_object(RFILE *p)
             PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)");
             break;
         }
-        v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
-        if (type == TYPE_SET) {
-            R_REF(v);
-        } else {
-            /* must use delayed registration of frozensets because they must
-             * be init with a refcount of 1
-             */
-            idx = r_ref_reserve(flag, p);
-            if (idx < 0)
-                Py_CLEAR(v); /* signal error */
-        }
-        if (v == NULL)
-            break;
 
-        for (i = 0; i < n; i++) {
-            v2 = r_object(p);
-            if ( v2 == NULL ) {
-                if (!PyErr_Occurred())
-                    PyErr_SetString(PyExc_TypeError,
-                        "NULL object in marshal data for set");
-                Py_DECREF(v);
-                v = NULL;
+        if (n == 0 && type == TYPE_FROZENSET) {
+            /* call frozenset() to get the empty frozenset singleton */
+            v = PyObject_CallFunction((PyObject*)&PyFrozenSet_Type, NULL);
+            if (v == NULL)
                 break;
+            R_REF(v);
+            retval = v;
+        }
+        else {
+            v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
+            if (type == TYPE_SET) {
+                R_REF(v);
+            } else {
+                /* must use delayed registration of frozensets because they must
+                 * be init with a refcount of 1
+                 */
+                idx = r_ref_reserve(flag, p);
+                if (idx < 0)
+                    Py_CLEAR(v); /* signal error */
             }
-            if (PySet_Add(v, v2) == -1) {
-                Py_DECREF(v);
-                Py_DECREF(v2);
-                v = NULL;
+            if (v == NULL)
                 break;
+
+            for (i = 0; i < n; i++) {
+                v2 = r_object(p);
+                if ( v2 == NULL ) {
+                    if (!PyErr_Occurred())
+                        PyErr_SetString(PyExc_TypeError,
+                            "NULL object in marshal data for set");
+                    Py_DECREF(v);
+                    v = NULL;
+                    break;
+                }
+                if (PySet_Add(v, v2) == -1) {
+                    Py_DECREF(v);
+                    Py_DECREF(v2);
+                    v = NULL;
+                    break;
+                }
+                Py_DECREF(v2);
             }
-            Py_DECREF(v2);
+            if (type != TYPE_SET)
+                v = r_ref_insert(v, idx, flag, p);
+            retval = v;
         }
-        if (type != TYPE_SET)
-            v = r_ref_insert(v, idx, flag, p);
-        retval = v;
         break;
 
     case TYPE_CODE:
-- 
cgit v1.2.1


From 480900011c18266464da20e649cb5fcd6e0b759e Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@gmail.com>
Date: Fri, 18 Mar 2016 21:52:22 +0100
Subject: _tracemalloc: add domain to trace keys

* hashtable.h: key has now a variable size
* _tracemalloc uses (pointer: void*, domain: unsigned int) as key for traces
---
 Python/marshal.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index 7a4b9d29b4..64084f4703 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -263,7 +263,7 @@ w_ref(PyObject *v, char *flag, WFILE *p)
     if (Py_REFCNT(v) == 1)
         return 0;
 
-    entry = _Py_hashtable_get_entry(p->hashtable, v);
+    entry = _Py_HASHTABLE_GET_ENTRY(p->hashtable, v);
     if (entry != NULL) {
         /* write the reference index to the stream */
         _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, &w, sizeof(w), entry);
@@ -571,7 +571,8 @@ static int
 w_init_refs(WFILE *wf, int version)
 {
     if (version >= 3) {
-        wf->hashtable = _Py_hashtable_new(sizeof(int), _Py_hashtable_hash_ptr,
+        wf->hashtable = _Py_hashtable_new(sizeof(void *), sizeof(int),
+                                          _Py_hashtable_hash_ptr,
                                           _Py_hashtable_compare_direct);
         if (wf->hashtable == NULL) {
             PyErr_NoMemory();
@@ -582,9 +583,11 @@ w_init_refs(WFILE *wf, int version)
 }
 
 static int
-w_decref_entry(_Py_hashtable_entry_t *entry, void *Py_UNUSED(data))
+w_decref_entry(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, void *Py_UNUSED(data))
 {
-    Py_XDECREF(entry->key);
+    void *entry_key = *(void **)_Py_HASHTABLE_ENTRY_KEY(entry);
+    assert(ht->key_size == sizeof(entry_key));
+    Py_XDECREF(entry_key);
     return 0;
 }
 
-- 
cgit v1.2.1


From 8dfb07a75eb73008ce70f9dd2dbea346c0e45b18 Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@gmail.com>
Date: Mon, 21 Mar 2016 14:36:39 +0100
Subject: Ooops, revert changeset ea9efa06c137

Change pushed by mistake, the patch is still under review :-/

"""
_tracemalloc: add domain to trace keys

* hashtable.h: key has now a variable size
* _tracemalloc uses (pointer: void*, domain: unsigned int) as key for traces
"""
---
 Python/marshal.c | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index 64084f4703..7a4b9d29b4 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -263,7 +263,7 @@ w_ref(PyObject *v, char *flag, WFILE *p)
     if (Py_REFCNT(v) == 1)
         return 0;
 
-    entry = _Py_HASHTABLE_GET_ENTRY(p->hashtable, v);
+    entry = _Py_hashtable_get_entry(p->hashtable, v);
     if (entry != NULL) {
         /* write the reference index to the stream */
         _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, &w, sizeof(w), entry);
@@ -571,8 +571,7 @@ static int
 w_init_refs(WFILE *wf, int version)
 {
     if (version >= 3) {
-        wf->hashtable = _Py_hashtable_new(sizeof(void *), sizeof(int),
-                                          _Py_hashtable_hash_ptr,
+        wf->hashtable = _Py_hashtable_new(sizeof(int), _Py_hashtable_hash_ptr,
                                           _Py_hashtable_compare_direct);
         if (wf->hashtable == NULL) {
             PyErr_NoMemory();
@@ -583,11 +582,9 @@ w_init_refs(WFILE *wf, int version)
 }
 
 static int
-w_decref_entry(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, void *Py_UNUSED(data))
+w_decref_entry(_Py_hashtable_entry_t *entry, void *Py_UNUSED(data))
 {
-    void *entry_key = *(void **)_Py_HASHTABLE_ENTRY_KEY(entry);
-    assert(ht->key_size == sizeof(entry_key));
-    Py_XDECREF(entry_key);
+    Py_XDECREF(entry->key);
     return 0;
 }
 
-- 
cgit v1.2.1


From c444230504eb916c4762c632bc724b3c27725a89 Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@gmail.com>
Date: Mon, 21 Mar 2016 22:00:58 +0100
Subject: hashtable.h now supports keys of any size

Issue #26588: hashtable.h now supports keys of any size, not only
sizeof(void*). It allows to support key larger than sizeof(void*), but also to
use less memory for key smaller than sizeof(void*).
---
 Python/marshal.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index 7a4b9d29b4..83a1885181 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -263,10 +263,10 @@ w_ref(PyObject *v, char *flag, WFILE *p)
     if (Py_REFCNT(v) == 1)
         return 0;
 
-    entry = _Py_hashtable_get_entry(p->hashtable, v);
+    entry = _Py_HASHTABLE_GET_ENTRY(p->hashtable, v);
     if (entry != NULL) {
         /* write the reference index to the stream */
-        _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, &w, sizeof(w), entry);
+        _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, entry, sizeof(w), &w);
         /* we don't store "long" indices in the dict */
         assert(0 <= w && w <= 0x7fffffff);
         w_byte(TYPE_REF, p);
@@ -571,7 +571,8 @@ static int
 w_init_refs(WFILE *wf, int version)
 {
     if (version >= 3) {
-        wf->hashtable = _Py_hashtable_new(sizeof(int), _Py_hashtable_hash_ptr,
+        wf->hashtable = _Py_hashtable_new(sizeof(PyObject *), sizeof(int),
+                                          _Py_hashtable_hash_ptr,
                                           _Py_hashtable_compare_direct);
         if (wf->hashtable == NULL) {
             PyErr_NoMemory();
@@ -582,9 +583,13 @@ w_init_refs(WFILE *wf, int version)
 }
 
 static int
-w_decref_entry(_Py_hashtable_entry_t *entry, void *Py_UNUSED(data))
+w_decref_entry(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry,
+               void *Py_UNUSED(data))
 {
-    Py_XDECREF(entry->key);
+    PyObject *entry_key;
+
+    _Py_HASHTABLE_ENTRY_READ_KEY(ht->key_size, entry, entry_key);
+    Py_XDECREF(entry_key);
     return 0;
 }
 
-- 
cgit v1.2.1


From a5624f4f81d487af1520f790b894f6f9e519de7b Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@gmail.com>
Date: Wed, 23 Mar 2016 09:25:01 +0100
Subject: Issue #26588:

* _Py_HASHTABLE_ENTRY_DATA: change type from "char *" to "const void *"
* Add _Py_HASHTABLE_ENTRY_WRITE_PKEY() macro
* Rename _Py_HASHTABLE_ENTRY_WRITE_DATA() macro to
  _Py_HASHTABLE_ENTRY_WRITE_PDATA()
* Add _Py_HASHTABLE_ENTRY_WRITE_DATA() macro
---
 Python/marshal.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index 83a1885181..3be77a82f4 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -266,7 +266,7 @@ w_ref(PyObject *v, char *flag, WFILE *p)
     entry = _Py_HASHTABLE_GET_ENTRY(p->hashtable, v);
     if (entry != NULL) {
         /* write the reference index to the stream */
-        _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, entry, sizeof(w), &w);
+        _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, entry, w);
         /* we don't store "long" indices in the dict */
         assert(0 <= w && w <= 0x7fffffff);
         w_byte(TYPE_REF, p);
-- 
cgit v1.2.1


From 61adfb874498d243f5bea2da569368f2b8b392a2 Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@gmail.com>
Date: Wed, 23 Mar 2016 09:52:13 +0100
Subject: Cleanup hashtable.h

Issue #26588:

* Pass the hash table rather than the key size to hash and compare functions
* _Py_HASHTABLE_READ_KEY() and _Py_HASHTABLE_ENTRY_READ_KEY() macros now expect
  the hash table as the first parameter, rather than the key size
* tracemalloc_get_traces_fill(): use _Py_HASHTABLE_ENTRY_READ_DATA() rather
  than pointer dereference
* Remove the _Py_HASHTABLE_ENTRY_WRITE_PKEY() macro
* Move "PKEY" and "PDATA" macros inside hashtable.c
---
 Python/marshal.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'Python/marshal.c')

diff --git a/Python/marshal.c b/Python/marshal.c
index 3be77a82f4..627a8428e5 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -588,7 +588,7 @@ w_decref_entry(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry,
 {
     PyObject *entry_key;
 
-    _Py_HASHTABLE_ENTRY_READ_KEY(ht->key_size, entry, entry_key);
+    _Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, entry_key);
     Py_XDECREF(entry_key);
     return 0;
 }
-- 
cgit v1.2.1