Merge pull request #302 from simplejson/class-serializationv3.18.0

Class serialization

8 files changed, 131 insertions, 87 deletions

diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml
index b4eb545..15ca1cc 100644
--- a/.github/workflows/build-and-deploy.yml
+++ b/.github/workflows/build-and-deploy.yml
@@ -65,7 +65,7 @@ jobs:
       - uses: actions/setup-python@v2
         name: Install Python
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Build sdist
         run: python setup.py sdist
diff --git a/CHANGES.txt b/CHANGES.txt
index 22d3ded..7d574e2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,8 @@
-Version 3.17.7 released XXXX-XX-XX
+Version 3.18.0 released 2022-11-14
 
+* Allow serialization of classes that implement for_json or _asdict by
+  ignoring TypeError when those methods are called
+  https://github.com/simplejson/simplejson/pull/302
 * Raise JSONDecodeError instead of ValueError in invalid unicode escape
   sequence edge case
   https://github.com/simplejson/simplejson/pull/298
diff --git a/conf.py b/conf.py
index 402899a..694446c 100644
--- a/conf.py
+++ b/conf.py
@@ -42,9 +42,9 @@ copyright = '2021, Bob Ippolito'
 # other places throughout the built documents.
 #
 # The short X.Y version.
-version = '3.17'
+version = '3.18'
 # The full version, including alpha/beta/rc tags.
-release = '3.17.7'
+release = '3.18.0'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
diff --git a/setup.py b/setup.py
index 7261f4d..336f443 100644
--- a/setup.py
+++ b/setup.py
@@ -12,7 +12,7 @@ from distutils.errors import CCompilerError, DistutilsExecError, \
     DistutilsPlatformError
 
 IS_PYPY = hasattr(sys, 'pypy_translation_info')
-VERSION = '3.17.7'
+VERSION = '3.18.0'
 DESCRIPTION = "Simple, fast, extensible JSON encoder/decoder for Python"
 
 with open('README.rst', 'r') as f:
diff --git a/simplejson/__init__.py b/simplejson/__init__.py
index c40d027..c79ad1b 100644
--- a/simplejson/__init__.py
+++ b/simplejson/__init__.py
@@ -118,7 +118,7 @@ Serializing multiple objects to JSON lines (newline-delimited JSON)::
 
 """
 from __future__ import absolute_import
-__version__ = '3.17.7'
+__version__ = '3.18.0'
 __all__ = [
     'dump', 'dumps', 'load', 'loads',
     'JSONDecoder', 'JSONDecodeError', 'JSONEncoder',
diff --git a/simplejson/_speedups.c b/simplejson/_speedups.c
index 6fefa99..ec054c7 100644
--- a/simplejson/_speedups.c
+++ b/simplejson/_speedups.c
@@ -25,6 +25,15 @@
 #define JSON_InternFromString PyString_InternFromString
 #endif /* PY_MAJOR_VERSION < 3 */
 
+#if PY_VERSION_HEX < 0x03090000
+#if !defined(PyObject_CallNoArgs)
+#define PyObject_CallNoArgs(callable) PyObject_CallFunctionObjArgs(callable, NULL);
+#endif
+#if !defined(PyObject_CallOneArg)
+#define PyObject_CallOneArg(callable, arg) PyObject_CallFunctionObjArgs(callable, arg, NULL);
+#endif
+#endif /* PY_VERSION_HEX < 0x03090000 */
+
 #if PY_VERSION_HEX < 0x02070000
 #if !defined(PyOS_string_to_double)
 #define PyOS_string_to_double json_PyOS_string_to_double
@@ -108,6 +117,9 @@ JSON_Accu_Destroy(JSON_Accu *acc);
 #define ERR_STRING_CONTROL "Invalid control character %r at"
 #define ERR_STRING_ESC1 "Invalid \\X escape sequence %r"
 #define ERR_STRING_ESC4 "Invalid \\uXXXX escape sequence"
+#define FOR_JSON_METHOD_NAME "for_json"
+#define ASDICT_METHOD_NAME "_asdict"
+
 
 typedef struct _PyScannerObject {
     PyObject_HEAD
@@ -243,12 +255,10 @@ static int
 _convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr);
 static PyObject *
 _convertPyInt_FromSsize_t(Py_ssize_t *size_ptr);
+static int
+_call_json_method(PyObject *obj, const char *method_name, PyObject **result);
 static PyObject *
 encoder_encode_float(PyEncoderObject *s, PyObject *obj);
-static int
-_is_namedtuple(PyObject *obj);
-static int
-_has_for_json_hook(PyObject *obj);
 static PyObject *
 moduleinit(void);
 
@@ -383,32 +393,26 @@ maybe_quote_bigint(PyEncoderObject* s, PyObject *encoded, PyObject *obj)
 }
 
 static int
-_is_namedtuple(PyObject *obj)
+_call_json_method(PyObject *obj, const char *method_name, PyObject **result)
 {
     int rval = 0;
-    /* We intentionally accept anything with a duck typed _asdict method rather
-     * than requiring it to pass PyTuple_Check(obj). */
-    PyObject *_asdict = PyObject_GetAttrString(obj, "_asdict");
-    if (_asdict == NULL) {
+    PyObject *method = PyObject_GetAttrString(obj, method_name);
+    if (method == NULL) {
         PyErr_Clear();
         return 0;
     }
-    rval = PyCallable_Check(_asdict);
-    Py_DECREF(_asdict);
-    return rval;
-}
-
-static int
-_has_for_json_hook(PyObject *obj)
-{
-    int rval = 0;
-    PyObject *for_json = PyObject_GetAttrString(obj, "for_json");
-    if (for_json == NULL) {
-        PyErr_Clear();
-        return 0;
+    if (PyCallable_Check(method)) {
+        PyObject *tmp = PyObject_CallNoArgs(method);
+        if (tmp == NULL && PyErr_ExceptionMatches(PyExc_TypeError)) {
+            PyErr_Clear();
+        } else {
+            // This will set result to NULL if a TypeError occurred,
+            // which must be checked by the caller
+            *result = tmp;
+            rval = 1;
+        }
     }
-    rval = PyCallable_Check(for_json);
-    Py_DECREF(for_json);
+    Py_DECREF(method);
     return rval;
 }
 
@@ -640,7 +644,7 @@ encoder_stringify_key(PyEncoderObject *s, PyObject *key)
         if (!(PyInt_CheckExact(key) || PyLong_CheckExact(key))) {
             /* See #118, do not trust custom str/repr */
             PyObject *res;
-            PyObject *tmp = PyObject_CallFunctionObjArgs((PyObject *)&PyLong_Type, key, NULL);
+            PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyLong_Type, key);
             if (tmp == NULL) {
                 return NULL;
             }
@@ -794,7 +798,7 @@ join_list_string(PyObject *lst)
         if (joinfn == NULL)
             return NULL;
     }
-    return PyObject_CallFunctionObjArgs(joinfn, lst, NULL);
+    return PyObject_CallOneArg(joinfn, lst);
 }
 #endif /* PY_MAJOR_VERSION < 3 */
 
@@ -1486,7 +1490,7 @@ _parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_
 
     /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */
     if (s->pairs_hook != Py_None) {
-        val = PyObject_CallFunctionObjArgs(s->pairs_hook, pairs, NULL);
+        val = PyObject_CallOneArg(s->pairs_hook, pairs);
         if (val == NULL)
             goto bail;
         Py_DECREF(pairs);
@@ -1496,7 +1500,7 @@ _parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_
 
     /* if object_hook is not None: rval = object_hook(rval) */
     if (s->object_hook != Py_None) {
-        val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL);
+        val = PyObject_CallOneArg(s->object_hook, rval);
         if (val == NULL)
             goto bail;
         Py_DECREF(rval);
@@ -1650,7 +1654,7 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss
 
     /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */
     if (s->pairs_hook != Py_None) {
-        val = PyObject_CallFunctionObjArgs(s->pairs_hook, pairs, NULL);
+        val = PyObject_CallOneArg(s->pairs_hook, pairs);
         if (val == NULL)
             goto bail;
         Py_DECREF(pairs);
@@ -1660,7 +1664,7 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss
 
     /* if object_hook is not None: rval = object_hook(rval) */
     if (s->object_hook != Py_None) {
-        val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL);
+        val = PyObject_CallOneArg(s->object_hook, rval);
         if (val == NULL)
             goto bail;
         Py_DECREF(rval);
@@ -1853,7 +1857,7 @@ _parse_constant(PyScannerObject *s, PyObject *constant, Py_ssize_t idx, Py_ssize
     PyObject *rval;
 
     /* rval = parse_constant(constant) */
-    rval = PyObject_CallFunctionObjArgs(s->parse_constant, constant, NULL);
+    rval = PyObject_CallOneArg(s->parse_constant, constant);
     idx += PyString_GET_SIZE(constant);
     *next_idx_ptr = idx;
     return rval;
@@ -1939,7 +1943,7 @@ _match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssiz
     if (is_float) {
         /* parse as a float using a fast path if available, otherwise call user defined method */
         if (s->parse_float != (PyObject *)&PyFloat_Type) {
-            rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL);
+            rval = PyObject_CallOneArg(s->parse_float, numstr);
         }
         else {
             /* rval = PyFloat_FromDouble(PyOS_ascii_atof(PyString_AS_STRING(numstr))); */
@@ -1953,7 +1957,7 @@ _match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssiz
     else {
         /* parse as an int using a fast path if available, otherwise call user defined method */
         if (s->parse_int != (PyObject *)&PyInt_Type) {
-            rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL);
+            rval = PyObject_CallOneArg(s->parse_int, numstr);
         }
         else {
             rval = PyInt_FromString(PyString_AS_STRING(numstr), NULL, 10);
@@ -2057,7 +2061,7 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_
     if (is_float) {
         /* parse as a float using a fast path if available, otherwise call user defined method */
         if (s->parse_float != (PyObject *)&PyFloat_Type) {
-            rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL);
+            rval = PyObject_CallOneArg(s->parse_float, numstr);
         }
         else {
 #if PY_MAJOR_VERSION >= 3
@@ -2069,7 +2073,7 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_
     }
     else {
         /* no fast path for unicode -> int, just call */
-        rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL);
+        rval = PyObject_CallOneArg(s->parse_int, numstr);
     }
     Py_DECREF(numstr);
     *next_idx_ptr = idx;
@@ -2748,7 +2752,7 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj)
     else {
         /* See #118, do not trust custom str/repr */
         PyObject *res;
-        PyObject *tmp = PyObject_CallFunctionObjArgs((PyObject *)&PyFloat_Type, obj, NULL);
+        PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyFloat_Type, obj);
         if (tmp == NULL) {
             return NULL;
         }
@@ -2767,7 +2771,7 @@ encoder_encode_string(PyEncoderObject *s, PyObject *obj)
     if (s->fast_encode) {
         return py_encode_basestring_ascii(NULL, obj);
     }
-    encoded = PyObject_CallFunctionObjArgs(s->encoder, obj, NULL);
+    encoded = PyObject_CallOneArg(s->encoder, obj);
     if (encoded != NULL &&
 #if PY_MAJOR_VERSION < 3
         !PyString_Check(encoded) &&
@@ -2798,6 +2802,7 @@ encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ss
     /* Encode Python object obj to a JSON term, rval is a PyList */
     int rv = -1;
     do {
+        PyObject *newobj;
         if (obj == Py_None || obj == Py_True || obj == Py_False) {
             PyObject *cstr = _encoded_const(obj);
             if (cstr != NULL)
@@ -2817,7 +2822,7 @@ encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ss
             }
             else {
                 /* See #118, do not trust custom str/repr */
-                PyObject *tmp = PyObject_CallFunctionObjArgs((PyObject *)&PyLong_Type, obj, NULL);
+                PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyLong_Type, obj);
                 if (tmp == NULL) {
                     encoded = NULL;
                 }
@@ -2838,35 +2843,37 @@ encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ss
             if (encoded != NULL)
                 rv = _steal_accumulate(rval, encoded);
         }
-        else if (s->for_json && _has_for_json_hook(obj)) {
-            PyObject *newobj;
-            if (Py_EnterRecursiveCall(" while encoding a JSON object"))
-                return rv;
-            newobj = PyObject_CallMethod(obj, "for_json", NULL);
-            if (newobj != NULL) {
-                rv = encoder_listencode_obj(s, rval, newobj, indent_level);
+        else if (s->for_json && _call_json_method(obj, FOR_JSON_METHOD_NAME, &newobj)) {
+            if (newobj == NULL) {
+                return -1;
+            }
+            if (Py_EnterRecursiveCall(" while encoding a JSON object")) {
                 Py_DECREF(newobj);
+                return rv;
             }
+            rv = encoder_listencode_obj(s, rval, newobj, indent_level);
+            Py_DECREF(newobj);
             Py_LeaveRecursiveCall();
         }
-        else if (s->namedtuple_as_object && _is_namedtuple(obj)) {
-            PyObject *newobj;
-            if (Py_EnterRecursiveCall(" while encoding a JSON object"))
+        else if (s->namedtuple_as_object && _call_json_method(obj, ASDICT_METHOD_NAME, &newobj)) {
+            if (newobj == NULL) {
+                return -1;
+            }
+            if (Py_EnterRecursiveCall(" while encoding a JSON object")) {
+                Py_DECREF(newobj);
                 return rv;
-            newobj = PyObject_CallMethod(obj, "_asdict", NULL);
-            if (newobj != NULL) {
-                if (!PyDict_Check(newobj)) {
-                    PyErr_Format(
-                        PyExc_TypeError,
-                        "_asdict() must return a dict, not %.80s",
-                        Py_TYPE(newobj)->tp_name
-                    );
-                    Py_DECREF(newobj);
-                    return -1;
-                }
+            }
+            if (PyDict_Check(newobj)) {
                 rv = encoder_listencode_dict(s, rval, newobj, indent_level);
-                Py_DECREF(newobj);
+            } else {
+                PyErr_Format(
+                    PyExc_TypeError,
+                    "_asdict() must return a dict, not %.80s",
+                    Py_TYPE(newobj)->tp_name
+                );
+                rv = -1;
             }
+            Py_DECREF(newobj);
             Py_LeaveRecursiveCall();
         }
         else if (PyList_Check(obj) || (s->tuple_as_array && PyTuple_Check(obj))) {
@@ -2924,7 +2931,7 @@ encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ss
             }
             if (Py_EnterRecursiveCall(" while encoding a JSON object"))
                 return rv;
-            newobj = PyObject_CallFunctionObjArgs(s->defaultfn, obj, NULL);
+            newobj = PyObject_CallOneArg(s->defaultfn, obj);
             if (newobj == NULL) {
                 Py_XDECREF(ident);
                 Py_LeaveRecursiveCall();
diff --git a/simplejson/encoder.py b/simplejson/encoder.py
index 2f81cab..2bc8bba 100644
--- a/simplejson/encoder.py
+++ b/simplejson/encoder.py
@@ -450,6 +450,15 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
          not isinstance(_int_as_string_bitcount, integer_types))):
         raise TypeError("int_as_string_bitcount must be a positive integer")
 
+    def call_method(obj, method_name):
+        method = getattr(obj, method_name, None)
+        if callable(method):
+            try:
+                return (method(),)
+            except TypeError:
+                pass
+        return None
+
     def _encode_int(value):
         skip_quoting = (
             _int_as_string_bitcount is None
@@ -512,15 +521,15 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
                 yield buf + str(value)
             else:
                 yield buf
-                for_json = _for_json and getattr(value, 'for_json', None)
-                if for_json and callable(for_json):
-                    chunks = _iterencode(for_json(), _current_indent_level)
+                for_json = _for_json and call_method(value, 'for_json')
+                if for_json:
+                    chunks = _iterencode(for_json[0], _current_indent_level)
                 elif isinstance(value, list):
                     chunks = _iterencode_list(value, _current_indent_level)
                 else:
-                    _asdict = _namedtuple_as_object and getattr(value, '_asdict', None)
-                    if _asdict and callable(_asdict):
-                        dct = _asdict()
+                    _asdict = _namedtuple_as_object and call_method(value, '_asdict')
+                    if _asdict:
+                        dct = _asdict[0]
                         if not isinstance(dct, dict):
                             raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,))
                         chunks = _iterencode_dict(dct,
@@ -636,15 +645,15 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
             elif _use_decimal and isinstance(value, Decimal):
                 yield str(value)
             else:
-                for_json = _for_json and getattr(value, 'for_json', None)
-                if for_json and callable(for_json):
-                    chunks = _iterencode(for_json(), _current_indent_level)
+                for_json = _for_json and call_method(value, 'for_json')
+                if for_json:
+                    chunks = _iterencode(for_json[0], _current_indent_level)
                 elif isinstance(value, list):
                     chunks = _iterencode_list(value, _current_indent_level)
                 else:
-                    _asdict = _namedtuple_as_object and getattr(value, '_asdict', None)
-                    if _asdict and callable(_asdict):
-                        dct = _asdict()
+                    _asdict = _namedtuple_as_object and call_method(value, '_asdict')
+                    if _asdict:
+                        dct = _asdict[0]
                         if not isinstance(dct, dict):
                             raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,))
                         chunks = _iterencode_dict(dct,
@@ -682,17 +691,17 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
         elif isinstance(o, float):
             yield _floatstr(o)
         else:
-            for_json = _for_json and getattr(o, 'for_json', None)
-            if for_json and callable(for_json):
-                for chunk in _iterencode(for_json(), _current_indent_level):
+            for_json = _for_json and call_method(o, 'for_json')
+            if for_json:
+                for chunk in _iterencode(for_json[0], _current_indent_level):
                     yield chunk
             elif isinstance(o, list):
                 for chunk in _iterencode_list(o, _current_indent_level):
                     yield chunk
             else:
-                _asdict = _namedtuple_as_object and getattr(o, '_asdict', None)
-                if _asdict and callable(_asdict):
-                    dct = _asdict()
+                _asdict = _namedtuple_as_object and call_method(o, '_asdict')
+                if _asdict:
+                    dct = _asdict[0]
                     if not isinstance(dct, dict):
                         raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,))
                     for chunk in _iterencode_dict(dct, _current_indent_level):
diff --git a/simplejson/tests/test_namedtuple.py b/simplejson/tests/test_namedtuple.py
index 8035a5f..cc0f8aa 100644
--- a/simplejson/tests/test_namedtuple.py
+++ b/simplejson/tests/test_namedtuple.py
@@ -110,22 +110,47 @@ class TestNamedTuple(unittest.TestCase):
 
     def test_asdict_not_callable_dump(self):
         for f in CONSTRUCTORS:
-            self.assertRaises(TypeError,
-                json.dump, f(DeadDuck()), StringIO(), namedtuple_as_object=True)
+            self.assertRaises(
+                TypeError,
+                json.dump,
+                f(DeadDuck()),
+                StringIO(),
+                namedtuple_as_object=True
+            )
             sio = StringIO()
             json.dump(f(DeadDict()), sio, namedtuple_as_object=True)
             self.assertEqual(
                 json.dumps(f({})),
                 sio.getvalue())
+            self.assertRaises(
+                TypeError,
+                json.dump,
+                f(Value),
+                StringIO(),
+                namedtuple_as_object=True
+            )
 
     def test_asdict_not_callable_dumps(self):
         for f in CONSTRUCTORS:
             self.assertRaises(TypeError,
                 json.dumps, f(DeadDuck()), namedtuple_as_object=True)
+            self.assertRaises(
+                TypeError,
+                json.dumps,
+                f(Value),
+                namedtuple_as_object=True
+            )
             self.assertEqual(
                 json.dumps(f({})),
                 json.dumps(f(DeadDict()), namedtuple_as_object=True))
 
+    def test_asdict_unbound_method_dumps(self):
+        for f in CONSTRUCTORS:
+            self.assertEqual(
+                json.dumps(f(Value), default=lambda v: v.__name__),
+                json.dumps(f(Value.__name__))
+            )
+
     def test_asdict_does_not_return_dict(self):
         if not mock:
             if hasattr(unittest, "SkipTest"):