diff options
-rw-r--r-- | .github/workflows/build-and-deploy.yml | 2 | ||||
-rw-r--r-- | CHANGES.txt | 5 | ||||
-rw-r--r-- | conf.py | 4 | ||||
-rw-r--r-- | setup.py | 2 | ||||
-rw-r--r-- | simplejson/__init__.py | 2 | ||||
-rw-r--r-- | simplejson/_speedups.c | 129 | ||||
-rw-r--r-- | simplejson/encoder.py | 45 | ||||
-rw-r--r-- | simplejson/tests/test_namedtuple.py | 29 |
8 files changed, 131 insertions, 87 deletions
diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index b4eb545..15ca1cc 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -65,7 +65,7 @@ jobs: - uses: actions/setup-python@v2 name: Install Python with: - python-version: '3.10' + python-version: '3.11' - name: Build sdist run: python setup.py sdist diff --git a/CHANGES.txt b/CHANGES.txt index 22d3ded..7d574e2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,8 @@ -Version 3.17.7 released XXXX-XX-XX +Version 3.18.0 released 2022-11-14 +* Allow serialization of classes that implement for_json or _asdict by + ignoring TypeError when those methods are called + https://github.com/simplejson/simplejson/pull/302 * Raise JSONDecodeError instead of ValueError in invalid unicode escape sequence edge case https://github.com/simplejson/simplejson/pull/298 @@ -42,9 +42,9 @@ copyright = '2021, Bob Ippolito' # other places throughout the built documents. # # The short X.Y version. -version = '3.17' +version = '3.18' # The full version, including alpha/beta/rc tags. -release = '3.17.7' +release = '3.18.0' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: @@ -12,7 +12,7 @@ from distutils.errors import CCompilerError, DistutilsExecError, \ DistutilsPlatformError IS_PYPY = hasattr(sys, 'pypy_translation_info') -VERSION = '3.17.7' +VERSION = '3.18.0' DESCRIPTION = "Simple, fast, extensible JSON encoder/decoder for Python" with open('README.rst', 'r') as f: diff --git a/simplejson/__init__.py b/simplejson/__init__.py index c40d027..c79ad1b 100644 --- a/simplejson/__init__.py +++ b/simplejson/__init__.py @@ -118,7 +118,7 @@ Serializing multiple objects to JSON lines (newline-delimited JSON):: """ from __future__ import absolute_import -__version__ = '3.17.7' +__version__ = '3.18.0' __all__ = [ 'dump', 'dumps', 'load', 'loads', 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', diff --git a/simplejson/_speedups.c b/simplejson/_speedups.c index 6fefa99..ec054c7 100644 --- a/simplejson/_speedups.c +++ b/simplejson/_speedups.c @@ -25,6 +25,15 @@ #define JSON_InternFromString PyString_InternFromString #endif /* PY_MAJOR_VERSION < 3 */ +#if PY_VERSION_HEX < 0x03090000 +#if !defined(PyObject_CallNoArgs) +#define PyObject_CallNoArgs(callable) PyObject_CallFunctionObjArgs(callable, NULL); +#endif +#if !defined(PyObject_CallOneArg) +#define PyObject_CallOneArg(callable, arg) PyObject_CallFunctionObjArgs(callable, arg, NULL); +#endif +#endif /* PY_VERSION_HEX < 0x03090000 */ + #if PY_VERSION_HEX < 0x02070000 #if !defined(PyOS_string_to_double) #define PyOS_string_to_double json_PyOS_string_to_double @@ -108,6 +117,9 @@ JSON_Accu_Destroy(JSON_Accu *acc); #define ERR_STRING_CONTROL "Invalid control character %r at" #define ERR_STRING_ESC1 "Invalid \\X escape sequence %r" #define ERR_STRING_ESC4 "Invalid \\uXXXX escape sequence" +#define FOR_JSON_METHOD_NAME "for_json" +#define ASDICT_METHOD_NAME "_asdict" + typedef struct _PyScannerObject { PyObject_HEAD @@ -243,12 +255,10 @@ static int _convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr); static PyObject * _convertPyInt_FromSsize_t(Py_ssize_t *size_ptr); +static int +_call_json_method(PyObject *obj, const char *method_name, PyObject **result); static PyObject * encoder_encode_float(PyEncoderObject *s, PyObject *obj); -static int -_is_namedtuple(PyObject *obj); -static int -_has_for_json_hook(PyObject *obj); static PyObject * moduleinit(void); @@ -383,32 +393,26 @@ maybe_quote_bigint(PyEncoderObject* s, PyObject *encoded, PyObject *obj) } static int -_is_namedtuple(PyObject *obj) +_call_json_method(PyObject *obj, const char *method_name, PyObject **result) { int rval = 0; - /* We intentionally accept anything with a duck typed _asdict method rather - * than requiring it to pass PyTuple_Check(obj). */ - PyObject *_asdict = PyObject_GetAttrString(obj, "_asdict"); - if (_asdict == NULL) { + PyObject *method = PyObject_GetAttrString(obj, method_name); + if (method == NULL) { PyErr_Clear(); return 0; } - rval = PyCallable_Check(_asdict); - Py_DECREF(_asdict); - return rval; -} - -static int -_has_for_json_hook(PyObject *obj) -{ - int rval = 0; - PyObject *for_json = PyObject_GetAttrString(obj, "for_json"); - if (for_json == NULL) { - PyErr_Clear(); - return 0; + if (PyCallable_Check(method)) { + PyObject *tmp = PyObject_CallNoArgs(method); + if (tmp == NULL && PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Clear(); + } else { + // This will set result to NULL if a TypeError occurred, + // which must be checked by the caller + *result = tmp; + rval = 1; + } } - rval = PyCallable_Check(for_json); - Py_DECREF(for_json); + Py_DECREF(method); return rval; } @@ -640,7 +644,7 @@ encoder_stringify_key(PyEncoderObject *s, PyObject *key) if (!(PyInt_CheckExact(key) || PyLong_CheckExact(key))) { /* See #118, do not trust custom str/repr */ PyObject *res; - PyObject *tmp = PyObject_CallFunctionObjArgs((PyObject *)&PyLong_Type, key, NULL); + PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyLong_Type, key); if (tmp == NULL) { return NULL; } @@ -794,7 +798,7 @@ join_list_string(PyObject *lst) if (joinfn == NULL) return NULL; } - return PyObject_CallFunctionObjArgs(joinfn, lst, NULL); + return PyObject_CallOneArg(joinfn, lst); } #endif /* PY_MAJOR_VERSION < 3 */ @@ -1486,7 +1490,7 @@ _parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_ /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ if (s->pairs_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->pairs_hook, pairs, NULL); + val = PyObject_CallOneArg(s->pairs_hook, pairs); if (val == NULL) goto bail; Py_DECREF(pairs); @@ -1496,7 +1500,7 @@ _parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_ /* if object_hook is not None: rval = object_hook(rval) */ if (s->object_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL); + val = PyObject_CallOneArg(s->object_hook, rval); if (val == NULL) goto bail; Py_DECREF(rval); @@ -1650,7 +1654,7 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ if (s->pairs_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->pairs_hook, pairs, NULL); + val = PyObject_CallOneArg(s->pairs_hook, pairs); if (val == NULL) goto bail; Py_DECREF(pairs); @@ -1660,7 +1664,7 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss /* if object_hook is not None: rval = object_hook(rval) */ if (s->object_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL); + val = PyObject_CallOneArg(s->object_hook, rval); if (val == NULL) goto bail; Py_DECREF(rval); @@ -1853,7 +1857,7 @@ _parse_constant(PyScannerObject *s, PyObject *constant, Py_ssize_t idx, Py_ssize PyObject *rval; /* rval = parse_constant(constant) */ - rval = PyObject_CallFunctionObjArgs(s->parse_constant, constant, NULL); + rval = PyObject_CallOneArg(s->parse_constant, constant); idx += PyString_GET_SIZE(constant); *next_idx_ptr = idx; return rval; @@ -1939,7 +1943,7 @@ _match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssiz if (is_float) { /* parse as a float using a fast path if available, otherwise call user defined method */ if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL); + rval = PyObject_CallOneArg(s->parse_float, numstr); } else { /* rval = PyFloat_FromDouble(PyOS_ascii_atof(PyString_AS_STRING(numstr))); */ @@ -1953,7 +1957,7 @@ _match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssiz else { /* parse as an int using a fast path if available, otherwise call user defined method */ if (s->parse_int != (PyObject *)&PyInt_Type) { - rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL); + rval = PyObject_CallOneArg(s->parse_int, numstr); } else { rval = PyInt_FromString(PyString_AS_STRING(numstr), NULL, 10); @@ -2057,7 +2061,7 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ if (is_float) { /* parse as a float using a fast path if available, otherwise call user defined method */ if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL); + rval = PyObject_CallOneArg(s->parse_float, numstr); } else { #if PY_MAJOR_VERSION >= 3 @@ -2069,7 +2073,7 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ } else { /* no fast path for unicode -> int, just call */ - rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL); + rval = PyObject_CallOneArg(s->parse_int, numstr); } Py_DECREF(numstr); *next_idx_ptr = idx; @@ -2748,7 +2752,7 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj) else { /* See #118, do not trust custom str/repr */ PyObject *res; - PyObject *tmp = PyObject_CallFunctionObjArgs((PyObject *)&PyFloat_Type, obj, NULL); + PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyFloat_Type, obj); if (tmp == NULL) { return NULL; } @@ -2767,7 +2771,7 @@ encoder_encode_string(PyEncoderObject *s, PyObject *obj) if (s->fast_encode) { return py_encode_basestring_ascii(NULL, obj); } - encoded = PyObject_CallFunctionObjArgs(s->encoder, obj, NULL); + encoded = PyObject_CallOneArg(s->encoder, obj); if (encoded != NULL && #if PY_MAJOR_VERSION < 3 !PyString_Check(encoded) && @@ -2798,6 +2802,7 @@ encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ss /* Encode Python object obj to a JSON term, rval is a PyList */ int rv = -1; do { + PyObject *newobj; if (obj == Py_None || obj == Py_True || obj == Py_False) { PyObject *cstr = _encoded_const(obj); if (cstr != NULL) @@ -2817,7 +2822,7 @@ encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ss } else { /* See #118, do not trust custom str/repr */ - PyObject *tmp = PyObject_CallFunctionObjArgs((PyObject *)&PyLong_Type, obj, NULL); + PyObject *tmp = PyObject_CallOneArg((PyObject *)&PyLong_Type, obj); if (tmp == NULL) { encoded = NULL; } @@ -2838,35 +2843,37 @@ encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ss if (encoded != NULL) rv = _steal_accumulate(rval, encoded); } - else if (s->for_json && _has_for_json_hook(obj)) { - PyObject *newobj; - if (Py_EnterRecursiveCall(" while encoding a JSON object")) - return rv; - newobj = PyObject_CallMethod(obj, "for_json", NULL); - if (newobj != NULL) { - rv = encoder_listencode_obj(s, rval, newobj, indent_level); + else if (s->for_json && _call_json_method(obj, FOR_JSON_METHOD_NAME, &newobj)) { + if (newobj == NULL) { + return -1; + } + if (Py_EnterRecursiveCall(" while encoding a JSON object")) { Py_DECREF(newobj); + return rv; } + rv = encoder_listencode_obj(s, rval, newobj, indent_level); + Py_DECREF(newobj); Py_LeaveRecursiveCall(); } - else if (s->namedtuple_as_object && _is_namedtuple(obj)) { - PyObject *newobj; - if (Py_EnterRecursiveCall(" while encoding a JSON object")) + else if (s->namedtuple_as_object && _call_json_method(obj, ASDICT_METHOD_NAME, &newobj)) { + if (newobj == NULL) { + return -1; + } + if (Py_EnterRecursiveCall(" while encoding a JSON object")) { + Py_DECREF(newobj); return rv; - newobj = PyObject_CallMethod(obj, "_asdict", NULL); - if (newobj != NULL) { - if (!PyDict_Check(newobj)) { - PyErr_Format( - PyExc_TypeError, - "_asdict() must return a dict, not %.80s", - Py_TYPE(newobj)->tp_name - ); - Py_DECREF(newobj); - return -1; - } + } + if (PyDict_Check(newobj)) { rv = encoder_listencode_dict(s, rval, newobj, indent_level); - Py_DECREF(newobj); + } else { + PyErr_Format( + PyExc_TypeError, + "_asdict() must return a dict, not %.80s", + Py_TYPE(newobj)->tp_name + ); + rv = -1; } + Py_DECREF(newobj); Py_LeaveRecursiveCall(); } else if (PyList_Check(obj) || (s->tuple_as_array && PyTuple_Check(obj))) { @@ -2924,7 +2931,7 @@ encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ss } if (Py_EnterRecursiveCall(" while encoding a JSON object")) return rv; - newobj = PyObject_CallFunctionObjArgs(s->defaultfn, obj, NULL); + newobj = PyObject_CallOneArg(s->defaultfn, obj); if (newobj == NULL) { Py_XDECREF(ident); Py_LeaveRecursiveCall(); diff --git a/simplejson/encoder.py b/simplejson/encoder.py index 2f81cab..2bc8bba 100644 --- a/simplejson/encoder.py +++ b/simplejson/encoder.py @@ -450,6 +450,15 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, not isinstance(_int_as_string_bitcount, integer_types))): raise TypeError("int_as_string_bitcount must be a positive integer") + def call_method(obj, method_name): + method = getattr(obj, method_name, None) + if callable(method): + try: + return (method(),) + except TypeError: + pass + return None + def _encode_int(value): skip_quoting = ( _int_as_string_bitcount is None @@ -512,15 +521,15 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, yield buf + str(value) else: yield buf - for_json = _for_json and getattr(value, 'for_json', None) - if for_json and callable(for_json): - chunks = _iterencode(for_json(), _current_indent_level) + for_json = _for_json and call_method(value, 'for_json') + if for_json: + chunks = _iterencode(for_json[0], _current_indent_level) elif isinstance(value, list): chunks = _iterencode_list(value, _current_indent_level) else: - _asdict = _namedtuple_as_object and getattr(value, '_asdict', None) - if _asdict and callable(_asdict): - dct = _asdict() + _asdict = _namedtuple_as_object and call_method(value, '_asdict') + if _asdict: + dct = _asdict[0] if not isinstance(dct, dict): raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) chunks = _iterencode_dict(dct, @@ -636,15 +645,15 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, elif _use_decimal and isinstance(value, Decimal): yield str(value) else: - for_json = _for_json and getattr(value, 'for_json', None) - if for_json and callable(for_json): - chunks = _iterencode(for_json(), _current_indent_level) + for_json = _for_json and call_method(value, 'for_json') + if for_json: + chunks = _iterencode(for_json[0], _current_indent_level) elif isinstance(value, list): chunks = _iterencode_list(value, _current_indent_level) else: - _asdict = _namedtuple_as_object and getattr(value, '_asdict', None) - if _asdict and callable(_asdict): - dct = _asdict() + _asdict = _namedtuple_as_object and call_method(value, '_asdict') + if _asdict: + dct = _asdict[0] if not isinstance(dct, dict): raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) chunks = _iterencode_dict(dct, @@ -682,17 +691,17 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, elif isinstance(o, float): yield _floatstr(o) else: - for_json = _for_json and getattr(o, 'for_json', None) - if for_json and callable(for_json): - for chunk in _iterencode(for_json(), _current_indent_level): + for_json = _for_json and call_method(o, 'for_json') + if for_json: + for chunk in _iterencode(for_json[0], _current_indent_level): yield chunk elif isinstance(o, list): for chunk in _iterencode_list(o, _current_indent_level): yield chunk else: - _asdict = _namedtuple_as_object and getattr(o, '_asdict', None) - if _asdict and callable(_asdict): - dct = _asdict() + _asdict = _namedtuple_as_object and call_method(o, '_asdict') + if _asdict: + dct = _asdict[0] if not isinstance(dct, dict): raise TypeError("_asdict() must return a dict, not %s" % (type(dct).__name__,)) for chunk in _iterencode_dict(dct, _current_indent_level): diff --git a/simplejson/tests/test_namedtuple.py b/simplejson/tests/test_namedtuple.py index 8035a5f..cc0f8aa 100644 --- a/simplejson/tests/test_namedtuple.py +++ b/simplejson/tests/test_namedtuple.py @@ -110,22 +110,47 @@ class TestNamedTuple(unittest.TestCase): def test_asdict_not_callable_dump(self): for f in CONSTRUCTORS: - self.assertRaises(TypeError, - json.dump, f(DeadDuck()), StringIO(), namedtuple_as_object=True) + self.assertRaises( + TypeError, + json.dump, + f(DeadDuck()), + StringIO(), + namedtuple_as_object=True + ) sio = StringIO() json.dump(f(DeadDict()), sio, namedtuple_as_object=True) self.assertEqual( json.dumps(f({})), sio.getvalue()) + self.assertRaises( + TypeError, + json.dump, + f(Value), + StringIO(), + namedtuple_as_object=True + ) def test_asdict_not_callable_dumps(self): for f in CONSTRUCTORS: self.assertRaises(TypeError, json.dumps, f(DeadDuck()), namedtuple_as_object=True) + self.assertRaises( + TypeError, + json.dumps, + f(Value), + namedtuple_as_object=True + ) self.assertEqual( json.dumps(f({})), json.dumps(f(DeadDict()), namedtuple_as_object=True)) + def test_asdict_unbound_method_dumps(self): + for f in CONSTRUCTORS: + self.assertEqual( + json.dumps(f(Value), default=lambda v: v.__name__), + json.dumps(f(Value.__name__)) + ) + def test_asdict_does_not_return_dict(self): if not mock: if hasattr(unittest, "SkipTest"): |