2.3.0 and iterable_as_array encoding optioniterable_as_array-gh1

9 files changed, 132 insertions, 31 deletions

diff --git a/CHANGES.txt b/CHANGES.txt
index 494a7d6..69ebb57 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,8 @@
+Version 2.3.0 released 2011-XX-XX
+
+* New iterable_as_array encoder option to perform lazy serialization of
+  any iterable objects, without having to convert to tuple or list.
+
 Version 2.2.0 released 2011-09-04
 
 * Remove setuptools requirement, reverted to pure distutils
diff --git a/conf.py b/conf.py
index a4cfadf..4b4537a 100644
--- a/conf.py
+++ b/conf.py
@@ -42,9 +42,9 @@ copyright = '2011, Bob Ippolito'
 # other places throughout the built documents.
 #
 # The short X.Y version.
-version = '2.2'
+version = '2.3'
 # The full version, including alpha/beta/rc tags.
-release = '2.2.0'
+release = '2.3.0'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
diff --git a/index.rst b/index.rst
index 11f076e..b965982 100644
--- a/index.rst
+++ b/index.rst
@@ -129,7 +129,7 @@ Using :mod:`simplejson.tool` from the shell to validate and pretty-print::
 Basic Usage
 -----------
 
-.. function:: dump(obj, fp[, skipkeys[, ensure_ascii[, check_circular[, allow_nan[, cls[, indent[, separators[, encoding[, default[, use_decimal[, namedtuple_as_object[, tuple_as_array[, **kw]]]]]]]]]]]]])
+.. function:: dump(obj, fp[, skipkeys[, ensure_ascii[, check_circular[, allow_nan[, cls[, indent[, separators[, encoding[, default[, use_decimal[, namedtuple_as_object[, tuple_as_array[, iterable_as_array[, **kw]]]]]]]]]]]]]])
 
    Serialize *obj* as a JSON formatted stream to *fp* (a ``.write()``-supporting
    file-like object).
@@ -192,7 +192,7 @@ Basic Usage
   If *namedtuple_as_object* is true (default: ``True``),
   :class:`tuple` subclasses with ``_asdict()`` methods will be encoded
   as JSON objects.
-  
+
   .. versionchanged:: 2.2.0
     *namedtuple_as_object* is new in 2.2.0.
 
@@ -202,6 +202,14 @@ Basic Usage
   .. versionchanged:: 2.2.0
     *tuple_as_array* is new in 2.2.0.
 
+  If *iterable_as_array* is true (default: ``False``),
+  any object not in the above table that implements ``__iter__()``
+  will be encoded as a JSON array.
+
+  .. versionchanged:: 2.3.0
+    *iterable_as_array* is new in 2.3.0.
+
+
     .. note::
 
         JSON is not a framed protocol so unlike :mod:`pickle` or :mod:`marshal` it
@@ -209,7 +217,7 @@ Basic Usage
         container protocol to delimit them.
 
 
-.. function:: dumps(obj[, skipkeys[, ensure_ascii[, check_circular[, allow_nan[, cls[, indent[, separators[, encoding[, default[, use_decimal[, namedtuple_as_object[, tuple_as_array[, **kw]]]]]]]]]]]]])
+.. function:: dumps(obj[, skipkeys[, ensure_ascii[, check_circular[, allow_nan[, cls[, indent[, separators[, encoding[, default[, use_decimal[, namedtuple_as_object[, tuple_as_array[, iterable_as_array[, **kw]]]]]]]]]]]]]])
 
    Serialize *obj* to a JSON formatted :class:`str`.
 
@@ -274,6 +282,14 @@ Basic Usage
    .. versionchanged:: 2.1.0
       *use_decimal* is new in 2.1.0.
 
+   If *iterable_as_array* is true (default: ``False``),
+   any object not in the above table that implements ``__iter__()``
+   will be encoded as a JSON array.
+
+   .. versionchanged:: 2.3.0
+     *iterable_as_array* is new in 2.3.0.
+
+
    To use a custom :class:`JSONDecoder` subclass, specify it with the ``cls``
    kwarg.  Additional keyword arguments will be passed to the constructor of the
    class.
@@ -406,7 +422,7 @@ Encoders and decoders
       :exc:`JSONDecodeError` will be raised if the given JSON
       document is not valid.
 
-.. class:: JSONEncoder([skipkeys[, ensure_ascii[, check_circular[, allow_nan[, sort_keys[, indent[, separators[, encoding[, default[, use_decimal[, namedtuple_as_object[, tuple_as_array]]]]]]]]]]]])
+.. class:: JSONEncoder([skipkeys[, ensure_ascii[, check_circular[, allow_nan[, sort_keys[, indent[, separators[, encoding[, default[, use_decimal[, namedtuple_as_object[, tuple_as_array[, iterable_as_array]]]]]]]]]]]]])
 
    Extensible JSON encoder for Python data structures.
 
@@ -496,6 +512,13 @@ Encoders and decoders
    .. versionchanged:: 2.2.0
      *tuple_as_array* is new in 2.2.0.
 
+   If *iterable_as_array* is true (default: ``False``),
+   any object not in the above table that implements ``__iter__()``
+   will be encoded as a JSON array.
+
+   .. versionchanged:: 2.3.0
+     *iterable_as_array* is new in 2.3.0.
+
    .. method:: default(o)
 
       Implement this method in a subclass such that it returns a serializable
diff --git a/setup.py b/setup.py
index 7dc6d78..b2b0e5f 100644
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@ from distutils.errors import CCompilerError, DistutilsExecError, \
     DistutilsPlatformError
 
 IS_PYPY = hasattr(sys, 'pypy_translation_info')
-VERSION = '2.2.0'
+VERSION = '2.3.0'
 DESCRIPTION = "Simple, fast, extensible JSON encoder/decoder for Python"
 LONG_DESCRIPTION = open('README.rst', 'r').read()
 
@@ -36,13 +36,13 @@ class ve_build_ext(build_ext):
     def run(self):
         try:
             build_ext.run(self)
-        except DistutilsPlatformError, x:
+        except DistutilsPlatformError:
             raise BuildFailed()
 
     def build_extension(self, ext):
         try:
             build_ext.build_extension(self, ext)
-        except ext_errors, x:
+        except ext_errors:
             raise BuildFailed()
 
 
diff --git a/simplejson/__init__.py b/simplejson/__init__.py
index 3cd09df..a1c0578 100644
--- a/simplejson/__init__.py
+++ b/simplejson/__init__.py
@@ -5,9 +5,8 @@ interchange format.
 :mod:`simplejson` exposes an API familiar to users of the standard library
 :mod:`marshal` and :mod:`pickle` modules. It is the externally maintained
 version of the :mod:`json` library contained in Python 2.6, but maintains
-compatibility with Python 2.4 and Python 2.5 and (currently) has
-significant performance advantages, even without using the optional C
-extension for speedups.
+compatibility back to Python 2.5 and (currently) has significant performance
+advantages, even without using the optional C extension for speedups.
 
 Encoding basic Python object hierarchies::
 
@@ -97,7 +96,7 @@ Using simplejson.tool from the shell to validate and pretty-print::
     $ echo '{ 1.2:3.4}' | python -m simplejson.tool
     Expecting property name: line 1 column 2 (char 2)
 """
-__version__ = '2.2.0'
+__version__ = '2.3.0'
 __all__ = [
     'dump', 'dumps', 'load', 'loads',
     'JSONDecoder', 'JSONDecodeError', 'JSONEncoder',
@@ -138,12 +137,14 @@ _default_encoder = JSONEncoder(
     use_decimal=True,
     namedtuple_as_object=True,
     tuple_as_array=True,
+    iterable_as_array=False,
 )
 
 def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
         allow_nan=True, cls=None, indent=None, separators=None,
         encoding='utf-8', default=None, use_decimal=True,
         namedtuple_as_object=True, tuple_as_array=True,
+        iterable_as_array=False,
         **kw):
     """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
     ``.write()``-supporting file-like object).
@@ -189,10 +190,14 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
     If *namedtuple_as_object* is true (default: ``True``),
     :class:`tuple` subclasses with ``_asdict()`` methods will be encoded
     as JSON objects.
-    
+
     If *tuple_as_array* is true (default: ``True``),
     :class:`tuple` (and subclasses) will be encoded as JSON arrays.
 
+    If *iterable_as_array* is true (default: ``False``),
+    any object not in the above table that implements ``__iter__()``
+    will be encoded as a JSON array.
+
     To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
     ``.default()`` method to serialize additional types), specify it with
     the ``cls`` kwarg.
@@ -202,8 +207,9 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
     if (not skipkeys and ensure_ascii and
         check_circular and allow_nan and
         cls is None and indent is None and separators is None and
-        encoding == 'utf-8' and default is None and use_decimal
-        and namedtuple_as_object and tuple_as_array and not kw):
+        encoding == 'utf-8' and default is None and use_decimal and
+        namedtuple_as_object and tuple_as_array and
+        not iterable_as_array and not kw):
         iterable = _default_encoder.iterencode(obj)
     else:
         if cls is None:
@@ -214,6 +220,7 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
             default=default, use_decimal=use_decimal,
             namedtuple_as_object=namedtuple_as_object,
             tuple_as_array=tuple_as_array,
+            iterable_as_array=iterable_as_array,
             **kw).iterencode(obj)
     # could accelerate with writelines in some versions of Python, at
     # a debuggability cost
@@ -226,6 +233,7 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
         encoding='utf-8', default=None, use_decimal=True,
         namedtuple_as_object=True,
         tuple_as_array=True,
+        iterable_as_array=False,
         **kw):
     """Serialize ``obj`` to a JSON formatted ``str``.
 
@@ -268,10 +276,14 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
     If *namedtuple_as_object* is true (default: ``True``),
     :class:`tuple` subclasses with ``_asdict()`` methods will be encoded
     as JSON objects.
-    
+
     If *tuple_as_array* is true (default: ``True``),
     :class:`tuple` (and subclasses) will be encoded as JSON arrays.
 
+    If *iterable_as_array* is true (default: ``False``),
+    any object not in the above table that implements ``__iter__()``
+    will be encoded as a JSON array.
+
     To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
     ``.default()`` method to serialize additional types), specify it with
     the ``cls`` kwarg.
@@ -281,8 +293,9 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
     if (not skipkeys and ensure_ascii and
         check_circular and allow_nan and
         cls is None and indent is None and separators is None and
-        encoding == 'utf-8' and default is None and use_decimal
-        and namedtuple_as_object and tuple_as_array and not kw):
+        encoding == 'utf-8' and default is None and use_decimal and
+        namedtuple_as_object and tuple_as_array and
+        not iterable_as_array and not kw):
         return _default_encoder.encode(obj)
     if cls is None:
         cls = JSONEncoder
@@ -293,6 +306,7 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
         use_decimal=use_decimal,
         namedtuple_as_object=namedtuple_as_object,
         tuple_as_array=tuple_as_array,
+        iterable_as_array=iterable_as_array,
         **kw).encode(obj)
 
 
@@ -438,7 +452,7 @@ def _toggle_speedups(enabled):
     if enabled:
         dec.scanstring = dec.c_scanstring or dec.py_scanstring
         enc.c_make_encoder = c_make_encoder
-        enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or 
+        enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or
             enc.py_encode_basestring_ascii)
         scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner
     else:
diff --git a/simplejson/_speedups.c b/simplejson/_speedups.c
index f8b0565..783bac8 100644
--- a/simplejson/_speedups.c
+++ b/simplejson/_speedups.c
@@ -89,6 +89,7 @@ typedef struct _PyEncoderObject {
     int use_decimal;
     int namedtuple_as_object;
     int tuple_as_array;
+    int iterable_as_array;
 } PyEncoderObject;
 
 static PyMemberDef encoder_members[] = {
@@ -2017,11 +2018,11 @@ static int
 encoder_init(PyObject *self, PyObject *args, PyObject *kwds)
 {
     /* initialize Encoder object */
-    static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", "key_memo", "use_decimal", "namedtuple_as_object", "tuple_as_array", NULL};
+    static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", "key_memo", "use_decimal", "namedtuple_as_object", "tuple_as_array", "iterable_as_array", NULL};
 
     PyEncoderObject *s;
     PyObject *markers, *defaultfn, *encoder, *indent, *key_separator;
-    PyObject *item_separator, *sort_keys, *skipkeys, *allow_nan, *key_memo, *use_decimal, *namedtuple_as_object, *tuple_as_array;
+    PyObject *item_separator, *sort_keys, *skipkeys, *allow_nan, *key_memo, *use_decimal, *namedtuple_as_object, *tuple_as_array, *iterable_as_array;
 
     assert(PyEncoder_Check(self));
     s = (PyEncoderObject *)self;
@@ -2029,7 +2030,7 @@ encoder_init(PyObject *self, PyObject *args, PyObject *kwds)
     if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOOOOOO:make_encoder", kwlist,
         &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator,
         &sort_keys, &skipkeys, &allow_nan, &key_memo, &use_decimal,
-        &namedtuple_as_object, &tuple_as_array))
+        &namedtuple_as_object, &tuple_as_array, &iterable_as_array))
         return -1;
 
     s->markers = markers;
@@ -2046,6 +2047,7 @@ encoder_init(PyObject *self, PyObject *args, PyObject *kwds)
     s->use_decimal = PyObject_IsTrue(use_decimal);
     s->namedtuple_as_object = PyObject_IsTrue(namedtuple_as_object);
     s->tuple_as_array = PyObject_IsTrue(tuple_as_array);
+    s->iterable_as_array = PyObject_IsTrue(iterable_as_array);
 
     Py_INCREF(s->markers);
     Py_INCREF(s->defaultfn);
@@ -2210,6 +2212,16 @@ encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssi
         else {
             PyObject *ident = NULL;
             PyObject *newobj;
+            if (s->iterable_as_array) {
+                newobj = PyObject_GetIter(obj);
+                if (newobj == NULL)
+                    PyErr_Clear();
+                else {
+                    rv = encoder_listencode_list(s, rval, newobj, indent_level);
+                    Py_DECREF(newobj);
+                    break;
+                }
+            }
             if (s->markers != Py_None) {
                 int has_key;
                 ident = PyLong_FromVoidPtr(obj);
diff --git a/simplejson/encoder.py b/simplejson/encoder.py
index 5ec7440..75ba993 100644
--- a/simplejson/encoder.py
+++ b/simplejson/encoder.py
@@ -107,7 +107,7 @@ class JSONEncoder(object):
             check_circular=True, allow_nan=True, sort_keys=False,
             indent=None, separators=None, encoding='utf-8', default=None,
             use_decimal=True, namedtuple_as_object=True,
-            tuple_as_array=True):
+            tuple_as_array=True, iterable_as_array=False):
         """Constructor for JSONEncoder, with sensible defaults.
 
         If skipkeys is false, then it is a TypeError to attempt
@@ -157,9 +157,14 @@ class JSONEncoder(object):
 
         If namedtuple_as_object is true (the default), tuple subclasses with
         ``_asdict()`` methods will be encoded as JSON objects.
-        
+
         If tuple_as_array is true (the default), tuple (and subclasses) will
         be encoded as JSON arrays.
+
+        If *iterable_as_array* is true (default: ``False``),
+        any object not in the above table that implements ``__iter__()``
+        will be encoded as a JSON array.
+
         """
 
         self.skipkeys = skipkeys
@@ -170,6 +175,7 @@ class JSONEncoder(object):
         self.use_decimal = use_decimal
         self.namedtuple_as_object = namedtuple_as_object
         self.tuple_as_array = tuple_as_array
+        self.iterable_as_array = iterable_as_array
         if isinstance(indent, (int, long)):
             indent = ' ' * indent
         self.indent = indent
@@ -285,13 +291,15 @@ class JSONEncoder(object):
                 markers, self.default, _encoder, self.indent,
                 self.key_separator, self.item_separator, self.sort_keys,
                 self.skipkeys, self.allow_nan, key_memo, self.use_decimal,
-                self.namedtuple_as_object, self.tuple_as_array)
+                self.namedtuple_as_object, self.tuple_as_array,
+                self.iterable_as_array)
         else:
             _iterencode = _make_iterencode(
                 markers, self.default, _encoder, self.indent, floatstr,
                 self.key_separator, self.item_separator, self.sort_keys,
                 self.skipkeys, _one_shot, self.use_decimal,
-                self.namedtuple_as_object, self.tuple_as_array)
+                self.namedtuple_as_object, self.tuple_as_array,
+                self.iterable_as_array)
         try:
             return _iterencode(o, 0)
         finally:
@@ -328,6 +336,7 @@ class JSONEncoderForHTML(JSONEncoder):
 def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
         _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
         _use_decimal, _namedtuple_as_object, _tuple_as_array,
+        _iterable_as_array,
         ## HACK: hand-optimized bytecode; turn globals into locals
         False=False,
         True=True,
@@ -343,6 +352,7 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
         long=long,
         str=str,
         tuple=tuple,
+        iter=iter,
     ):
 
     def _iterencode_list(lst, _current_indent_level):
@@ -520,6 +530,16 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
         elif _use_decimal and isinstance(o, Decimal):
             yield str(o)
         else:
+            while _iterable_as_array:
+                # Markers are not checked here because it is valid for an
+                # iterable to return self.
+                try:
+                    o = iter(o)
+                except TypeError:
+                    break
+                for chunk in _iterencode_list(o, _current_indent_level):
+                    yield chunk
+                return
             if markers is not None:
                 markerid = id(o)
                 if markerid in markers:
diff --git a/simplejson/tests/test_iterable.py b/simplejson/tests/test_iterable.py
new file mode 100644
index 0000000..4b37d00
--- /dev/null
+++ b/simplejson/tests/test_iterable.py
@@ -0,0 +1,31 @@
+import unittest
+from StringIO import StringIO
+
+import simplejson as json
+
+def iter_dumps(obj, **kw):
+    return ''.join(json.JSONEncoder(**kw).iterencode(obj))
+
+def sio_dump(obj, **kw):
+    sio = StringIO()
+    json.dumps(obj, **kw)
+    return sio.getvalue()
+
+class TestIterable(unittest.TestCase):
+    def test_iterable(self):
+        l = [1, 2, 3]
+        for dumps in (json.dumps, iter_dumps, sio_dump):
+            expect = dumps(l)
+            default_expect = dumps(sum(l))
+            # Default is False
+            self.assertRaises(TypeError, dumps, iter(l))
+            self.assertRaises(TypeError, dumps, iter(l), iterable_as_array=False)
+            self.assertEqual(expect, dumps(iter(l), iterable_as_array=True))
+            # Ensure that the "default" gets called
+            self.assertEqual(default_expect, dumps(iter(l), default=sum))
+            self.assertEqual(default_expect, dumps(iter(l), iterable_as_array=False, default=sum))
+            # Ensure that the "default" does not get called
+            self.assertEqual(
+                default_expect,
+                dumps(iter(l), iterable_as_array=True, default=sum))
+        
\ No newline at end of file
diff --git a/simplejson/tests/test_tuple.py b/simplejson/tests/test_tuple.py
index 92856a7..cff9a75 100644
--- a/simplejson/tests/test_tuple.py
+++ b/simplejson/tests/test_tuple.py
@@ -43,7 +43,3 @@ class TestTuples(unittest.TestCase):
         self.assertEqual(
             json.dumps(repr(t)),
             sio.getvalue())
-
-class TestNamedTuple(unittest.TestCase):
-    def test_namedtuple_dump(self):
-        pass
\ No newline at end of file