- add new C extension "utils", so far includes distill_params

- repair test_processors which wasn't hitting the python functions
- add another suite to test_processors that does distill_params

9 files changed, 367 insertions, 53 deletions

diff --git a/CHANGES b/CHANGES
index 6d2332ad7..70c20ee18 100644
--- a/CHANGES
+++ b/CHANGES
@@ -284,6 +284,10 @@ underneath "0.7.xx".
     statement as is appropriate, else
     raise InvalidRequestError.  [ticket:2498]
 
+  - [feature] New C extension module "utils" has
+    been added for additional function speedups
+    as we have time to implement.
+
   - ResultProxy.last_inserted_ids is removed,
     replaced by inserted_primary_key.
 
diff --git a/lib/sqlalchemy/cextension/processors.c b/lib/sqlalchemy/cextension/processors.c
index 427db5d8e..c261142a7 100644
--- a/lib/sqlalchemy/cextension/processors.c
+++ b/lib/sqlalchemy/cextension/processors.c
@@ -1,5 +1,6 @@
 /*
 processors.c
+Copyright (C) 2010-2012 the SQLAlchemy authors and contributors <see AUTHORS file>
 Copyright (C) 2010 Gaetan de Menten gdementen@gmail.com
 
 This module is part of SQLAlchemy and is released under
diff --git a/lib/sqlalchemy/cextension/resultproxy.c b/lib/sqlalchemy/cextension/resultproxy.c
index ca3a8f40b..8c89baa25 100644
--- a/lib/sqlalchemy/cextension/resultproxy.c
+++ b/lib/sqlalchemy/cextension/resultproxy.c
@@ -1,5 +1,6 @@
 /*
 resultproxy.c
+Copyright (C) 2010-2012 the SQLAlchemy authors and contributors <see AUTHORS file>
 Copyright (C) 2010 Gaetan de Menten gdementen@gmail.com
 
 This module is part of SQLAlchemy and is released under
diff --git a/lib/sqlalchemy/cextension/utils.c b/lib/sqlalchemy/cextension/utils.c
new file mode 100644
index 000000000..8edd5d66c
--- /dev/null
+++ b/lib/sqlalchemy/cextension/utils.c
@@ -0,0 +1,189 @@
+/*
+utils.c
+Copyright (C) 2012 the SQLAlchemy authors and contributors <see AUTHORS file>
+
+This module is part of SQLAlchemy and is released under
+the MIT License: http://www.opensource.org/licenses/mit-license.php
+*/
+
+#include <Python.h>
+
+/*
+    Given arguments from the calling form *multiparams, **params,
+    return a list of bind parameter structures, usually a list of
+    dictionaries.
+
+    In the case of 'raw' execution which accepts positional parameters,
+    it may be a list of tuples or lists.
+
+ */
+static PyObject *
+distill_params(PyObject *self, PyObject *args)
+{
+	PyObject *multiparams, *params;
+	PyObject *enclosing_list, *double_enclosing_list;
+	PyObject *zero_element, *zero_element_item;
+	Py_ssize_t multiparam_size, zero_element_length;
+
+	if (!PyArg_UnpackTuple(args, "_distill_params", 2, 2, &multiparams, &params)) {
+		return NULL;
+	}
+
+	if (multiparams != Py_None) {
+		multiparam_size = PyTuple_Size(multiparams);
+		if (multiparam_size < 0) {
+			return NULL;
+		}
+	}
+
+	if (multiparams == Py_None || multiparam_size == 0) {
+		if (params != Py_None && PyDict_Size(params) != 0) {
+			enclosing_list = PyList_New(1);
+			if (enclosing_list == NULL) {
+				return NULL;
+			}
+			Py_INCREF(params);
+			if (PyList_SetItem(enclosing_list, 0, params) == -1) {
+				Py_DECREF(params);
+				Py_DECREF(enclosing_list);
+				return NULL;
+			}
+		}
+		else {
+			enclosing_list = PyList_New(0);
+			if (enclosing_list == NULL) {
+				return NULL;
+			}
+		}
+		return enclosing_list;
+	}
+	else if (multiparam_size == 1) {
+		zero_element = PyTuple_GetItem(multiparams, 0);
+		if (PyTuple_Check(zero_element) || PyList_Check(zero_element)) {
+			zero_element_length = PySequence_Length(zero_element);
+
+			if (zero_element_length != 0) {
+				zero_element_item = PySequence_GetItem(zero_element, 0);
+				if (zero_element_item == NULL) {
+					return NULL;
+				}
+			}
+
+			if (zero_element_length == 0 ||
+				PyObject_HasAttrString(zero_element_item, "__iter__") &&
+				!PyObject_HasAttrString(zero_element_item, "strip")
+				) {
+				/*
+				 * execute(stmt, [{}, {}, {}, ...])
+        		 * execute(stmt, [(), (), (), ...])
+				 */
+				Py_XDECREF(zero_element_item);
+				Py_INCREF(zero_element);
+				return zero_element;
+			}
+			else {
+				/*
+				 * execute(stmt, ("value", "value"))
+				 */
+				Py_XDECREF(zero_element_item);
+				enclosing_list = PyList_New(1);
+				if (enclosing_list == NULL) {
+					return NULL;
+				}
+				Py_INCREF(zero_element);
+				if (PyList_SetItem(enclosing_list, 0, zero_element) == -1) {
+					Py_DECREF(zero_element);
+					Py_DECREF(enclosing_list);
+					return NULL;
+				}
+				return enclosing_list;
+			}
+		}
+		else if (PyObject_HasAttrString(zero_element, "keys")) {
+			/*
+			 * execute(stmt, {"key":"value"})
+			 */
+			enclosing_list = PyList_New(1);
+			if (enclosing_list ==  NULL) {
+				return NULL;
+			}
+			Py_INCREF(zero_element);
+			if (PyList_SetItem(enclosing_list, 0, zero_element) == -1) {
+				Py_DECREF(zero_element);
+				Py_DECREF(enclosing_list);
+				return NULL;
+			}
+			return enclosing_list;
+		} else {
+			enclosing_list = PyList_New(1);
+			if (enclosing_list ==  NULL) {
+				return NULL;
+			}
+			double_enclosing_list = PyList_New(1);
+			if (double_enclosing_list == NULL) {
+				Py_DECREF(enclosing_list);
+				return NULL;
+			}
+			Py_INCREF(zero_element);
+			if (PyList_SetItem(enclosing_list, 0, zero_element) == -1) {
+				Py_DECREF(zero_element);
+				Py_DECREF(enclosing_list);
+				Py_DECREF(double_enclosing_list);
+				return NULL;
+			}
+			if (PyList_SetItem(double_enclosing_list, 0, enclosing_list) == -1) {
+				Py_DECREF(zero_element);
+				Py_DECREF(enclosing_list);
+				Py_DECREF(double_enclosing_list);
+				return NULL;
+			}
+			return double_enclosing_list;
+		}
+	}
+	else {
+		zero_element = PyTuple_GetItem(multiparams, 0);
+		if (PyObject_HasAttrString(zero_element, "__iter__") &&
+				!PyObject_HasAttrString(zero_element, "strip")
+			) {
+			Py_INCREF(multiparams);
+			return multiparams;
+		}
+		else {
+			enclosing_list = PyList_New(1);
+			if (enclosing_list ==  NULL) {
+				return NULL;
+			}
+			Py_INCREF(multiparams);
+			if (PyList_SetItem(enclosing_list, 0, multiparams) == -1) {
+				Py_DECREF(multiparams);
+				Py_DECREF(enclosing_list);
+				return NULL;
+			}
+			return enclosing_list;
+		}
+	}
+}
+
+#ifndef PyMODINIT_FUNC  /* declarations for DLL import/export */
+#define PyMODINIT_FUNC void
+#endif
+
+
+static PyMethodDef module_methods[] = {
+    {"_distill_params", distill_params, METH_VARARGS,
+     "Distill an execute() parameter structure."},
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+PyMODINIT_FUNC
+initcutils(void)
+{
+    PyObject *m;
+
+    m = Py_InitModule3("cutils", module_methods,
+                       "Internal utility functions.");
+    if (m == NULL)
+        return;
+
+}
+
diff --git a/lib/sqlalchemy/engine/util.py b/lib/sqlalchemy/engine/util.py
index bbbfe3cff..6bf8f2d3f 100644
--- a/lib/sqlalchemy/engine/util.py
+++ b/lib/sqlalchemy/engine/util.py
@@ -26,46 +26,6 @@ def _coerce_config(configuration, prefix):
         util.coerce_kw_type(options, option, type_)
     return options
 
-
-def _distill_params(multiparams, params):
-    """Given arguments from the calling form *multiparams, **params,
-    return a list of bind parameter structures, usually a list of
-    dictionaries.
-
-    In the case of 'raw' execution which accepts positional parameters,
-    it may be a list of tuples or lists.
-
-    """
-
-    if not multiparams:
-        if params:
-            return [params]
-        else:
-            return []
-    elif len(multiparams) == 1:
-        zero = multiparams[0]
-        if isinstance(zero, (list, tuple)):
-            if not zero or hasattr(zero[0], '__iter__') and \
-                    not hasattr(zero[0], 'strip'):
-                # execute(stmt, [{}, {}, {}, ...])
-                # execute(stmt, [(), (), (), ...])
-                return zero
-            else:
-                # execute(stmt, ("value", "value"))
-                return [zero]
-        elif hasattr(zero, 'keys'):
-            # execute(stmt, {"key":"value"})
-            return [zero]
-        else:
-            # execute(stmt, "value")
-            return [[zero]]
-    else:
-        if hasattr(multiparams[0], '__iter__') and \
-            not hasattr(multiparams[0], 'strip'):
-            return multiparams
-        else:
-            return [multiparams]
-
 def connection_memoize(key):
     """Decorator, memoize a function in a connection.info stash.
 
@@ -83,3 +43,51 @@ def connection_memoize(key):
             return val
 
     return decorated
+
+def py_fallback():
+    def _distill_params(multiparams, params):
+        """Given arguments from the calling form *multiparams, **params,
+        return a list of bind parameter structures, usually a list of
+        dictionaries.
+
+        In the case of 'raw' execution which accepts positional parameters,
+        it may be a list of tuples or lists.
+
+        """
+
+        if not multiparams:
+            if params:
+                return [params]
+            else:
+                return []
+        elif len(multiparams) == 1:
+            zero = multiparams[0]
+            if isinstance(zero, (list, tuple)):
+                if not zero or hasattr(zero[0], '__iter__') and \
+                        not hasattr(zero[0], 'strip'):
+                    # execute(stmt, [{}, {}, {}, ...])
+                    # execute(stmt, [(), (), (), ...])
+                    return zero
+                else:
+                    # execute(stmt, ("value", "value"))
+                    return [zero]
+            elif hasattr(zero, 'keys'):
+                # execute(stmt, {"key":"value"})
+                return [zero]
+            else:
+                # execute(stmt, "value")
+                return [[zero]]
+        else:
+            if hasattr(multiparams[0], '__iter__') and \
+                not hasattr(multiparams[0], 'strip'):
+                return multiparams
+            else:
+                return [multiparams]
+
+    return locals()
+try:
+    from sqlalchemy.cutils import _distill_params
+except ImportError:
+    globals().update(py_fallback())
+
+
diff --git a/lib/sqlalchemy/processors.py b/lib/sqlalchemy/processors.py
index 240263feb..ddca43a6c 100644
--- a/lib/sqlalchemy/processors.py
+++ b/lib/sqlalchemy/processors.py
@@ -29,10 +29,11 @@ def str_to_datetime_processor_factory(regexp, type_):
                 m = rmatch(value)
             except TypeError:
                 raise ValueError("Couldn't parse %s string '%r' "
-                                "- value is not a string." % (type_.__name__ , value))
+                                "- value is not a string." %
+                                (type_.__name__, value))
             if m is None:
                 raise ValueError("Couldn't parse %s string: "
-                                "'%s'" % (type_.__name__ , value))
+                                "'%s'" % (type_.__name__, value))
             if has_named_groups:
                 groups = m.groupdict(0)
                 return type_(**dict(zip(groups.iterkeys(),
diff --git a/setup.py b/setup.py
index e27b78b16..200ce055e 100644
--- a/setup.py
+++ b/setup.py
@@ -42,7 +42,9 @@ ext_modules = [
     Extension('sqlalchemy.cprocessors',
            sources=['lib/sqlalchemy/cextension/processors.c']),
     Extension('sqlalchemy.cresultproxy',
-           sources=['lib/sqlalchemy/cextension/resultproxy.c'])
+           sources=['lib/sqlalchemy/cextension/resultproxy.c']),
+    Extension('sqlalchemy.cutils',
+           sources=['lib/sqlalchemy/cextension/utils.c'])
     ]
 
 ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
diff --git a/test/engine/test_execute.py b/test/engine/test_execute.py
index 9a2fd0a5a..a23f8d05a 100644
--- a/test/engine/test_execute.py
+++ b/test/engine/test_execute.py
@@ -1107,7 +1107,7 @@ class EngineEventsTest(fixtures.TestBase):
         e1.execute(select([1]))
         e1.execute(select([1]).compile(dialect=e1.dialect).statement)
         e1.execute(select([1]).compile(dialect=e1.dialect))
-        e1._execute_compiled(select([1]).compile(dialect=e1.dialect), [], {})
+        e1._execute_compiled(select([1]).compile(dialect=e1.dialect), (), {})
 
     def test_exception_event(self):
         engine = engines.testing_engine()
diff --git a/test/engine/test_processors.py b/test/engine/test_processors.py
index d6b994e78..d05de6902 100644
--- a/test/engine/test_processors.py
+++ b/test/engine/test_processors.py
@@ -1,11 +1,6 @@
 from test.lib import fixtures
-from test.lib.testing import assert_raises_message
+from test.lib.testing import assert_raises_message, eq_
 
-from sqlalchemy import processors
-try:
-    from sqlalchemy import cprocessors
-except ImportError:
-    cprocessors = None
 
 class _DateProcessorTest(fixtures.TestBase):
     def test_date_no_string(self):
@@ -52,9 +47,122 @@ class _DateProcessorTest(fixtures.TestBase):
 
 
 class PyDateProcessorTest(_DateProcessorTest):
-    module = processors
-
+    @classmethod
+    def setup_class(cls):
+        from sqlalchemy import processors
+        cls.module = type("util", (object,),
+                dict(
+                    (k, staticmethod(v))
+                        for k, v in processors.py_fallback().items()
+                )
+            )
 
 class CDateProcessorTest(_DateProcessorTest):
     __requires__ = ('cextensions',)
-    module = cprocessors
+    @classmethod
+    def setup_class(cls):
+        from sqlalchemy import cprocessors
+        cls.module = cprocessors
+
+
+class _DistillArgsTest(fixtures.TestBase):
+    def test_distill_none(self):
+        eq_(
+            self.module._distill_params(None, None),
+            []
+        )
+
+    def test_distill_no_multi_no_param(self):
+        eq_(
+            self.module._distill_params((), {}),
+            []
+        )
+
+    def test_distill_dict_multi_none_param(self):
+        eq_(
+            self.module._distill_params(None, {"foo": "bar"}),
+            [{"foo": "bar"}]
+        )
+
+    def test_distill_dict_multi_empty_param(self):
+        eq_(
+            self.module._distill_params((), {"foo": "bar"}),
+            [{"foo": "bar"}]
+        )
+
+    def test_distill_single_dict(self):
+        eq_(
+            self.module._distill_params(({"foo": "bar"},), {}),
+            [{"foo": "bar"}]
+        )
+
+    def test_distill_single_list_strings(self):
+        eq_(
+            self.module._distill_params((["foo", "bar"],), {}),
+            [["foo", "bar"]]
+        )
+
+    def test_distill_single_list_tuples(self):
+        eq_(
+            self.module._distill_params(([("foo", "bar"), ("bat", "hoho")],), {}),
+            [('foo', 'bar'), ('bat', 'hoho')]
+        )
+
+    def test_distill_single_list_tuple(self):
+        eq_(
+            self.module._distill_params(([("foo", "bar")],), {}),
+            [('foo', 'bar')]
+        )
+
+    def test_distill_multi_list_tuple(self):
+        eq_(
+            self.module._distill_params(
+                ([("foo", "bar")], [("bar", "bat")]),
+                {}
+            ),
+            ([('foo', 'bar')], [('bar', 'bat')])
+        )
+
+    def test_distill_multi_strings(self):
+        eq_(
+            self.module._distill_params(("foo", "bar"), {}),
+            [('foo', 'bar')]
+        )
+
+    def test_distill_single_list_dicts(self):
+        eq_(
+            self.module._distill_params(([{"foo": "bar"}, {"foo": "hoho"}],), {}),
+            [{'foo': 'bar'}, {'foo': 'hoho'}]
+        )
+
+    def test_distill_single_string(self):
+        eq_(
+            self.module._distill_params(("arg",), {}),
+            [["arg"]]
+        )
+
+    def test_distill_multi_string_tuple(self):
+        eq_(
+            self.module._distill_params((("arg", "arg"),), {}),
+            [("arg", "arg")]
+        )
+
+
+
+class PyDistillArgsTest(_DistillArgsTest):
+    @classmethod
+    def setup_class(cls):
+        from sqlalchemy.engine import util
+        cls.module = type("util", (object,),
+                dict(
+                    (k, staticmethod(v))
+                        for k, v in util.py_fallback().items()
+                )
+        )
+
+class CDistillArgsTest(_DistillArgsTest):
+    __requires__ = ('cextensions', )
+    @classmethod
+    def setup_class(cls):
+        from sqlalchemy import cutils as util
+        cls.module = util