Array tokenization seems working.

8 files changed, 126 insertions, 33 deletions

diff --git a/ChangeLog b/ChangeLog
index 2856d32..11b2157 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -17,8 +17,17 @@
 
 	* scripts/buildtypes.py: new version to include array data.
 
+2005-03-15  Federico Di Gregorio  <fog@debian.org>
+
+	* lib/extensions.py: Added AsIs import.
+
 2005-03-12  Federico Di Gregorio  <fog@debian.org>
 
+	* psycopg/cursor.h: removed "qattr", not used anymore and added
+	"cast", holding the typecaster currently in use.
+
+	* Release 1.99.13.
+
 	* psycopg/cursor_type.c (psyco_curs_executemany): implemented as a
 	wrapper to extract python arguments and then call
 	_psyco_curs_execute().
diff --git a/MANIFEST.in b/MANIFEST.in
index f24d138..680d383 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,8 +1,8 @@
 recursive-include psycopg *.c *.h
 recursive-include lib *.py
+recursive-include tests *.py
 recursive-include ZPsycopgDA *.py *.gif *.dtml
 recursive-include examples *.py somehackers.jpg whereareyou.jpg
-#recursive-include tests *.py
 recursive-include doc TODO HACKING SUCCESS ChangeLog-1.x
 include scripts/maketypes.sh scripts/buildtypes.py
 include AUTHORS README INSTALL ChangeLog setup.py setup.cfg
diff --git a/NEWS b/NEWS
index d5ff88a..32a1f27 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,11 @@
+What's new in psycopg 1.99.13
+-----------------------------
+
+* Added missing .executemany() method.
+
+* Optimized type cast from PostgreSQL to Python (psycopg should be even
+  faster than before.)
+
 What's new in psycopg 1.99.12
 -----------------------------
 
diff --git a/lib/extensions.py b/lib/extensions.py
index ccadf2b..6a18878 100644
--- a/lib/extensions.py
+++ b/lib/extensions.py
@@ -25,7 +25,7 @@ This module holds all the extensions to the DBAPI-2.0 provided by psycopg:
 from _psycopg import UNICODE, INTEGER, LONGINTEGER, BOOLEAN, FLOAT
 from _psycopg import TIME, DATE, INTERVAL
 
-from _psycopg import Boolean, QuotedString
+from _psycopg import Boolean, QuotedString, AsIs
 try:
     from _psycopg import DateFromMx, TimeFromMx, TimestampFromMx
     from _psycopg import IntervalFromMx
diff --git a/psycopg/typecast.c b/psycopg/typecast.c
index da13cb8..aa23bf0 100644
--- a/psycopg/typecast.c
+++ b/psycopg/typecast.c
@@ -468,11 +468,9 @@ typecast_cast(PyObject *obj, unsigned char *str, int len, PyObject *curs)
     ((cursorObject*)curs)->caster = obj;
     
     if (self->ccast) {
-        Dprintf("typecast_call: calling C cast function");
         res = self->ccast(str, len, curs);
     }
     else if (self->pcast) {
-        Dprintf("typecast_call: calling python callable");
         res = PyObject_CallFunction(self->pcast, "s#O", str, len, curs);
     }
     else {
diff --git a/psycopg/typecast_array.c b/psycopg/typecast_array.c
index d87c10d..74dee6c 100644
--- a/psycopg/typecast_array.c
+++ b/psycopg/typecast_array.c
@@ -19,9 +19,12 @@
  * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  */
 
+#define MAX_DIMENSIONS 16
+
 
 /** typecast_array_scan - scan a string looking for array items **/
 
+#define ASCAN_ERROR -1
 #define ASCAN_EOF    0
 #define ASCAN_BEGIN  1
 #define ASCAN_END    2
@@ -32,16 +35,20 @@ static int
 typecast_array_tokenize(unsigned char *str, int strlength,
                         int *pos, unsigned char** token, int *length)
 {
-    int i;
-    int quoted = 0;
-
+    int i, l, res = ASCAN_TOKEN;
+    int qs = 0; /* 2 = in quotes, 1 = quotes closed */
+    
     /* first we check for quotes, used when the content of the item contains
        special or quoted characters */
+    
     if (str[*pos] == '"') {
-        quoted = 1;
+        qs = 2;
         *pos += 1;
     }
 
+    Dprintf("typecast_array_tokenize: '%s'; %d/%d",
+            &str[*pos], *pos, strlength);
+
     for (i = *pos ; i < strlength ; i++) {
         switch (str[i]) {
         case '{':
@@ -49,49 +56,116 @@ typecast_array_tokenize(unsigned char *str, int strlength,
             return ASCAN_BEGIN;
 
         case '}':
-            *pos = i+1;
-            return ASCAN_END;
-
+            /* we tokenize the last item in the array and then return it to
+               the user togheter with the closing bracket marker */
+            res = ASCAN_END;
+            goto tokenize;
+
+        case '"':
+            /* this will close the quoting only if the previous character was
+               NOT a backslash */
+            if (qs == 2 && str[i-1] != '\\') qs = 1;
+            continue;
+               
+        case '\\':
+            /* something has been quoted, sigh, we'll need a copy buffer */
+            res = ASCAN_QUOTED;
+            continue;
+            
         case ',':
-            *token = &str[*pos];
-            *length = i - *pos;
-            if (quoted == 1)
-                *length -= 1;
-            *pos = i+1;
-            return ASCAN_TOKEN;
-
-        default:
-            /* nothing to do right now */
-            break;
+            /* if we're inside quotes we use the comma as a normal char */
+            if (qs == 2)
+                continue;
+            else
+                goto tokenize;
         }
     }
 
-    *token = &str[*pos];
-    *length = i - *pos;
-    if (quoted == 1)
-        *length -= 1;
+    res = ASCAN_EOF;
+    
+ tokenize:
+    l = i - *pos - qs;
+    
+    /* if res is ASCAN_QUOTED we need to copy the string to a newly allocated
+       buffer and return it */
+    if (res == ASCAN_QUOTED) {
+        unsigned char *buffer = PyMem_Malloc(l+1);
+        if (buffer == NULL) return ASCAN_ERROR;
+
+        *token = buffer;
+        
+        for (i = *pos; i < l+*pos; i++) {
+            if (str[i] != '\\')
+                *(buffer++) = str[i];
+        }
+        *buffer = '\0';
+        *length = (int)buffer - (int)*token;
+        *pos = i+2;
+    }
+    else {
+        *token = &str[*pos];
+        *length = l;
+        *pos = i+1;
+        if (res == ASCAN_END && str[*pos] == ',')
+            *pos += 1; /* skip both the bracket and the comma */
+    }
     
-    return ASCAN_EOF;
+    return res;
 }
 
 static int
 typecast_array_scan(unsigned char *str, int strlength,
                     PyObject *curs, PyObject *base, PyObject *array)
 {
-    int state, length, pos = 0;
+    int state, length, bracket = 0, pos = 0;
     unsigned char *token;
+
+    PyObject *stack[MAX_DIMENSIONS];
+    int stack_index = 0;
     
     while (1) {
         state = typecast_array_tokenize(str, strlength, &pos, &token, &length);
-        
-        if (state == ASCAN_TOKEN || state ==  ASCAN_EOF) {
+        if (state == ASCAN_TOKEN
+            || state == ASCAN_QUOTED 
+            || (state ==  ASCAN_EOF && bracket == 0)
+            || (state == ASCAN_END && bracket == 0)) {
+            
             PyObject *obj = typecast_cast(base, token, length, curs);
+
+            /* before anything else we free the memory */
+            if (state == ASCAN_QUOTED) PyMem_Free(token);
             if (obj == NULL) return 0;
+
             PyList_Append(array, obj);
             Py_DECREF(obj);
         }
-        else {
-            Dprintf("** RECURSION (not supported right now)!!");
+        else if (state == ASCAN_BEGIN) {
+            PyObject *sub = PyList_New(0);            
+            if (sub == NULL) return 0;
+
+            PyList_Append(array, sub);
+            Py_DECREF(sub);
+
+            if (stack_index == MAX_DIMENSIONS)
+                return 0;
+
+            stack[stack_index++] = array;
+            array = sub;
+        }
+        else if (state == ASCAN_ERROR) {
+            return 0;
+        }
+
+        /* reset the closing bracket marker just before cheking for ASCAN_END:
+           this is to make sure we don't mistake two closing brackets for an
+           empty item */
+        bracket = 0;
+        
+        if (state == ASCAN_END) {
+            if (--stack_index < 0)
+                return 0;
+            array = stack[stack_index];
+            bracket = 1;
         }
 
         if (state ==  ASCAN_EOF) break;
@@ -115,6 +189,8 @@ typecast_GENERIC_ARRAY_cast(unsigned char *str, int len, PyObject *curs)
         PyErr_SetString(Error, "array does not start with '{'");
         return NULL;
     }
+
+    Dprintf("typecast_GENERIC_ARRAY_cast: scanning %s", str);
     
     obj = PyList_New(0);
 
diff --git a/sandbox/array.py b/sandbox/array.py
index b9975a0..af09140 100644
--- a/sandbox/array.py
+++ b/sandbox/array.py
@@ -9,7 +9,9 @@ print curs.fetchone()
 curs.execute("SELECT ARRAY['1','2','3'] AS foo")
 print curs.fetchone()
 
-curs.execute("""SELECT ARRAY['','"',''] AS foo""")
+curs.execute("""SELECT ARRAY[',','"','\\\\'] AS foo""")
 d = curs.fetchone()
 print d, '->', d[0][0], d[0][1], d[0][2]
 
+curs.execute("SELECT ARRAY[ARRAY[1,2],ARRAY[3,4]] AS foo")
+print curs.fetchone()
diff --git a/setup.py b/setup.py
index 046d056..c985c76 100644
--- a/setup.py
+++ b/setup.py
@@ -47,7 +47,7 @@ from distutils.core import setup, Extension
 from distutils.sysconfig import get_python_inc
 import distutils.ccompiler
 
-PSYCOPG_VERSION = '1.99.13/devel'
+PSYCOPG_VERSION = '1.99.13'
 version_flags   = []
 
 have_pydatetime = False