support packing long and tuple. add missing files.

3 files changed, 563 insertions, 17 deletions

diff --git a/msgpack.pyx b/msgpack.pyx
index 6885317..ceae9b6 100644
--- a/msgpack.pyx
+++ b/msgpack.pyx
@@ -3,15 +3,17 @@
 
 cdef extern from "Python.h":
     ctypedef char* const_char_ptr "const char*"
+    ctypedef struct PyObject
     cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len)
 
 cdef extern from "stdlib.h":
     void* malloc(int)
     void free(void*)
+
 cdef extern from "string.h":
     int memcpy(char*dst, char*src, unsigned int size)
 
-cdef extern from "msgpack/pack.h":
+cdef extern from "pack.h":
     ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len)
 
     struct msgpack_packer:
@@ -30,7 +32,7 @@ cdef extern from "msgpack/pack.h":
     void msgpack_pack_raw(msgpack_packer* pk, size_t l)
     void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
 
-cdef extern from "msgpack/unpack.h":
+cdef extern from "unpack.h":
     ctypedef struct msgpack_unpacker
 
 
@@ -98,7 +100,7 @@ cdef class Packer:
         """
         msgpack_pack_map(&self.pk, len)
 
-    def __call__(self, object o):
+    def pack(self, object o):
         cdef long long intval
         cdef double fval
         cdef char* rawval 
@@ -109,6 +111,9 @@ cdef class Packer:
             msgpack_pack_true(&self.pk)
         elif o is False:
             msgpack_pack_false(&self.pk)
+        elif isinstance(o, long):
+            intval = o
+            msgpack_pack_long_long(&self.pk, intval)
         elif isinstance(o, int):
             intval = o
             msgpack_pack_long_long(&self.pk, intval)
@@ -120,24 +125,21 @@ cdef class Packer:
             msgpack_pack_raw(&self.pk, len(o))
             msgpack_pack_raw_body(&self.pk, rawval, len(o))
         elif isinstance(o, unicode):
-            # todo
-            pass
+            o = o.encode('utf-8')
+            rawval = o
+            msgpack_pack_raw(&self.pk, len(o))
+            msgpack_pack_raw_body(&self.pk, rawval, len(o))
         elif isinstance(o, dict):
             msgpack_pack_map(&self.pk, len(o))
             for k,v in o.iteritems():
-                self(k)
-                self(v)
-        elif isinstance(o, tuple):
-            msgpack_pack_array(&self.pk, len(o))
-            for v in o:
-                self(v)
-        elif isinstance(o, list):
+                self.pack(k)
+                self.pack(v)
+        elif isinstance(o, tuple) or isinstance(o, list):
             msgpack_pack_array(&self.pk, len(o))
             for v in o:
-                self(v)
-        elif hasattr(o, "__msgpack__"):
-            o.__msgpack__(self)
+                self.pack(v)
         else:
+            # TODO: Serialize with defalt() like simplejson.
             raise TypeError, "can't serialize %r" % (o,)
 
 cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l):
@@ -155,8 +157,25 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l):
         packer.length += l
     return 0
 
+cdef extern from "msgpack/zone.h":
+    ctypedef struct msgpack_zone
+
+cdef extern from "unpack.c":
+    ctypedef struct template_context:
+        pass
+    int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off)
+    void template_init(template_context* ctx)
+    PyObject* template_data(template_context* ctx)
+
+
 cdef class Unpacker:
     def __init__(self):
         pass
-    def unpack(strm):
-        pass
+
+    def unpack(self, bytes_):
+        cdef const_char_ptr p = bytes_
+        cdef template_context ctx
+        cdef size_t off = 0
+        template_init(&ctx)
+        template_execute(&ctx, p, len(bytes_), &off)
+        return <object> template_data(&ctx)
diff --git a/pack.h b/pack.h
new file mode 100644
index 0000000..4a336d5
--- /dev/null
+++ b/pack.h
@@ -0,0 +1,130 @@
+/*
+ * MessagePack for Python packing routine
+ *
+ * Copyright (C) 2009 Naoki INADA
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_PACK_H__
+#define MSGPACK_PACK_H__
+
+#if _MSC_VER
+typedef signed char uint8_t;
+typedef unsigned char uint8_t;
+typedef short int16_t;
+typedef unsigned short uint16_t;
+typedef int int32_t;
+typedef unsigned int uint32_t;
+typedef long long int64_t;
+typedef unsigned long long uint64_t;
+#elif
+#include <stdint.h>
+#endif
+
+#include <stddef.h>
+#include <stdlib.h>
+#include "msgpack/pack_define.h"
+#include "msgpack/object.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len);
+
+typedef struct msgpack_packer {
+	void* data;
+	msgpack_packer_write callback;
+} msgpack_packer;
+
+static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback);
+
+static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback);
+static void msgpack_packer_free(msgpack_packer* pk);
+
+static int msgpack_pack_short(msgpack_packer* pk, short d);
+static int msgpack_pack_int(msgpack_packer* pk, int d);
+static int msgpack_pack_long(msgpack_packer* pk, long d);
+static int msgpack_pack_long_long(msgpack_packer* pk, long long d);
+static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d);
+static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d);
+static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d);
+static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d);
+
+static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d);
+static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d);
+static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d);
+static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d);
+static int msgpack_pack_int8(msgpack_packer* pk, int8_t d);
+static int msgpack_pack_int16(msgpack_packer* pk, int16_t d);
+static int msgpack_pack_int32(msgpack_packer* pk, int32_t d);
+static int msgpack_pack_int64(msgpack_packer* pk, int64_t d);
+
+static int msgpack_pack_float(msgpack_packer* pk, float d);
+static int msgpack_pack_double(msgpack_packer* pk, double d);
+
+static int msgpack_pack_nil(msgpack_packer* pk);
+static int msgpack_pack_true(msgpack_packer* pk);
+static int msgpack_pack_false(msgpack_packer* pk);
+
+static int msgpack_pack_array(msgpack_packer* pk, unsigned int n);
+
+static int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
+
+static int msgpack_pack_raw(msgpack_packer* pk, size_t l);
+static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
+
+int msgpack_pack_object(msgpack_packer* pk, msgpack_object d);
+
+
+
+#define msgpack_pack_inline_func(name) \
+	static inline int msgpack_pack ## name
+
+#define msgpack_pack_inline_func_cint(name) \
+	static inline int msgpack_pack ## name
+
+#define msgpack_pack_user msgpack_packer*
+
+#define msgpack_pack_append_buffer(user, buf, len) \
+	return (*(user)->callback)((user)->data, (const char*)buf, len)
+
+#include "msgpack/pack_template.h"
+
+static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback)
+{
+	pk->data = data;
+	pk->callback = callback;
+}
+
+static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback)
+{
+	msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer));
+	if(!pk) { return NULL; }
+	msgpack_packer_init(pk, data, callback);
+	return pk;
+}
+
+static inline void msgpack_packer_free(msgpack_packer* pk)
+{
+	free(pk);
+}
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* msgpack/pack.h */
+
diff --git a/unpack.h b/unpack.h
new file mode 100644
index 0000000..d578360
--- /dev/null
+++ b/unpack.h
@@ -0,0 +1,397 @@
+/*
+ * MessagePack for Python unpacking routine
+ *
+ * Copyright (C) 2009 Naoki INADA
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#include "msgpack/unpack.h"
+#include "msgpack/unpack_define.h"
+#include <stdlib.h>
+
+#include "Python.h"
+
+
+typedef struct {
+    int reserved;
+} unpack_user;
+
+
+#define msgpack_unpack_struct(name) \
+	struct template ## name
+
+#define msgpack_unpack_func(ret, name) \
+	ret template ## name
+
+#define msgpack_unpack_callback(name) \
+	template_callback ## name
+
+#define msgpack_unpack_object PyObject*
+
+#define msgpack_unpack_user unpack_user
+
+
+struct template_context;
+typedef struct template_context template_context;
+
+static void template_init(template_context* ctx);
+
+static msgpack_unpack_object template_data(template_context* ctx);
+
+static int template_execute(template_context* ctx,
+		const char* data, size_t len, size_t* off);
+
+
+static inline msgpack_unpack_object template_callback_root(unpack_user* u)
+{ PyObject *o = Py_None; Py_INCREF(o); return o; }
+
+static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o)
+{ *o = PyInt_FromLong((long)d); return 0; }
+
+static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o)
+{ *o = PyInt_FromLong((long)d); return 0; }
+
+static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o)
+{
+    if (d >= 0x80000000UL) {
+        *o = PyLong_FromUnsignedLongLong((unsigned long long)d);
+    } else {
+        *o = PyInt_FromLong((long)d);
+    }
+    return 0;
+}
+
+static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o)
+{ *o = PyLong_FromUnsignedLongLong(d); return 0; }
+
+static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o)
+{ *o = PyInt_FromLong(d); return 0; }
+
+static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o)
+{ *o = PyInt_FromLong(d); return 0; }
+
+static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o)
+{ *o = PyInt_FromLong(d); return 0; }
+
+static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o)
+{ *o = PyLong_FromLongLong(d); return 0; }
+
+static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o)
+{ *o = PyFloat_FromDouble((double)d); return 0; }
+
+static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o)
+{ *o = PyFloat_FromDouble(d); return 0; }
+
+static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o)
+{ *o = Py_None; Py_INCREF(o); return 0; }
+
+static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o)
+{ *o = Py_True; Py_INCREF(o); return 0; }
+
+static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o)
+{ *o = Py_False; Py_INCREF(o); return 0; }
+
+static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
+{
+    /* TODO: use PyList_New(n). */
+    *o = PyList_New(0);
+    return 0;
+}
+
+static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o)
+{
+    PyList_Append(*c, o);
+    return 0;
+}
+
+static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
+{
+    *o = PyDict_New();
+    return 0;
+}
+
+static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
+{
+    PyDict_SetItem(*c, k, v);
+	return 0;
+}
+
+static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
+{
+    *o = PyString_FromStringAndSize(p, l);
+	return 0;
+}
+
+#include "msgpack/unpack_template.h"
+
+
+#if 0
+#define CTX_CAST(m) ((template_context*)(m))
+#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced
+
+
+static const size_t COUNTER_SIZE = sizeof(unsigned int);
+
+static inline void init_count(void* buffer)
+{
+	*(volatile unsigned int*)buffer = 1;
+}
+
+static inline void decl_count(void* buffer)
+{
+	//if(--*(unsigned int*)buffer == 0) {
+	if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) {
+		free(buffer);
+	}
+}
+
+static inline void incr_count(void* buffer)
+{
+	//++*(unsigned int*)buffer;
+	__sync_add_and_fetch((unsigned int*)buffer, 1);
+}
+
+static inline unsigned int get_count(void* buffer)
+{
+	return *(volatile unsigned int*)buffer;
+}
+
+
+
+bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size)
+{
+	if(initial_buffer_size < COUNTER_SIZE) {
+		initial_buffer_size = COUNTER_SIZE;
+	}
+
+	char* buffer = (char*)malloc(initial_buffer_size);
+	if(buffer == NULL) {
+		return false;
+	}
+
+	void* ctx = malloc(sizeof(template_context));
+	if(ctx == NULL) {
+		free(buffer);
+		return false;
+	}
+
+	msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE);
+	if(z == NULL) {
+		free(ctx);
+		free(buffer);
+		return false;
+	}
+
+	mpac->buffer = buffer;
+	mpac->used = COUNTER_SIZE;
+	mpac->free = initial_buffer_size - mpac->used;
+	mpac->off = COUNTER_SIZE;
+	mpac->parsed = 0;
+	mpac->initial_buffer_size = initial_buffer_size;
+	mpac->z = z;
+	mpac->ctx = ctx;
+
+	init_count(mpac->buffer);
+
+	template_init(CTX_CAST(mpac->ctx));
+	CTX_CAST(mpac->ctx)->user.z = mpac->z;
+	CTX_CAST(mpac->ctx)->user.referenced = false;
+
+	return true;
+}
+
+void msgpack_unpacker_destroy(msgpack_unpacker* mpac)
+{
+	msgpack_zone_free(mpac->z);
+	free(mpac->ctx);
+	decl_count(mpac->buffer);
+}
+
+
+msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size)
+{
+	msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker));
+	if(mpac == NULL) {
+		return NULL;
+	}
+
+	if(!msgpack_unpacker_init(mpac, initial_buffer_size)) {
+		free(mpac);
+		return NULL;
+	}
+
+	return mpac;
+}
+
+void msgpack_unpacker_free(msgpack_unpacker* mpac)
+{
+	msgpack_unpacker_destroy(mpac);
+	free(mpac);
+}
+
+bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size)
+{
+	if(mpac->used == mpac->off && get_count(mpac->buffer) == 1
+			&& !CTX_REFERENCED(mpac)) {
+		// rewind buffer
+		mpac->free += mpac->used - COUNTER_SIZE;
+		mpac->used = COUNTER_SIZE;
+		mpac->off = COUNTER_SIZE;
+
+		if(mpac->free >= size) {
+			return true;
+		}
+	}
+
+	if(mpac->off == COUNTER_SIZE) {
+		size_t next_size = (mpac->used + mpac->free) * 2;  // include COUNTER_SIZE
+		while(next_size < size + mpac->used) {
+			next_size *= 2;
+		}
+
+		char* tmp = (char*)realloc(mpac->buffer, next_size);
+		if(tmp == NULL) {
+			return false;
+		}
+
+		mpac->buffer = tmp;
+		mpac->free = next_size - mpac->used;
+
+	} else {
+		size_t next_size = mpac->initial_buffer_size;  // include COUNTER_SIZE
+		size_t not_parsed = mpac->used - mpac->off;
+		while(next_size < size + not_parsed + COUNTER_SIZE) {
+			next_size *= 2;
+		}
+
+		char* tmp = (char*)malloc(next_size);
+		if(tmp == NULL) {
+			return false;
+		}
+
+		init_count(tmp);
+
+		memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed);
+
+		if(CTX_REFERENCED(mpac)) {
+			if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) {
+				free(tmp);
+				return false;
+			}
+			CTX_REFERENCED(mpac) = false;
+		} else {
+			decl_count(mpac->buffer);
+		}
+
+		mpac->buffer = tmp;
+		mpac->used = not_parsed + COUNTER_SIZE;
+		mpac->free = next_size - mpac->used;
+		mpac->off = COUNTER_SIZE;
+	}
+
+	return true;
+}
+
+int msgpack_unpacker_execute(msgpack_unpacker* mpac)
+{
+	size_t off = mpac->off;
+	int ret = template_execute(CTX_CAST(mpac->ctx),
+			mpac->buffer, mpac->used, &mpac->off);
+	if(mpac->off > off) {
+		mpac->parsed += mpac->off - off;
+	}
+	return ret;
+}
+
+msgpack_unpack_object msgpack_unpacker_data(msgpack_unpacker* mpac)
+{
+	return template_data(CTX_CAST(mpac->ctx));
+}
+
+msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac)
+{
+	if(!msgpack_unpacker_flush_zone(mpac)) {
+		return false;
+	}
+
+	msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE);
+	if(r == NULL) {
+		return NULL;
+	}
+
+	msgpack_zone* old = mpac->z;
+	mpac->z = r;
+
+	return old;
+}
+
+void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac)
+{
+	msgpack_zone_clear(mpac->z);
+}
+
+bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac)
+{
+	if(CTX_REFERENCED(mpac)) {
+		if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) {
+			return false;
+		}
+		CTX_REFERENCED(mpac) = false;
+
+		incr_count(mpac->buffer);
+	}
+
+	return true;
+}
+
+void msgpack_unpacker_reset(msgpack_unpacker* mpac)
+{
+	template_init(CTX_CAST(mpac->ctx));
+	// don't reset referenced flag
+	mpac->parsed = 0;
+}
+
+
+msgpack_unpack_return
+msgpack_unpack(const char* data, size_t len, size_t* off,
+		msgpack_zone* z, msgpack_unpack_object* result)
+{
+	template_context ctx;
+	template_init(&ctx);
+
+	ctx.user.z = z;
+	ctx.user.referenced = false;
+
+	size_t noff = 0;
+	if(off != NULL) { noff = *off; }
+
+	int ret = template_execute(&ctx, data, len, &noff);
+	if(ret < 0) {
+		return MSGPACK_UNPACK_PARSE_ERROR;
+	}
+
+	if(off != NULL) { *off = noff; }
+
+	if(ret == 0) {
+		return MSGPACK_UNPACK_CONTINUE;
+	}
+
+	*result = template_data(&ctx);
+
+	if(noff < len) {
+		return MSGPACK_UNPACK_EXTRA_BYTES;
+	}
+
+	return MSGPACK_UNPACK_SUCCESS;
+}
+#endif