Make msgpack package instead of module. and using Cython in setup script.

4 files changed, 505 insertions, 0 deletions

diff --git a/python/msgpack/__init__.py b/python/msgpack/__init__.py
new file mode 100644
index 0000000..797b29c
--- /dev/null
+++ b/python/msgpack/__init__.py
@@ -0,0 +1,3 @@
+# coding: utf-8
+from _msgpack import *
+
diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx
new file mode 100644
index 0000000..aa4006b
--- /dev/null
+++ b/python/msgpack/_msgpack.pyx
@@ -0,0 +1,202 @@
+# coding: utf-8
+
+from cStringIO import StringIO
+
+cdef extern from "Python.h":
+    ctypedef char* const_char_ptr "const char*"
+    ctypedef struct PyObject
+    cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len)
+
+cdef extern from "stdlib.h":
+    void* malloc(int)
+    void free(void*)
+
+cdef extern from "string.h":
+    int memcpy(char*dst, char*src, unsigned int size)
+
+cdef extern from "pack.h":
+    ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len)
+
+    struct msgpack_packer:
+        void *data
+        msgpack_packer_write callback
+
+    void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback)
+    void msgpack_pack_int(msgpack_packer* pk, int d)
+    void msgpack_pack_nil(msgpack_packer* pk)
+    void msgpack_pack_true(msgpack_packer* pk)
+    void msgpack_pack_false(msgpack_packer* pk)
+    void msgpack_pack_long_long(msgpack_packer* pk, long long d)
+    void msgpack_pack_double(msgpack_packer* pk, double d)
+    void msgpack_pack_array(msgpack_packer* pk, size_t l)
+    void msgpack_pack_map(msgpack_packer* pk, size_t l)
+    void msgpack_pack_raw(msgpack_packer* pk, size_t l)
+    void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
+
+
+cdef int BUFF_SIZE=2*1024
+
+cdef class Packer:
+    """Packer that pack data into strm.
+
+    strm must have `write(bytes)` method.
+    size specifies local buffer size.
+    """
+    cdef char* buff
+    cdef unsigned int length
+    cdef unsigned int allocated
+    cdef msgpack_packer pk
+    cdef object strm
+
+    def __init__(self, strm, int size=0):
+        if size <= 0:
+            size = BUFF_SIZE
+
+        self.strm = strm
+        self.buff = <char*> malloc(size)
+        self.allocated = size
+        self.length = 0
+
+        msgpack_packer_init(&self.pk, <void*>self, <msgpack_packer_write>_packer_write)
+
+    def __del__(self):
+        free(self.buff);
+
+    def flush(self):
+        """Flash local buffer and output stream if it has 'flush()' method."""
+        if self.length > 0:
+            self.strm.write(PyString_FromStringAndSize(self.buff, self.length))
+            self.length = 0
+        if hasattr(self.strm, 'flush'):
+            self.strm.flush()
+
+    def pack_list(self, len):
+        """Start packing sequential objects.
+
+        Example:
+
+            packer.pack_list(2)
+            packer.pack('foo')
+            packer.pack('bar')
+
+        This code is same as below code:
+
+            packer.pack(['foo', 'bar'])
+        """
+        msgpack_pack_array(&self.pk, len)
+
+    def pack_dict(self, len):
+        """Start packing key-value objects.
+
+        Example:
+
+            packer.pack_dict(1)
+            packer.pack('foo')
+            packer.pack('bar')
+
+        This code is same as below code:
+
+            packer.pack({'foo', 'bar'})
+        """
+        msgpack_pack_map(&self.pk, len)
+
+    cdef __pack(self, object o):
+        cdef long long intval
+        cdef double fval
+        cdef char* rawval 
+
+        if o is None:
+            msgpack_pack_nil(&self.pk)
+        elif o is True:
+            msgpack_pack_true(&self.pk)
+        elif o is False:
+            msgpack_pack_false(&self.pk)
+        elif isinstance(o, long):
+            intval = o
+            msgpack_pack_long_long(&self.pk, intval)
+        elif isinstance(o, int):
+            intval = o
+            msgpack_pack_long_long(&self.pk, intval)
+        elif isinstance(o, float):
+            fval = o
+            msgpack_pack_double(&self.pk, fval)
+        elif isinstance(o, str):
+            rawval = o
+            msgpack_pack_raw(&self.pk, len(o))
+            msgpack_pack_raw_body(&self.pk, rawval, len(o))
+        elif isinstance(o, unicode):
+            o = o.encode('utf-8')
+            rawval = o
+            msgpack_pack_raw(&self.pk, len(o))
+            msgpack_pack_raw_body(&self.pk, rawval, len(o))
+        elif isinstance(o, dict):
+            msgpack_pack_map(&self.pk, len(o))
+            for k,v in o.iteritems():
+                self.pack(k)
+                self.pack(v)
+        elif isinstance(o, tuple) or isinstance(o, list):
+            msgpack_pack_array(&self.pk, len(o))
+            for v in o:
+                self.pack(v)
+        else:
+            # TODO: Serialize with defalt() like simplejson.
+            raise TypeError, "can't serialize %r" % (o,)
+
+    def pack(self, obj, flush=True):
+        self.__pack(obj)
+        if flush:
+            self.flush()
+
+cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l):
+    if packer.length + l > packer.allocated:
+        if packer.length > 0:
+            packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length))
+        if l > 64:
+            packer.strm.write(PyString_FromStringAndSize(b, l))
+            packer.length = 0
+        else:
+            memcpy(packer.buff, b, l)
+            packer.length = l
+    else:
+        memcpy(packer.buff + packer.length, b, l)
+        packer.length += l
+    return 0
+
+def pack(object o, object stream):
+    packer = Packer(stream)
+    packer.pack(o)
+    packer.flush()
+
+def packs(object o):
+    buf = StringIO()
+    packer = Packer(buf)
+    packer.pack(o)
+    packer.flush()
+    return buf.getvalue()
+
+cdef extern from "unpack.h":
+    ctypedef struct template_context:
+        pass
+    int template_execute(template_context* ctx, const_char_ptr data,
+            size_t len, size_t* off)
+    void template_init(template_context* ctx)
+    object template_data(template_context* ctx)
+
+
+def unpacks(object packed_bytes):
+    """Unpack packed_bytes to object. Returns unpacked object."""
+    cdef const_char_ptr p = packed_bytes
+    cdef template_context ctx
+    cdef size_t off = 0
+    template_init(&ctx)
+    template_execute(&ctx, p, len(packed_bytes), &off)
+    return template_data(&ctx)
+
+def unpack(object stream):
+    """unpack from stream."""
+    packed = stream.read()
+    return unpacks(packed)
+
+cdef class Unpacker:
+    """Do nothing. This function is for symmetric to Packer"""
+    unpack = staticmethod(unpacks)
diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h
new file mode 100644
index 0000000..f3935fb
--- /dev/null
+++ b/python/msgpack/pack.h
@@ -0,0 +1,121 @@
+/*
+ * MessagePack for Python packing routine
+ *
+ * Copyright (C) 2009 Naoki INADA
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#if _MSC_VER
+typedef signed char uint8_t;
+typedef unsigned char uint8_t;
+typedef short int16_t;
+typedef unsigned short uint16_t;
+typedef int int32_t;
+typedef unsigned int uint32_t;
+typedef long long int64_t;
+typedef unsigned long long uint64_t;
+#else
+#include <stdint.h>
+#endif
+
+#include <stddef.h>
+#include <stdlib.h>
+#include "msgpack/pack_define.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len);
+
+typedef struct msgpack_packer {
+	void* data;
+	msgpack_packer_write callback;
+} msgpack_packer;
+
+static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback);
+
+static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback);
+static inline void msgpack_packer_free(msgpack_packer* pk);
+
+static inline int msgpack_pack_short(msgpack_packer* pk, short d);
+static inline int msgpack_pack_int(msgpack_packer* pk, int d);
+static inline int msgpack_pack_long(msgpack_packer* pk, long d);
+static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d);
+static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d);
+static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d);
+static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d);
+static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d);
+
+static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d);
+static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d);
+static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d);
+static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d);
+static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d);
+static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d);
+static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d);
+static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d);
+
+static inline int msgpack_pack_float(msgpack_packer* pk, float d);
+static inline int msgpack_pack_double(msgpack_packer* pk, double d);
+
+static inline int msgpack_pack_nil(msgpack_packer* pk);
+static inline int msgpack_pack_true(msgpack_packer* pk);
+static inline int msgpack_pack_false(msgpack_packer* pk);
+
+static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n);
+
+static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
+
+static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
+static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
+
+
+
+#define msgpack_pack_inline_func(name) \
+	static inline int msgpack_pack ## name
+
+#define msgpack_pack_inline_func_cint(name) \
+	static inline int msgpack_pack ## name
+
+#define msgpack_pack_user msgpack_packer*
+
+#define msgpack_pack_append_buffer(user, buf, len) \
+	return (*(user)->callback)((user)->data, (const char*)buf, len)
+
+#include "msgpack/pack_template.h"
+
+static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback)
+{
+	pk->data = data;
+	pk->callback = callback;
+}
+
+static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback)
+{
+	msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer));
+	if(!pk) { return NULL; }
+	msgpack_packer_init(pk, data, callback);
+	return pk;
+}
+
+static inline void msgpack_packer_free(msgpack_packer* pk)
+{
+	free(pk);
+}
+
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h
new file mode 100644
index 0000000..694e816
--- /dev/null
+++ b/python/msgpack/unpack.h
@@ -0,0 +1,179 @@
+/*
+ * MessagePack for Python unpacking routine
+ *
+ * Copyright (C) 2009 Naoki INADA
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#include <map>
+#include <string>
+
+#define MSGPACK_MAX_STACK_SIZE  (1024)
+#include "msgpack/unpack_define.h"
+
+using namespace std;
+
+typedef  struct unpack_user {
+    struct array_stack_type{unsigned int size, last;};
+    array_stack_type array_stack[MSGPACK_MAX_STACK_SIZE];
+    int array_current;
+
+    map<string, PyObject*> str_cache;
+
+    ~unpack_user() {
+        map<string, PyObject*>::iterator it, itend;
+        itend = str_cache.end();
+        for (it = str_cache.begin(); it != itend; ++it) {
+            Py_DECREF(it->second);
+        }
+    }
+} unpack_user;
+
+
+#define msgpack_unpack_struct(name) \
+	struct template ## name
+
+#define msgpack_unpack_func(ret, name) \
+	static inline ret template ## name
+
+#define msgpack_unpack_callback(name) \
+	template_callback ## name
+
+#define msgpack_unpack_object PyObject*
+
+#define msgpack_unpack_user unpack_user
+
+
+struct template_context;
+typedef struct template_context template_context;
+
+static inline msgpack_unpack_object template_callback_root(unpack_user* u)
+{
+    u->array_current = -1;
+    return NULL;
+}
+
+static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o)
+{ *o = PyInt_FromLong((long)d); return 0; }
+
+static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o)
+{ *o = PyInt_FromLong((long)d); return 0; }
+
+static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o)
+{
+    if (d > LONG_MAX) {
+        *o = PyLong_FromUnsignedLong((unsigned long)d);
+    } else {
+        *o = PyInt_FromLong((long)d);
+    }
+    return 0;
+}
+
+static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o)
+{ *o = PyLong_FromUnsignedLongLong(d); return 0; }
+
+static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o)
+{ *o = PyInt_FromLong(d); return 0; }
+
+static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o)
+{ *o = PyInt_FromLong(d); return 0; }
+
+static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o)
+{ *o = PyInt_FromLong(d); return 0; }
+
+static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o)
+{ *o = PyLong_FromLongLong(d); return 0; }
+
+static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o)
+{ *o = PyFloat_FromDouble((double)d); return 0; }
+
+static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o)
+{ *o = PyFloat_FromDouble(d); return 0; }
+
+static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o)
+{ Py_INCREF(Py_None); *o = Py_None; return 0; }
+
+static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o)
+{ Py_INCREF(Py_True); *o = Py_True; return 0; }
+
+static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o)
+{ Py_INCREF(Py_False); *o = Py_False; return 0; }
+
+static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
+{
+    if (n > 0) {
+        int cur = ++u->array_current;
+        u->array_stack[cur].size = n;
+        u->array_stack[cur].last = 0;
+        *o = PyList_New(n);
+    }
+    else {
+        *o = PyList_New(0);
+    }
+    return 0;
+}
+
+static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o)
+{
+    int cur = u->array_current;
+    int n = u->array_stack[cur].size;
+    int last = u->array_stack[cur].last;
+
+    PyList_SetItem(*c, last, o);
+    last++;
+    if (last >= n) {
+        u->array_current--;
+    }
+    else {
+        u->array_stack[cur].last = last;
+    }
+    return 0;
+}
+
+static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
+{
+    *o = PyDict_New();
+    return 0;
+}
+
+static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
+{
+    PyDict_SetItem(*c, k, v);
+    Py_DECREF(k);
+    Py_DECREF(v);
+    return 0;
+}
+
+static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
+{
+    if (l < 16) {
+        string s(p, l);
+        map<string,PyObject*>::iterator it = u->str_cache.find(s);
+        if (it != u->str_cache.end()) {
+            *o = it->second;
+            Py_INCREF(*o);
+        }
+        else {
+            *o = PyString_FromStringAndSize(p, l);
+            Py_INCREF(*o);
+            u->str_cache[s] = *o;
+        }
+    }
+    else {
+        *o = PyString_FromStringAndSize(p, l);
+    }
+    return 0;
+}
+
+#include "msgpack/unpack_template.h"