Merge commit 'd5f9995' into read_bytes

Conflicts:
	msgpack/_msgpack.pyx

16 files changed, 437 insertions, 142 deletions

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 4fd5cbc..6a4d27b 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,24 @@
+0.3.0
+=====
+:release date: in development
+
+Changes
+-------
+* Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman)
+
+
+0.2.3
+=======
+:release date: in development
+
+Changes
+-------
+* Warn when use_list is not specified. It's default value will be changed in 0.3.
+
+Bugs fixed
+-----------
+* Can't pack subclass of dict.
+
 0.2.2
 =======
 :release date: 2012-09-21
diff --git a/ROADMAP.md b/ROADMAP.md
new file mode 100644
index 0000000..5245cc0
--- /dev/null
+++ b/ROADMAP.md
@@ -0,0 +1,7 @@
+0.2 series
+==========
+Improve compatibility to simplejson.
+
+0.3 series
+==========
+Add features msgpack-ruby has.
diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 30fb9fc..728e4f3 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -1,12 +1,16 @@
 # coding: utf-8
 #cython: embedsignature=True
 
+import warnings
+
 from cpython cimport *
 cdef extern from "Python.h":
     ctypedef char* const_char_ptr "const char*"
     ctypedef char* const_void_ptr "const void*"
     ctypedef struct PyObject
     cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
+    char* __FILE__
+    int __LINE__
 
 from libc.stdlib cimport *
 from libc.string cimport *
@@ -139,11 +143,19 @@ cdef class Packer(object):
             ret = msgpack_pack_raw(&self.pk, len(o))
             if ret == 0:
                 ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
-        elif PyDict_Check(o):
+        elif PyDict_CheckExact(o):
             d = <dict>o
             ret = msgpack_pack_map(&self.pk, len(d))
             if ret == 0:
-                for k,v in d.iteritems():
+                for k, v in d.iteritems():
+                    ret = self._pack(k, nest_limit-1)
+                    if ret != 0: break
+                    ret = self._pack(v, nest_limit-1)
+                    if ret != 0: break
+        elif PyDict_Check(o):
+            ret = msgpack_pack_map(&self.pk, len(o))
+            if ret == 0:
+                for k, v in o.items():
                     ret = self._pack(k, nest_limit-1)
                     if ret != 0: break
                     ret = self._pack(v, nest_limit-1)
@@ -170,6 +182,17 @@ cdef class Packer(object):
         self.pk.length = 0
         return buf
 
+    cpdef pack_array_header(self, size_t size):
+        msgpack_pack_array(&self.pk, size)
+        buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+        self.pk.length = 0
+        return buf
+
+    cpdef pack_map_header(self, size_t size):
+        msgpack_pack_map(&self.pk, size)
+        buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+        self.pk.length = 0
+        return buf
 
 def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'):
     """
@@ -187,8 +210,9 @@ def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use
 
 cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
-        int use_list
+        bint use_list
         PyObject* object_hook
+        bint has_pairs_hook # call object_hook with k-v pairs
         PyObject* list_hook
         char *encoding
         char *unicode_errors
@@ -200,71 +224,98 @@ cdef extern from "unpack.h":
         unsigned int ct
         PyObject* key
 
-    int template_execute(template_context* ctx, const_char_ptr data,
-                         size_t len, size_t* off) except -1
+    ctypedef int (*execute_fn)(template_context* ctx, const_char_ptr data,
+                               size_t len, size_t* off) except -1
+    execute_fn template_construct
+    execute_fn template_skip
+    execute_fn read_array_header
+    execute_fn read_map_header
     void template_init(template_context* ctx)
     object template_data(template_context* ctx)
 
+cdef inline init_ctx(template_context *ctx, object object_hook, object object_pairs_hook, object list_hook, bint use_list, encoding, unicode_errors):
+    template_init(ctx)
+    ctx.user.use_list = use_list
+    ctx.user.object_hook = ctx.user.list_hook = <PyObject*>NULL
+
+    if object_hook is not None and object_pairs_hook is not None:
+        raise ValueError("object_pairs_hook and object_hook are mutually exclusive.")
+
+    if object_hook is not None:
+        if not PyCallable_Check(object_hook):
+            raise TypeError("object_hook must be a callable.")
+        ctx.user.object_hook = <PyObject*>object_hook
+
+    if object_pairs_hook is None:
+        ctx.user.has_pairs_hook = False
+    else:
+        if not PyCallable_Check(object_pairs_hook):
+            raise TypeError("object_pairs_hook must be a callable.")
+        ctx.user.object_hook = <PyObject*>object_pairs_hook
+        ctx.user.has_pairs_hook = True
+
+    if list_hook is not None:
+        if not PyCallable_Check(list_hook):
+            raise TypeError("list_hook must be a callable.")
+        ctx.user.list_hook = <PyObject*>list_hook
+
+    if encoding is None:
+        ctx.user.encoding = NULL
+        ctx.user.unicode_errors = NULL
+    else:
+        if isinstance(encoding, unicode):
+            _bencoding = encoding.encode('ascii')
+        else:
+            _bencoding = encoding
+        ctx.user.encoding = PyBytes_AsString(_bencoding)
+        if isinstance(unicode_errors, unicode):
+            _berrors = unicode_errors.encode('ascii')
+        else:
+            _berrors = unicode_errors
+        ctx.user.unicode_errors = PyBytes_AsString(_berrors)
 
 def unpackb(object packed, object object_hook=None, object list_hook=None,
-            bint use_list=0, encoding=None, unicode_errors="strict",
+            bint use_list=1, encoding=None, unicode_errors="strict",
+            object_pairs_hook=None,
             ):
+    """Unpack packed_bytes to object. Returns an unpacked object.
+
+    Raises `ValueError` when `packed` contains extra bytes.
     """
-    Unpack packed_bytes to object. Returns an unpacked object."""
     cdef template_context ctx
     cdef size_t off = 0
     cdef int ret
 
     cdef char* buf
     cdef Py_ssize_t buf_len
-    PyObject_AsReadBuffer(packed, <const_void_ptr*>&buf, &buf_len)
 
-    if encoding is None:
-        enc = NULL
-        err = NULL
-    else:
-        if isinstance(encoding, unicode):
-            bencoding = encoding.encode('ascii')
-        else:
-            bencoding = encoding
-        if isinstance(unicode_errors, unicode):
-            berrors = unicode_errors.encode('ascii')
-        else:
-            berrors = unicode_errors
-        enc = PyBytes_AsString(bencoding)
-        err = PyBytes_AsString(berrors)
+    PyObject_AsReadBuffer(packed, <const_void_ptr*>&buf, &buf_len)
 
-    template_init(&ctx)
-    ctx.user.use_list = use_list
-    ctx.user.object_hook = ctx.user.list_hook = NULL
-    ctx.user.encoding = <const_char_ptr>enc
-    ctx.user.unicode_errors = <const_char_ptr>err
-    if object_hook is not None:
-        if not PyCallable_Check(object_hook):
-            raise TypeError("object_hook must be a callable.")
-        ctx.user.object_hook = <PyObject*>object_hook
-    if list_hook is not None:
-        if not PyCallable_Check(list_hook):
-            raise TypeError("list_hook must be a callable.")
-        ctx.user.list_hook = <PyObject*>list_hook
-    ret = template_execute(&ctx, buf, buf_len, &off)
+    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors)
+    ret = template_construct(&ctx, buf, buf_len, &off)
     if ret == 1:
-        return template_data(&ctx)
+        obj = template_data(&ctx)
+        if off < buf_len:
+            raise ValueError("Extra data.")
+        return obj
     else:
         return None
 
 
 def unpack(object stream, object object_hook=None, object list_hook=None,
-           bint use_list=0, encoding=None, unicode_errors="strict",
+           bint use_list=1, encoding=None, unicode_errors="strict",
+           object_pairs_hook=None,
            ):
-    """
-    unpack an object from stream.
+    """Unpack an object from `stream`.
+
+    Raises `ValueError` when `stream` has extra bytes.
     """
     return unpackb(stream.read(), use_list=use_list,
-                   object_hook=object_hook, list_hook=list_hook,
+                   object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
                    encoding=encoding, unicode_errors=unicode_errors,
                    )
 
+
 cdef class Unpacker(object):
     """
     Streaming unpacker.
@@ -277,10 +328,13 @@ cdef class Unpacker(object):
     (default: min(1024**2, max_buffer_size))
 
     If `use_list` is true, msgpack list is deserialized to Python list.
-    Otherwise, it is deserialized to Python tuple. (default: False)
+    Otherwise, it is deserialized to Python tuple.
 
     `object_hook` is same to simplejson. If it is not None, it should be callable
-    and Unpacker calls it when deserializing key-value.
+    and Unpacker calls it with a dict argument after deserializing a map.
+
+    `object_pairs_hook` is same to simplejson. If it is not None, it should be callable
+    and Unpacker calls it with a list of key-value pairs after deserializing a map.
 
     `encoding` is encoding used for decoding msgpack bytes. If it is None (default),
     msgpack bytes is deserialized to Python bytes.
@@ -315,7 +369,6 @@ cdef class Unpacker(object):
     cdef object file_like
     cdef object file_like_read
     cdef Py_ssize_t read_size
-    cdef bint use_list
     cdef object object_hook
     cdef object _bencoding
     cdef object _berrors
@@ -330,10 +383,10 @@ cdef class Unpacker(object):
         free(self.buf)
         self.buf = NULL
 
-    def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0,
-                 object object_hook=None, object list_hook=None,
-                 encoding=None, unicode_errors='strict', int max_buffer_size=0):
-        self.use_list = use_list
+    def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1,
+                 object object_hook=None, object object_pairs_hook=None, object list_hook=None,
+                 encoding=None, unicode_errors='strict', int max_buffer_size=0,
+                 ):
         self.file_like = file_like
         if file_like:
             self.file_like_read = file_like.read
@@ -353,31 +406,7 @@ cdef class Unpacker(object):
         self.buf_size = read_size
         self.buf_head = 0
         self.buf_tail = 0
-        template_init(&self.ctx)
-        self.ctx.user.use_list = use_list
-        self.ctx.user.object_hook = self.ctx.user.list_hook = <PyObject*>NULL
-        if object_hook is not None:
-            if not PyCallable_Check(object_hook):
-                raise TypeError("object_hook must be a callable.")
-            self.ctx.user.object_hook = <PyObject*>object_hook
-        if list_hook is not None:
-            if not PyCallable_Check(list_hook):
-                raise TypeError("list_hook must be a callable.")
-            self.ctx.user.list_hook = <PyObject*>list_hook
-        if encoding is None:
-            self.ctx.user.encoding = NULL
-            self.ctx.user.unicode_errors = NULL
-        else:
-            if isinstance(encoding, unicode):
-                self._bencoding = encoding.encode('ascii')
-            else:
-                self._bencoding = encoding
-            self.ctx.user.encoding = PyBytes_AsString(self._bencoding)
-            if isinstance(unicode_errors, unicode):
-                self._berrors = unicode_errors.encode('ascii')
-            else:
-                self._berrors = unicode_errors
-            self.ctx.user.unicode_errors = PyBytes_AsString(self._berrors)
+        init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors)
 
     def feed(self, object next_bytes):
         cdef char* buf
@@ -438,20 +467,20 @@ cdef class Unpacker(object):
         else:
             self.file_like = None
 
-    cpdef unpack(self):
-        """unpack one object"""
+    cdef object _unpack(self, execute_fn execute):
         cdef int ret
+        cdef object obj
         while 1:
-            ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
+            ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
             if ret == 1:
-                o = template_data(&self.ctx)
+                obj = template_data(&self.ctx)
                 template_init(&self.ctx)
-                return o
+                return obj
             elif ret == 0:
                 if self.file_like is not None:
                     self.read_from_file()
                     continue
-                raise StopIteration("No more unpack data.")
+                raise StopIteration("No more data to unpack.")
             else:
                 raise ValueError("Unpack failed: error = %d" % (ret,))
 
@@ -465,11 +494,27 @@ cdef class Unpacker(object):
             ret += self.file_like.read(nbytes - len(ret))
         return ret
 
+    def unpack(self):
+        """unpack one object"""
+        return self._unpack(template_construct)
+
+    def skip(self):
+        """read and ignore one object, returning None"""
+        return self._unpack(template_skip)
+
+    def read_array_header(self):
+        """assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents."""
+        return self._unpack(read_array_header)
+
+    def read_map_header(self):
+        """assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs."""
+        return self._unpack(read_map_header)
+
     def __iter__(self):
         return self
 
     def __next__(self):
-        return self.unpack()
+        return self._unpack(template_construct)
 
     # for debug.
     #def _buf(self):
diff --git a/msgpack/_version.py b/msgpack/_version.py
index f343b7a..e750485 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 2, 2)
+version = (0, 3, 0, 'dev1')
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index a106f9c..5ec7dbc 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -22,6 +22,7 @@
 typedef struct unpack_user {
     int use_list;
     PyObject *object_hook;
+    bool has_pairs_hook;
     PyObject *list_hook;
     const char *encoding;
     const char *unicode_errors;
@@ -41,6 +42,7 @@ typedef struct unpack_user {
 
 #define msgpack_unpack_user unpack_user
 
+typedef int (*execute_fn)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off);
 
 struct template_context;
 typedef struct template_context template_context;
@@ -160,9 +162,7 @@ static inline int template_callback_array_item(unpack_user* u, unsigned int curr
 static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_object* c)
 {
     if (u->list_hook) {
-        PyObject *arglist = Py_BuildValue("(O)", *c);
-        PyObject *new_c = PyEval_CallObject(u->list_hook, arglist);
-        Py_DECREF(arglist);
+        PyObject *new_c = PyEval_CallFunction(u->list_hook, "(O)", *c);
         Py_DECREF(*c);
         *c = new_c;
     }
@@ -171,16 +171,31 @@ static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_obj
 
 static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
 {
-    PyObject *p = PyDict_New();
+    PyObject *p;
+    if (u->has_pairs_hook) {
+        p = PyList_New(n); // Or use tuple?
+    }
+    else {
+        p = PyDict_New();
+    }
     if (!p)
         return -1;
     *o = p;
     return 0;
 }
 
-static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
+static inline int template_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
 {
-    if (PyDict_SetItem(*c, k, v) == 0) {
+    if (u->has_pairs_hook) {
+        msgpack_unpack_object item = PyTuple_Pack(2, k, v);
+        if (!item)
+            return -1;
+        Py_DECREF(k);
+        Py_DECREF(v);
+        PyList_SET_ITEM(*c, current, item);
+        return 0;
+    }
+    else if (PyDict_SetItem(*c, k, v) == 0) {
         Py_DECREF(k);
         Py_DECREF(v);
         return 0;
@@ -191,9 +206,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje
 static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
 {
     if (u->object_hook) {
-        PyObject *arglist = Py_BuildValue("(O)", *c);
-        PyObject *new_c = PyEval_CallObject(u->object_hook, arglist);
-        Py_DECREF(arglist);
+        PyObject *new_c = PyEval_CallFunction(u->object_hook, "(O)", *c);
         Py_DECREF(*c);
         *c = new_c;
     }
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index b844a24..7d07601 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -95,6 +95,7 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context
 }
 
 
+template <bool construct>
 msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
 {
 	assert(len >= *off);
@@ -117,14 +118,17 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 
 	int ret;
 
+#define construct_cb(name) \
+    construct && msgpack_unpack_callback(name)
+
 #define push_simple_value(func) \
-	if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \
+	if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \
 	goto _push
 #define push_fixed_value(func, arg) \
-	if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \
+	if(construct_cb(func)(user, arg, &obj) < 0) { goto _failed; } \
 	goto _push
 #define push_variable_value(func, base, pos, len) \
-	if(msgpack_unpack_callback(func)(user, \
+	if(construct_cb(func)(user, \
 		(const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
 	goto _push
 
@@ -140,9 +144,9 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 
 #define start_container(func, count_, ct_) \
 	if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
-	if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
+	if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
 	if((count_) == 0) { obj = stack[top].obj; \
-		msgpack_unpack_callback(func##_end)(user, &obj); \
+		construct_cb(func##_end)(user, &obj); \
 		goto _push; } \
 	stack[top].ct = ct_; \
 	stack[top].size  = count_; \
@@ -340,10 +344,10 @@ _push:
 	c = &stack[top-1];
 	switch(c->ct) {
 	case CT_ARRAY_ITEM:
-		if(msgpack_unpack_callback(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
+		if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
 		if(++c->count == c->size) {
 			obj = c->obj;
-			msgpack_unpack_callback(_array_end)(user, &obj);
+			construct_cb(_array_end)(user, &obj);
 			--top;
 			/*printf("stack pop %d\n", top);*/
 			goto _push;
@@ -354,10 +358,10 @@ _push:
 		c->ct = CT_MAP_VALUE;
 		goto _header_again;
 	case CT_MAP_VALUE:
-		if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
+		if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; }
 		if(++c->count == c->size) {
 			obj = c->obj;
-			msgpack_unpack_callback(_map_end)(user, &obj);
+			construct_cb(_map_end)(user, &obj);
 			--top;
 			/*printf("stack pop %d\n", top);*/
 			goto _push;
@@ -377,6 +381,8 @@ _header_again:
 
 
 _finish:
+	if (!construct)
+		msgpack_unpack_callback(_nil)(user, &obj);
 	stack[0].obj = obj;
 	++p;
 	ret = 1;
@@ -399,15 +405,13 @@ _end:
 	*off = p - (const unsigned char*)data;
 
 	return ret;
+#undef construct_cb
 }
 
-
-#undef msgpack_unpack_func
-#undef msgpack_unpack_callback
-#undef msgpack_unpack_struct
-#undef msgpack_unpack_object
-#undef msgpack_unpack_user
-
+#undef SWITCH_RANGE_BEGIN
+#undef SWITCH_RANGE
+#undef SWITCH_RANGE_DEFAULT
+#undef SWITCH_RANGE_END
 #undef push_simple_value
 #undef push_fixed_value
 #undef push_variable_value
@@ -415,6 +419,74 @@ _end:
 #undef again_fixed_trail_if_zero
 #undef start_container
 
+template <unsigned int fixed_offset, unsigned int var_offset>
+msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
+{
+	assert(len >= *off);
+	uint32_t size;
+	const unsigned char *const p = (unsigned char*)data + *off;
+
+#define inc_offset(inc) \
+	if (len - *off < inc) \
+		return 0; \
+	*off += inc;
+
+	switch (*p) {
+	case var_offset:
+		inc_offset(3);
+		size = _msgpack_load16(uint16_t, p + 1);
+		break;
+	case var_offset + 1:
+		inc_offset(5);
+		size = _msgpack_load32(uint32_t, p + 1);
+		break;
+#ifdef USE_CASE_RANGE
+	case fixed_offset + 0x0 ... fixed_offset + 0xf:
+#else
+	case fixed_offset + 0x0:
+	case fixed_offset + 0x1:
+	case fixed_offset + 0x2:
+	case fixed_offset + 0x3:
+	case fixed_offset + 0x4:
+	case fixed_offset + 0x5:
+	case fixed_offset + 0x6:
+	case fixed_offset + 0x7:
+	case fixed_offset + 0x8:
+	case fixed_offset + 0x9:
+	case fixed_offset + 0xa:
+	case fixed_offset + 0xb:
+	case fixed_offset + 0xc:
+	case fixed_offset + 0xd:
+	case fixed_offset + 0xe:
+	case fixed_offset + 0xf:
+#endif
+		++*off;
+		size = ((unsigned int)*p) & 0x0f;
+		break;
+	default:
+		PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
+		return -1;
+    }
+	msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj);
+	return 1;
+}
+
+#undef SWITCH_RANGE_BEGIN
+#undef SWITCH_RANGE
+#undef SWITCH_RANGE_DEFAULT
+#undef SWITCH_RANGE_END
+
+static const execute_fn template_construct = &template_execute<true>;
+static const execute_fn template_skip = &template_execute<false>;
+static const execute_fn read_array_header = &template_container_header<0x90, 0xdc>;
+static const execute_fn read_map_header = &template_container_header<0x80, 0xde>;
+
+#undef msgpack_unpack_func
+#undef msgpack_unpack_callback
+#undef msgpack_unpack_struct
+#undef msgpack_unpack_object
+#undef msgpack_unpack_user
+
 #undef NEXT_CS
 
 /* vim: set ts=4 sw=4 noexpandtab  */
diff --git a/setup.py b/setup.py
index 86b0b34..5be92bf 100644
--- a/setup.py
+++ b/setup.py
@@ -18,7 +18,7 @@ except ImportError:
 
 def cythonize(src):
     sys.stderr.write("cythonize: %r\n" % (src,))
-    cython_compiler.compile([src])
+    cython_compiler.compile([src], cplus=True, emit_linenums=True)
 
 def ensure_source(src):
     pyx = os.path.splitext(src)[0] + '.pyx'
@@ -34,13 +34,6 @@ Install Cython >= 0.16 or install msgpack from PyPI.
           os.stat(src).st_mtime < os.stat(pyx).st_mtime and
           have_cython):
         cythonize(pyx)
-
-    # Use C++ compiler on win32.
-    # MSVC9 doesn't provide stdint.h when using C Compiler.
-    if sys.platform == 'win32':
-        cpp = src + 'pp'
-        shutil.copy(src, cpp)
-        return cpp
     else:
         return src
 
@@ -67,7 +60,7 @@ if have_cython:
 else:
     Sdist = sdist
 
-sources = ['msgpack/_msgpack.c']
+sources = ['msgpack/_msgpack.cpp']
 libraries = []
 if sys.platform == 'win32':
     libraries.append('ws2_32')
diff --git a/test/test_buffer.py b/test/test_buffer.py
index 01310a0..785fb60 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -9,8 +9,8 @@ def test_unpack_buffer():
     from array import array
     buf = array('b')
     buf.fromstring(packb(('foo', 'bar')))
-    obj = unpackb(buf)
-    assert_equal((b'foo', b'bar'), obj)
+    obj = unpackb(buf, use_list=1)
+    assert_equal([b'foo', b'bar'], obj)
 
 if __name__ == '__main__':
     main()
diff --git a/test/test_case.py b/test/test_case.py
index b88714d..9cbf9bd 100644
--- a/test/test_case.py
+++ b/test/test_case.py
@@ -9,7 +9,7 @@ from msgpack import packb, unpackb
 def check(length, obj):
     v = packb(obj)
     assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v)))
-    assert_equal(unpackb(v), obj)
+    assert_equal(unpackb(v, use_list=0), obj)
 
 def test_1():
     for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1,
@@ -71,7 +71,7 @@ def test_array32():
 
 def match(obj, buf):
     assert_equal(packb(obj), buf)
-    assert_equal(unpackb(buf), obj)
+    assert_equal(unpackb(buf, use_list=0), obj)
 
 def test_match():
     cases = [
@@ -99,7 +99,7 @@ def test_match():
         match(v, p)
 
 def test_unicode():
-    assert_equal(b'foobar', unpackb(packb('foobar')))
+    assert_equal(b'foobar', unpackb(packb('foobar'), use_list=1))
 
 if __name__ == '__main__':
     main()
diff --git a/test/test_format.py b/test/test_format.py
index c03b3e2..ac08709 100644
--- a/test/test_format.py
+++ b/test/test_format.py
@@ -5,8 +5,8 @@ from nose import main
 from nose.tools import *
 from msgpack import unpackb
 
-def check(src, should):
-    assert_equal(unpackb(src), should)
+def check(src, should, use_list=0):
+    assert_equal(unpackb(src, use_list=use_list), should)
 
 def testSimpleValue():
     check(b"\x93\xc0\xc2\xc3", 
diff --git a/test/test_obj.py b/test/test_obj.py
index d155b73..881e627 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -18,25 +18,35 @@ def _encode_complex(obj):
 
 def test_encode_hook():
     packed = packb([3, 1+2j], default=_encode_complex)
-    unpacked = unpackb(packed)
+    unpacked = unpackb(packed, use_list=1)
     eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2})
 
 def test_decode_hook():
     packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
-    unpacked = unpackb(packed, object_hook=_decode_complex)
+    unpacked = unpackb(packed, object_hook=_decode_complex, use_list=1)
     eq_(unpacked[1], 1+2j)
 
+def test_decode_pairs_hook():
+    packed = packb([3, {1: 2, 3: 4}])
+    prod_sum = 1 * 2 + 3 * 4
+    unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1)
+    eq_(unpacked[1], prod_sum)
+
+@raises(ValueError)
+def test_only_one_obj_hook():
+    unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x)
+
 @raises(ValueError)
 def test_bad_hook():
     packed = packb([3, 1+2j], default=lambda o: o)
-    unpacked = unpackb(packed)
+    unpacked = unpackb(packed, use_list=1)
 
 def _arr_to_str(arr):
     return ''.join(str(c) for c in arr)
 
 def test_array_hook():
     packed = packb([1,2,3])
-    unpacked = unpackb(packed, list_hook=_arr_to_str)
+    unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1)
     eq_(unpacked, '123')
 
 if __name__ == '__main__':
diff --git a/test/test_pack.py b/test/test_pack.py
index 85d11a0..21c2bd7 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -11,8 +11,8 @@ from msgpack import packb, unpackb, Unpacker, Packer
 
 from io import BytesIO
 
-def check(data):
-    re = unpackb(packb(data))
+def check(data, use_list=False):
+    re = unpackb(packb(data), use_list=use_list)
     assert_equal(re, data)
 
 def testPack():
@@ -31,14 +31,14 @@ def testPack():
 
 def testPackUnicode():
     test_data = [
-        six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("Русский текст"),
+        six.u(""), six.u("abcd"), [six.u("defgh")], six.u("Русский текст"),
         ]
     for td in test_data:
-        re = unpackb(packb(td, encoding='utf-8'), encoding='utf-8')
+        re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8')
         assert_equal(re, td)
         packer = Packer(encoding='utf-8')
         data = packer.pack(td)
-        re = Unpacker(BytesIO(data), encoding='utf-8').unpack()
+        re = Unpacker(BytesIO(data), encoding='utf-8', use_list=1).unpack()
         assert_equal(re, td)
 
 def testPackUTF32():
@@ -46,11 +46,11 @@ def testPackUTF32():
         test_data = [
             six.u(""),
             six.u("abcd"),
-            (six.u("defgh"),),
+            [six.u("defgh")],
             six.u("Русский текст"),
             ]
         for td in test_data:
-            re = unpackb(packb(td, encoding='utf-32'), encoding='utf-32')
+            re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
             assert_equal(re, td)
     except LookupError:
         raise SkipTest
@@ -63,20 +63,19 @@ def testPackBytes():
         check(td)
 
 def testIgnoreUnicodeErrors():
-    re = unpackb(packb(b'abc\xeddef'),
-        encoding='utf-8', unicode_errors='ignore')
+    re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1)
     assert_equal(re, "abcdef")
 
 @raises(UnicodeDecodeError)
 def testStrictUnicodeUnpack():
-    unpackb(packb(b'abc\xeddef'), encoding='utf-8')
+    unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1)
 
 @raises(UnicodeEncodeError)
 def testStrictUnicodePack():
     packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict')
 
 def testIgnoreErrorsPack():
-    re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8')
+    re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1)
     assert_equal(re, six.u("abcdef"))
 
 @raises(TypeError)
@@ -84,12 +83,66 @@ def testNoEncoding():
     packb(six.u("abc"), encoding=None)
 
 def testDecodeBinary():
-    re = unpackb(packb("abc"), encoding=None)
+    re = unpackb(packb("abc"), encoding=None, use_list=1)
     assert_equal(re, b"abc")
 
 def testPackFloat():
     assert_equal(packb(1.0, use_single_float=True),  b'\xca' + struct.pack('>f', 1.0))
     assert_equal(packb(1.0, use_single_float=False), b'\xcb' + struct.pack('>d', 1.0))
 
+def testArraySize(sizes=[0, 5, 50, 1000]):
+    bio = six.BytesIO()
+    packer = Packer()
+    for size in sizes:
+        bio.write(packer.pack_array_header(size))
+        for i in range(size):
+            bio.write(packer.pack(i))
+
+    bio.seek(0)
+    unpacker = Unpacker(bio, use_list=1)
+    for size in sizes:
+        assert unpacker.unpack() == list(range(size))
+
+def testMapSize(sizes=[0, 5, 50, 1000]):
+    bio = six.BytesIO()
+    packer = Packer()
+    for size in sizes:
+        bio.write(packer.pack_map_header(size))
+        for i in range(size):
+            bio.write(packer.pack(i)) # key
+            bio.write(packer.pack(i * 2)) # value
+
+    bio.seek(0)
+    unpacker = Unpacker(bio)
+    for size in sizes:
+        assert unpacker.unpack() == dict((i, i * 2) for i in range(size))
+
+
+
+
+class odict(dict):
+    '''Reimplement OrderedDict to run test on Python 2.6'''
+    def __init__(self, seq):
+        self._seq = seq
+        dict.__init__(self, seq)
+
+    def items(self):
+        return self._seq[:]
+
+    def iteritems(self):
+        return iter(self._seq)
+
+    def keys(self):
+        return [x[0] for x in self._seq]
+
+def test_odict():
+    seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
+    od = odict(seq)
+    assert_equal(unpackb(packb(od), use_list=1), dict(seq))
+    def pair_hook(seq):
+        return seq
+    assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1), seq)
+
+
 if __name__ == '__main__':
     main()
diff --git a/test/test_read_size.py b/test/test_read_size.py
new file mode 100644
index 0000000..e130805
--- /dev/null
+++ b/test/test_read_size.py
@@ -0,0 +1,66 @@
+"""Test Unpacker's read_array_header and read_map_header methods"""
+from msgpack import packb, Unpacker
+UnexpectedTypeException = ValueError
+
+def test_read_array_header():
+    unpacker = Unpacker()
+    unpacker.feed(packb(['a', 'b', 'c']))
+    assert unpacker.read_array_header() == 3
+    assert unpacker.unpack() == b'a'
+    assert unpacker.unpack() == b'b'
+    assert unpacker.unpack() == b'c'
+    try:
+        unpacker.unpack()
+        assert 0, 'should raise exception'
+    except StopIteration:
+        assert 1, 'okay'
+
+
+def test_read_map_header():
+    unpacker = Unpacker()
+    unpacker.feed(packb({'a': 'A'}))
+    assert unpacker.read_map_header() == 1
+    assert unpacker.unpack() == B'a'
+    assert unpacker.unpack() == B'A'
+    try:
+        unpacker.unpack()
+        assert 0, 'should raise exception'
+    except StopIteration:
+        assert 1, 'okay'
+
+def test_incorrect_type_array():
+    unpacker = Unpacker()
+    unpacker.feed(packb(1))
+    try:
+        unpacker.read_array_header()
+        assert 0, 'should raise exception'
+    except UnexpectedTypeException:
+        assert 1, 'okay'
+
+def test_incorrect_type_map():
+    unpacker = Unpacker()
+    unpacker.feed(packb(1))
+    try:
+        unpacker.read_map_header()
+        assert 0, 'should raise exception'
+    except UnexpectedTypeException:
+        assert 1, 'okay'
+
+def test_correct_type_nested_array():
+    unpacker = Unpacker()
+    unpacker.feed(packb({'a': ['b', 'c', 'd']}))
+    try:
+        unpacker.read_array_header()
+        assert 0, 'should raise exception'
+    except UnexpectedTypeException:
+        assert 1, 'okay'
+
+def test_incorrect_type_nested_map():
+    unpacker = Unpacker()
+    unpacker.feed(packb([{'a': 'b'}]))
+    try:
+        unpacker.read_map_header()
+        assert 0, 'should raise exception'
+    except UnexpectedTypeException:
+        assert 1, 'okay'
+
diff --git a/test/test_seq.py b/test/test_seq.py
index d0f9ccc..72e935a 100644
--- a/test/test_seq.py
+++ b/test/test_seq.py
@@ -34,7 +34,7 @@ def test_exceeding_unpacker_read_size():
     f = io.BytesIO(dumpf.getvalue())
     dumpf.close()
 
-    unpacker = msgpack.Unpacker(f, read_size=read_size)
+    unpacker = msgpack.Unpacker(f, read_size=read_size, use_list=1)
 
     read_count = 0
     for idx, o in enumerate(unpacker):
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index c763f40..769f3ff 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -6,7 +6,7 @@ from msgpack import Unpacker, BufferFull
 import nose
 
 def test_foobar():
-    unpacker = Unpacker(read_size=3)
+    unpacker = Unpacker(read_size=3, use_list=1)
     unpacker.feed(b'foobar')
     assert unpacker.unpack() == ord(b'f')
     assert unpacker.unpack() == ord(b'o')
@@ -29,10 +29,24 @@ def test_foobar():
         k += 1
     assert k == len(b'foobar')
 
+def test_foobar_skip():
+    unpacker = Unpacker(read_size=3, use_list=1)
+    unpacker.feed(b'foobar')
+    assert unpacker.unpack() == ord(b'f')
+    unpacker.skip()
+    assert unpacker.unpack() == ord(b'o')
+    unpacker.skip()
+    assert unpacker.unpack() == ord(b'a')
+    unpacker.skip()
+    try:
+        o = unpacker.unpack()
+        assert 0, "should raise exception"
+    except StopIteration:
+        assert 1, "ok"
 
 def test_maxbuffersize():
     nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3)
-    unpacker = Unpacker(read_size=3, max_buffer_size=3)
+    unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1)
     unpacker.feed(b'fo')
     nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob')
     unpacker.feed(b'o')
diff --git a/tox.ini b/tox.ini
index 5e80dd0..214c4c4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,6 @@
 [tox]
-envlist = py26,py27,py32
+envlist = py26,py27,py32,py33
+
 [testenv]
 deps=
     nose