Merge pull request #77 from msgpack/newspec

[WIP] Support new spec.

10 files changed, 268 insertions, 93 deletions

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 77f6b81..79107b6 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -2,15 +2,37 @@
 from msgpack._version import version
 from msgpack.exceptions import *
 
+from collections import namedtuple
+
+ExtType = namedtuple('ExtType', 'code data')
+
 import os
 if os.environ.get('MSGPACK_PUREPYTHON'):
-    from msgpack.fallback import pack, packb, Packer, unpack, unpackb, Unpacker
+    from msgpack.fallback import Packer, unpack, unpackb, Unpacker
 else:
     try:
-        from msgpack._packer import pack, packb, Packer
+        from msgpack._packer import Packer
         from msgpack._unpacker import unpack, unpackb, Unpacker
     except ImportError:
-        from msgpack.fallback import pack, packb, Packer, unpack, unpackb, Unpacker
+        from msgpack.fallback import Packer, unpack, unpackb, Unpacker
+
+
+def pack(o, stream, **kwargs):
+    """
+    Pack object `o` and write it to `stream`
+
+    See :class:`Packer` for options.
+    """
+    packer = Packer(**kwargs)
+    stream.write(packer.pack(o))
+
+def packb(o, **kwargs):
+    """
+    Pack object `o` and return packed bytes
+
+    See :class:`Packer` for options.
+    """
+    return Packer(**kwargs).pack(o)
 
 # alias for compatibility to simplejson/marshal/pickle.
 load = unpack
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 6289192..7082445 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -13,6 +13,7 @@ cdef extern from "pack.h":
         char* buf
         size_t length
         size_t buf_size
+        bint use_bin_type
 
     int msgpack_pack_int(msgpack_packer* pk, int d)
     int msgpack_pack_nil(msgpack_packer* pk)
@@ -26,6 +27,7 @@ cdef extern from "pack.h":
     int msgpack_pack_array(msgpack_packer* pk, size_t l)
     int msgpack_pack_map(msgpack_packer* pk, size_t l)
     int msgpack_pack_raw(msgpack_packer* pk, size_t l)
+    int msgpack_pack_bin(msgpack_packer* pk, size_t l)
     int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
 
 cdef int DEFAULT_RECURSE_LIMIT=511
@@ -56,6 +58,9 @@ cdef class Packer(object):
     :param bool autoreset:
         Reset buffer after each pack and return it's content as `bytes`. (default: True).
         If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
+    :param bool use_bin_type:
+        Use bin type introduced in msgpack spec 2.0 for bytes.
+        It also enable str8 type for unicode.
     """
     cdef msgpack_packer pk
     cdef object _default
@@ -74,11 +79,13 @@ cdef class Packer(object):
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
-    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, bint autoreset=1):
+    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
+                 use_single_float=False, bint autoreset=1, bint use_bin_type=0):
         """
         """
         self.use_float = use_single_float
         self.autoreset = autoreset
+        self.pk.use_bin_type = use_bin_type
         if default is not None:
             if not PyCallable_Check(default):
                 raise TypeError("default must be a callable.")
@@ -110,6 +117,7 @@ cdef class Packer(object):
         cdef char* rawval
         cdef int ret
         cdef dict d
+        cdef size_t L
 
         if nest_limit < 0:
             raise PackValueError("recursion limit exceeded.")
@@ -140,9 +148,10 @@ cdef class Packer(object):
                ret = msgpack_pack_double(&self.pk, dval)
         elif PyBytes_Check(o):
             rawval = o
-            ret = msgpack_pack_raw(&self.pk, len(o))
+            L = len(o)
+            ret = msgpack_pack_bin(&self.pk, L)
             if ret == 0:
-                ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
+                ret = msgpack_pack_raw_body(&self.pk, rawval, L)
         elif PyUnicode_Check(o):
             if not self.encoding:
                 raise TypeError("Can't encode unicode string: no encoding is specified")
@@ -245,23 +254,3 @@ cdef class Packer(object):
     def bytes(self):
         """Return buffer content."""
         return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
-
-
-def pack(object o, object stream, default=None, str encoding='utf-8', str unicode_errors='strict'):
-    """
-    pack an object `o` and write it to stream)
-
-    See :class:`Packer` for options.
-    """
-    packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
-    stream.write(packer.pack(o))
-
-def packb(object o, default=None, encoding='utf-8', str unicode_errors='strict', bint use_single_float=False):
-    """
-    pack o and return packed bytes
-
-    See :class:`Packer` for options.
-    """
-    packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
-                    use_single_float=use_single_float)
-    return packer.pack(o)
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 1f4dd85..7b0c8a6 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -18,7 +18,6 @@ from msgpack.exceptions import (
         )
 
 
-
 cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
         bint use_list
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 8f9d646..dfaaa54 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -48,6 +48,9 @@ from msgpack.exceptions import (
         PackValueError,
         ExtraData)
 
+from msgpack import ExtType
+
+
 EX_SKIP                 = 0
 EX_CONSTRUCT            = 1
 EX_READ_ARRAY_HEADER    = 2
@@ -57,25 +60,10 @@ TYPE_IMMEDIATE          = 0
 TYPE_ARRAY              = 1
 TYPE_MAP                = 2
 TYPE_RAW                = 3
+TYPE_BIN                = 4
+TYPE_EXT                = 5
 
-DEFAULT_RECURSE_LIMIT=511
-
-def pack(o, stream, **kwargs):
-    """
-    Pack object `o` and write it to `stream`
-
-    See :class:`Packer` for options.
-    """
-    packer = Packer(**kwargs)
-    stream.write(packer.pack(o))
-
-def packb(o, **kwargs):
-    """
-    Pack object `o` and return packed bytes
-
-    See :class:`Packer` for options.
-    """
-    return Packer(**kwargs).pack(o)
+DEFAULT_RECURSE_LIMIT = 511
 
 def unpack(stream, **kwargs):
     """
@@ -128,6 +116,9 @@ class Unpacker(object):
     should be callable and Unpacker calls it with a list of key-value pairs
     after deserializing a map.
 
+    `ext_hook` is callback for ext (User defined) type. It called with two
+    arguments: (code, bytes). default: `msgpack.ExtType`
+
     `encoding` is the encoding used for decoding msgpack bytes.  If it is
     None (default), msgpack bytes are deserialized to Python bytes.
 
@@ -159,7 +150,8 @@ class Unpacker(object):
 
     def __init__(self, file_like=None, read_size=0, use_list=True,
                  object_hook=None, object_pairs_hook=None, list_hook=None,
-                 encoding=None, unicode_errors='strict', max_buffer_size=0):
+                 encoding=None, unicode_errors='strict', max_buffer_size=0,
+                 ext_hook=ExtType):
         if file_like is None:
             self._fb_feeding = True
         else:
@@ -171,18 +163,17 @@ class Unpacker(object):
         self._fb_buf_o = 0
         self._fb_buf_i = 0
         self._fb_buf_n = 0
-        self._max_buffer_size = (2**31-1 if max_buffer_size == 0
-                                         else max_buffer_size)
-        self._read_size = (read_size if read_size != 0
-                            else min(self._max_buffer_size, 2048))
+        self._max_buffer_size = max_buffer_size or 2**31-1
         if read_size > self._max_buffer_size:
             raise ValueError("read_size must be smaller than max_buffer_size")
+        self._read_size = read_size or min(self._max_buffer_size, 2048)
         self._encoding = encoding
         self._unicode_errors = unicode_errors
         self._use_list = use_list
         self._list_hook = list_hook
         self._object_hook = object_hook
         self._object_pairs_hook = object_pairs_hook
+        self._ext_hook = ext_hook
 
         if list_hook is not None and not callable(list_hook):
             raise ValueError('`list_hook` is not callable')
@@ -193,6 +184,8 @@ class Unpacker(object):
         if object_hook is not None and object_pairs_hook is not None:
             raise ValueError("object_pairs_hook and object_hook are mutually "
                              "exclusive")
+        if not callable(ext_hook):
+            raise ValueError("`ext_hook` is not callable")
 
     def feed(self, next_bytes):
         if isinstance(next_bytes, array.array):
@@ -239,26 +232,26 @@ class Unpacker(object):
         return b''.join(bufs)
 
     def _fb_read(self, n, write_bytes=None):
-        if (write_bytes is None and self._fb_buf_i < len(self._fb_buffers)
-                and self._fb_buf_o + n < len(self._fb_buffers[self._fb_buf_i])):
+        buffs = self._fb_buffers
+        if (write_bytes is None and self._fb_buf_i < len(buffs) and
+                self._fb_buf_o + n < len(buffs[self._fb_buf_i])):
             self._fb_buf_o += n
-            return self._fb_buffers[self._fb_buf_i][
-                    self._fb_buf_o-n:self._fb_buf_o]
+            return buffs[self._fb_buf_i][self._fb_buf_o - n:self._fb_buf_o]
+
         ret = b''
         while len(ret) != n:
-            if self._fb_buf_i == len(self._fb_buffers):
+            if self._fb_buf_i == len(buffs):
                 if self._fb_feeding:
                     break
                 tmp = self.file_like.read(self._read_size)
                 if not tmp:
                     break
-                self._fb_buffers.append(tmp)
+                buffs.append(tmp)
                 continue
             sliced = n - len(ret)
-            ret += self._fb_buffers[self._fb_buf_i][
-                        self._fb_buf_o:self._fb_buf_o + sliced]
+            ret += buffs[self._fb_buf_i][self._fb_buf_o:self._fb_buf_o + sliced]
             self._fb_buf_o += sliced
-            if self._fb_buf_o >= len(self._fb_buffers[self._fb_buf_i]):
+            if self._fb_buf_o >= len(buffs[self._fb_buf_i]):
                 self._fb_buf_o = 0
                 self._fb_buf_i += 1
         if len(ret) != n:
@@ -294,6 +287,30 @@ class Unpacker(object):
             obj = False
         elif b == 0xc3:
             obj = True
+        elif b == 0xc4:
+            typ = TYPE_BIN
+            n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+        elif b == 0xc5:
+            typ = TYPE_BIN
+            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+        elif b == 0xc6:
+            typ = TYPE_BIN
+            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+        elif b == 0xc7:  # ext 8
+            typ = TYPE_EXT
+            L, n = struct.unpack('Bb', self._fb_read(2, write_bytes))
+            obj = self._fb_read(L, write_bytes)
+        elif b == 0xc8:  # ext 16
+            typ = TYPE_EXT
+            L, n = struct.unpack('>Hb', self._fb_read(3, write_bytes))
+            obj = self._fb_read(L, write_bytes)
+        elif b == 0xc9:  # ext 32
+            typ = TYPE_EXT
+            L, n = struct.unpack('>Ib', self._fb_read(5, write_bytes))
+            obj = self._fb_read(L, write_bytes)
         elif b == 0xca:
             obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0]
         elif b == 0xcb:
@@ -314,14 +331,33 @@ class Unpacker(object):
             obj = struct.unpack(">i", self._fb_read(4, write_bytes))[0]
         elif b == 0xd3:
             obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0]
+        elif b == 0xd4:  # fixext 1
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b1s', self._fb_read(2, write_bytes))
+        elif b == 0xd5:  # fixext 2
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b2s', self._fb_read(3, write_bytes))
+        elif b == 0xd6:  # fixext 4
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b4s', self._fb_read(5, write_bytes))
+        elif b == 0xd7:  # fixext 8
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b8s', self._fb_read(9, write_bytes))
+        elif b == 0xd8:  # fixext 16
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b16s', self._fb_read(17, write_bytes))
+        elif b == 0xd9:
+            typ = TYPE_RAW
+            n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
         elif b == 0xda:
+            typ = TYPE_RAW
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
             obj = self._fb_read(n, write_bytes)
-            typ = TYPE_RAW
         elif b == 0xdb:
+            typ = TYPE_RAW
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
             obj = self._fb_read(n, write_bytes)
-            typ = TYPE_RAW
         elif b == 0xdc:
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
             typ = TYPE_ARRAY
@@ -372,10 +408,9 @@ class Unpacker(object):
                 return
             if self._object_pairs_hook is not None:
                 ret = self._object_pairs_hook(
-                        (self._fb_unpack(EX_CONSTRUCT, write_bytes),
-                         self._fb_unpack(EX_CONSTRUCT, write_bytes))
-                            for _ in xrange(n)
-                        )
+                    (self._fb_unpack(EX_CONSTRUCT, write_bytes),
+                     self._fb_unpack(EX_CONSTRUCT, write_bytes))
+                    for _ in xrange(n))
             else:
                 ret = {}
                 for _ in xrange(n):
@@ -390,6 +425,10 @@ class Unpacker(object):
             if self._encoding is not None:
                 obj = obj.decode(self._encoding, self._unicode_errors)
             return obj
+        if typ == TYPE_EXT:
+            return self._ext_hook(n, obj)
+        if typ == TYPE_BIN:
+            return obj
         assert typ == TYPE_IMMEDIATE
         return obj
 
@@ -446,11 +485,15 @@ class Packer(object):
     :param bool autoreset:
         Reset buffer after each pack and return it's content as `bytes`. (default: True).
         If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
+    :param bool use_bin_type:
+        Use bin type introduced in msgpack spec 2.0 for bytes.
+        It also enable str8 type for unicode.
     """
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
-                 use_single_float=False, autoreset=True):
+                 use_single_float=False, autoreset=True, use_bin_type=False):
         self._use_float = use_single_float
         self._autoreset = autoreset
+        self._use_bin_type = use_bin_type
         self._encoding = encoding
         self._unicode_errors = unicode_errors
         self._buffer = StringIO()
@@ -490,6 +533,17 @@ class Packer(object):
             if -0x8000000000000000 <= obj < -0x80000000:
                 return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
             raise PackValueError("Integer value out of range")
+        if self._use_bin_type and isinstance(obj, bytes):
+            n = len(obj)
+            if n <= 0xff:
+                self._buffer.write(struct.pack('>BB', 0xc4, n))
+            elif n <= 0xffff:
+                self._buffer.write(struct.pack(">BH", 0xc5, n))
+            elif n <= 0xffffffff:
+                self._buffer.write(struct.pack(">BI", 0xc6, n))
+            else:
+                raise PackValueError("Bytes is too large")
+            return self._buffer.write(obj)
         if isinstance(obj, (Unicode, bytes)):
             if isinstance(obj, Unicode):
                 if self._encoding is None:
@@ -500,19 +554,20 @@ class Packer(object):
             n = len(obj)
             if n <= 0x1f:
                 self._buffer.write(struct.pack('B', 0xa0 + n))
-                return self._buffer.write(obj)
-            if n <= 0xffff:
+            elif self._use_bin_type and n <= 0xff:
+                self._buffer.write(struct.pack('>BB', 0xd9, n))
+            elif n <= 0xffff:
                 self._buffer.write(struct.pack(">BH", 0xda, n))
-                return self._buffer.write(obj)
-            if n <= 0xffffffff:
+            elif n <= 0xffffffff:
                 self._buffer.write(struct.pack(">BI", 0xdb, n))
-                return self._buffer.write(obj)
-            raise PackValueError("String is too large")
+            else:
+                raise PackValueError("String is too large")
+            return self._buffer.write(obj)
         if isinstance(obj, float):
             if self._use_float:
                 return self._buffer.write(struct.pack(">Bf", 0xca, obj))
             return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
-        if isinstance(obj, list) or isinstance(obj, tuple):
+        if isinstance(obj, (list, tuple)):
             n = len(obj)
             self._fb_pack_array_header(n)
             for i in xrange(n):
diff --git a/msgpack/pack.h b/msgpack/pack.h
index 1539991..001a0c1 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -34,11 +34,11 @@ typedef struct msgpack_packer {
     char *buf;
     size_t length;
     size_t buf_size;
+    bool use_bin_type;
 } msgpack_packer;
 
 typedef struct Packer Packer;
 
-static inline int msgpack_pack_short(msgpack_packer* pk, short d);
 static inline int msgpack_pack_int(msgpack_packer* pk, int d);
 static inline int msgpack_pack_long(msgpack_packer* pk, long d);
 static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d);
@@ -68,6 +68,7 @@ static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n);
 static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
 
 static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
+static inline int msgpack_pack_bin(msgpack_packer* pk, size_t l);
 static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
 
 static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l)
diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index 9e00d7e..d228d7a 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -664,10 +664,13 @@ static inline int msgpack_pack_map(msgpack_packer* x, unsigned int n)
 
 static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
 {
-    if(l < 32) {
+    if (l < 32) {
         unsigned char d = 0xa0 | (uint8_t)l;
         msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
-    } else if(l < 65536) {
+    } else if (x->use_bin_type && l < 256) {  // str8 is new format introduced with bin.
+        unsigned char buf[2] = {0xd9, (uint8_t)l};
+        msgpack_pack_append_buffer(x, buf, 2);
+    } else if (l < 65536) {
         unsigned char buf[3];
         buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l);
         msgpack_pack_append_buffer(x, buf, 3);
@@ -678,6 +681,28 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
     }
 }
 
+/*
+ * bin
+ */
+static inline int msgpack_pack_bin(msgpack_packer *x, size_t l)
+{
+    if (!x->use_bin_type) {
+        return msgpack_pack_raw(x, l)
+    }
+    if (l < 256) {
+        unsigned char buf[2] = {0xc4, (unsigned char)l};
+        msgpack_pack_append_buffer(x, buf, 2);
+    } else if (l < 65536) {
+        unsigned char buf[3] = {0xc5};
+        _msgpack_store16(&buf[1], (uint16_t)l);
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else {
+        unsigned char buf[5] = {0xc6};
+        _msgpack_store32(&buf[1], (uint32_t)l);
+        msgpack_pack_append_buffer(x, buf, 5);
+    }
+}
+
 static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l)
 {
     msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index baeed1f..03c735e 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -226,4 +226,13 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char*
     return 0;
 }
 
+static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
+{
+    PyObject *py = PyBytes_FromStringAndSize(p, l);
+    if (!py)
+        return -1;
+    *o = py;
+    return 0;
+}
+
 #include "unpack_template.h"
diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h
index c81b990..0b14f52 100644
--- a/msgpack/unpack_define.h
+++ b/msgpack/unpack_define.h
@@ -34,6 +34,7 @@ extern "C" {
 #endif
 
 
+// CS is first byte & 0x1f
 typedef enum {
     CS_HEADER            = 0x00,  // nil
 
@@ -41,9 +42,9 @@ typedef enum {
     //CS_                = 0x02,  // false
     //CS_                = 0x03,  // true
 
-    //CS_                = 0x04,
-    //CS_                = 0x05,
-    //CS_                = 0x06,
+    CS_BIN_8             = 0x04,
+    CS_BIN_16            = 0x05,
+    CS_BIN_32            = 0x06,
     //CS_                = 0x07,
 
     //CS_                = 0x08,
@@ -59,12 +60,7 @@ typedef enum {
     CS_INT_32            = 0x12,
     CS_INT_64            = 0x13,
 
-    //CS_                = 0x14,
-    //CS_                = 0x15,
-    //CS_BIG_INT_16        = 0x16,
-    //CS_BIG_INT_32        = 0x17,
-    //CS_BIG_FLOAT_16      = 0x18,
-    //CS_BIG_FLOAT_32      = 0x19,
+    CS_RAW_8             = 0x19,
     CS_RAW_16            = 0x1a,
     CS_RAW_32            = 0x1b,
     CS_ARRAY_16          = 0x1c,
@@ -72,9 +68,8 @@ typedef enum {
     CS_MAP_16            = 0x1e,
     CS_MAP_32            = 0x1f,
 
-    //ACS_BIG_INT_VALUE,
-    //ACS_BIG_FLOAT_VALUE,
     ACS_RAW_VALUE,
+    ACS_BIN_VALUE,
 } msgpack_unpack_state;
 
 
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 29ac935..25229ac 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -151,8 +151,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
     */ \
     goto _header_again
 
-#define NEXT_CS(p) \
-    ((unsigned int)*p & 0x1f)
+#define NEXT_CS(p)  ((unsigned int)*p & 0x1f)
 
 #ifdef USE_CASE_RANGE
 #define SWITCH_RANGE_BEGIN     switch(*p) {
@@ -185,9 +184,6 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
                     push_simple_value(_false);
                 case 0xc3:  // true
                     push_simple_value(_true);
-                //case 0xc4:
-                //case 0xc5:
-                //case 0xc6:
                 //case 0xc7:
                 //case 0xc8:
                 //case 0xc9:
@@ -202,12 +198,15 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
                 case 0xd2:  // signed int 32
                 case 0xd3:  // signed int 64
                     again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
+                case 0xc4:  // bin 8
+                case 0xc5:  // bin 16
+                case 0xc6:  // bin 32
                 //case 0xd4:
                 //case 0xd5:
                 //case 0xd6:  // big integer 16
                 //case 0xd7:  // big integer 32
                 //case 0xd8:  // big float 16
-                //case 0xd9:  // big float 32
+                case 0xd9:  // raw 8
                 case 0xda:  // raw 16
                 case 0xdb:  // raw 32
                 case 0xdc:  // array 16
@@ -290,6 +289,18 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
             //    // FIXME
             //    push_variable_value(_big_float, data, n, trail);
 
+            case CS_BIN_8:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, *(uint8_t*)n, _bin_zero);
+            case CS_BIN_16:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load16(uint16_t,n), _bin_zero);
+            case CS_BIN_32:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load32(uint32_t,n), _bin_zero);
+            case ACS_BIN_VALUE:
+            _bin_zero:
+                push_variable_value(_bin, data, n, trail);
+
+            case CS_RAW_8:
+                again_fixed_trail_if_zero(ACS_RAW_VALUE, *(uint8_t*)n, _raw_zero);
             case CS_RAW_16:
                 again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
             case CS_RAW_32:
diff --git a/test/test_newspec.py b/test/test_newspec.py
new file mode 100644
index 0000000..8bc2cfe
--- /dev/null
+++ b/test/test_newspec.py
@@ -0,0 +1,69 @@
+# coding: utf-8
+
+from msgpack import packb, unpackb
+
+
+def test_str8():
+    header = b'\xd9'
+    data = b'x' * 32
+    b = packb(data.decode(), use_bin_type=True)
+    assert len(b) == len(data) + 2
+    assert b[0:2] == header + b'\x20'
+    assert b[2:] == data
+    assert unpackb(b) == data
+
+    data = b'x' * 255
+    b = packb(data.decode(), use_bin_type=True)
+    assert len(b) == len(data) + 2
+    assert b[0:2] == header + b'\xff'
+    assert b[2:] == data
+    assert unpackb(b) == data
+
+
+def test_bin8():
+    header = b'\xc4'
+    data = b''
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 2
+    assert b[0:2] == header + b'\x00'
+    assert b[2:] == data
+    assert unpackb(b) == data
+
+    data = b'x' * 255
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 2
+    assert b[0:2] == header + b'\xff'
+    assert b[2:] == data
+    assert unpackb(b) == data
+
+
+def test_bin16():
+    header = b'\xc5'
+    data = b'x' * 256
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 3
+    assert b[0:1] == header
+    assert b[1:3] == b'\x01\x00'
+    assert b[3:] == data
+    assert unpackb(b) == data
+
+    data = b'x' * 65535
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 3
+    assert b[0:1] == header
+    assert b[1:3] == b'\xff\xff'
+    assert b[3:] == data
+    assert unpackb(b) == data
+
+
+def test_bin32():
+    header = b'\xc6'
+    data = b'x' * 65536
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 5
+    assert b[0:1] == header
+    assert b[1:5] == b'\x00\x01\x00\x00'
+    assert b[5:] == data
+    assert unpackb(b) == data
+
+