diff options
author | INADA Naoki <songofacandy@gmail.com> | 2013-02-23 01:14:13 -0800 |
---|---|---|
committer | INADA Naoki <songofacandy@gmail.com> | 2013-02-23 01:14:13 -0800 |
commit | 04e0812ad409d3aa2d18f3f3de02812164a83fea (patch) | |
tree | da34edb37f4446de89bfe85de2e9b4ee1ca53f9b | |
parent | f0fd90a759f07bccfd36e4f9eac8c4b545fd47a4 (diff) | |
parent | 38a9ad98c9e8e8b73efb9726554737a60ec215c9 (diff) | |
download | msgpack-python-04e0812ad409d3aa2d18f3f3de02812164a83fea.tar.gz |
Merge pull request #51 from msgpack/revert-skip-reserved
Revert skipping reserved bytes.
-rw-r--r-- | msgpack/fallback.py | 9 | ||||
-rw-r--r-- | msgpack/unpack_template.h | 383 | ||||
-rw-r--r-- | test/test_reserved.py | 22 |
3 files changed, 179 insertions, 235 deletions
diff --git a/msgpack/fallback.py b/msgpack/fallback.py index f893e1a..e9dddd6 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -48,7 +48,7 @@ from msgpack.exceptions import ( PackValueError, ExtraData) -EX_SKIP = 0 +EX_SKIP = 0 EX_CONSTRUCT = 1 EX_READ_ARRAY_HEADER = 2 EX_READ_MAP_HEADER = 3 @@ -57,7 +57,6 @@ TYPE_IMMEDIATE = 0 TYPE_ARRAY = 1 TYPE_MAP = 2 TYPE_RAW = 3 -TYPE_RESERVED = 4 DEFAULT_RECURSE_LIMIT=511 @@ -338,13 +337,11 @@ class Unpacker(object): n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] typ = TYPE_MAP else: - typ = TYPE_RESERVED + raise UnpackValueError("Unknown header: 0x%x" % b) return typ, n, obj def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None): - typ = TYPE_RESERVED - while typ == TYPE_RESERVED: - typ, n, obj = self._read_header(execute, write_bytes) + typ, n, obj = self._read_header(execute, write_bytes) if execute == EX_READ_ARRAY_HEADER: if typ != TYPE_ARRAY: diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 6f86695..83b6918 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -60,10 +60,10 @@ msgpack_unpack_struct_decl(_context) { unsigned int trail; unsigned int top; /* - msgpack_unpack_struct(_stack)* stack; - unsigned int stack_size; - msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE]; - */ + msgpack_unpack_struct(_stack)* stack; + unsigned int stack_size; + msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE]; + */ msgpack_unpack_struct(_stack) stack[MSGPACK_EMBED_STACK_SIZE]; }; @@ -74,20 +74,20 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) ctx->trail = 0; ctx->top = 0; /* - ctx->stack = ctx->embed_stack; - ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; - */ + ctx->stack = ctx->embed_stack; + ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; + */ ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); } /* - msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx) - { - if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { - free(ctx->stack); - } - } - */ +msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx) +{ + if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { + free(ctx->stack); + } +} +*/ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) { @@ -109,8 +109,8 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c unsigned int top = ctx->top; msgpack_unpack_struct(_stack)* stack = ctx->stack; /* - unsigned int stack_size = ctx->stack_size; - */ + unsigned int stack_size = ctx->stack_size; + */ msgpack_unpack_user* user = &ctx->user; msgpack_unpack_object obj; @@ -119,7 +119,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c int ret; #define construct_cb(name) \ - construct && msgpack_unpack_callback(name) + construct && msgpack_unpack_callback(name) #define push_simple_value(func) \ if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \ @@ -129,7 +129,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c goto _push #define push_variable_value(func, base, pos, len) \ if(construct_cb(func)(user, \ - (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ goto _push #define again_fixed_trail(_cs, trail_len) \ @@ -155,24 +155,24 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ /*printf("stack push %d\n", top);*/ \ /* FIXME \ - if(top >= stack_size) { \ - if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ - size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \ - size_t nsize = csize * 2; \ - msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \ - if(tmp == NULL) { goto _failed; } \ - memcpy(tmp, ctx->stack, csize); \ - ctx->stack = stack = tmp; \ - ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ - } else { \ - size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \ - msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \ - if(tmp == NULL) { goto _failed; } \ - ctx->stack = stack = tmp; \ - ctx->stack_size = stack_size = stack_size * 2; \ - } \ - } \ - */ \ + if(top >= stack_size) { \ + if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ + size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \ + size_t nsize = csize * 2; \ + msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \ + if(tmp == NULL) { goto _failed; } \ + memcpy(tmp, ctx->stack, csize); \ + ctx->stack = stack = tmp; \ + ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ + } else { \ + size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \ + msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \ + if(tmp == NULL) { goto _failed; } \ + ctx->stack = stack = tmp; \ + ctx->stack_size = stack_size = stack_size * 2; \ + } \ + } \ + */ \ goto _header_again #define NEXT_CS(p) \ @@ -195,7 +195,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c switch(cs) { case CS_HEADER: SWITCH_RANGE_BEGIN - SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum + SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum push_fixed_value(_uint8, *(uint8_t*)p); SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum push_fixed_value(_int8, *(int8_t*)p); @@ -203,24 +203,18 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c switch(*p) { case 0xc0: // nil push_simple_value(_nil); - // reserved. - case 0xc1: // string - fprintf(stderr, "skipping %x\n", (int)*p); - goto _header_again; - // again_terminal_trail(NEXT_CS(p), p+1); + //case 0xc1: // string + // again_terminal_trail(NEXT_CS(p), p+1); case 0xc2: // false push_simple_value(_false); case 0xc3: // true push_simple_value(_true); - // reserved. - case 0xc4: - case 0xc5: - case 0xc6: - case 0xc7: - case 0xc8: - case 0xc9: - fprintf(stderr, "skipping %x\n", (int)*p); - goto _header_again; + //case 0xc4: + //case 0xc5: + //case 0xc6: + //case 0xc7: + //case 0xc8: + //case 0xc9: case 0xca: // float case 0xcb: // double case 0xcc: // unsigned int 8 @@ -232,15 +226,12 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c case 0xd2: // signed int 32 case 0xd3: // signed int 64 again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); - // reserved. - case 0xd4: - case 0xd5: - case 0xd6: // big integer 16 - case 0xd7: // big integer 32 - case 0xd8: // big float 16 - case 0xd9: // big float 32 - fprintf(stderr, "skipping %x\n", (int)*p); - goto _header_again; + //case 0xd4: + //case 0xd5: + //case 0xd6: // big integer 16 + //case 0xd7: // big integer 32 + //case 0xd8: // big float 16 + //case 0xd9: // big float 32 case 0xda: // raw 16 case 0xdb: // raw 32 case 0xdc: // array 16 @@ -249,7 +240,6 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c case 0xdf: // map 32 again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); default: - fprintf(stderr, "failed %x\n", (int)*p); goto _failed; } SWITCH_RANGE(0xa0, 0xbf) // FixRaw @@ -260,129 +250,128 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); SWITCH_RANGE_DEFAULT - fprintf(stderr, "failed2 %x\n", (int)*p); - goto _failed; + goto _failed; SWITCH_RANGE_END - // end CS_HEADER + // end CS_HEADER - _fixed_trail_again: - ++p; + _fixed_trail_again: + ++p; default: if((size_t)(pe - p) < trail) { goto _out; } n = p; p += trail - 1; switch(cs) { - //case CS_ - //case CS_ + //case CS_ + //case CS_ case CS_FLOAT: { - union { uint32_t i; float f; } mem; - mem.i = _msgpack_load32(uint32_t,n); - push_fixed_value(_float, mem.f); } + union { uint32_t i; float f; } mem; + mem.i = _msgpack_load32(uint32_t,n); + push_fixed_value(_float, mem.f); } case CS_DOUBLE: { - union { uint64_t i; double f; } mem; - mem.i = _msgpack_load64(uint64_t,n); + union { uint64_t i; double f; } mem; + mem.i = _msgpack_load64(uint64_t,n); #if defined(__arm__) && !(__ARM_EABI__) // arm-oabi - // https://github.com/msgpack/msgpack-perl/pull/1 - mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL); + // https://github.com/msgpack/msgpack-perl/pull/1 + mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL); #endif - push_fixed_value(_double, mem.f); } + push_fixed_value(_double, mem.f); } case CS_UINT_8: - push_fixed_value(_uint8, *(uint8_t*)n); + push_fixed_value(_uint8, *(uint8_t*)n); case CS_UINT_16: - push_fixed_value(_uint16, _msgpack_load16(uint16_t,n)); + push_fixed_value(_uint16, _msgpack_load16(uint16_t,n)); case CS_UINT_32: - push_fixed_value(_uint32, _msgpack_load32(uint32_t,n)); + push_fixed_value(_uint32, _msgpack_load32(uint32_t,n)); case CS_UINT_64: - push_fixed_value(_uint64, _msgpack_load64(uint64_t,n)); + push_fixed_value(_uint64, _msgpack_load64(uint64_t,n)); case CS_INT_8: - push_fixed_value(_int8, *(int8_t*)n); + push_fixed_value(_int8, *(int8_t*)n); case CS_INT_16: - push_fixed_value(_int16, _msgpack_load16(int16_t,n)); + push_fixed_value(_int16, _msgpack_load16(int16_t,n)); case CS_INT_32: - push_fixed_value(_int32, _msgpack_load32(int32_t,n)); + push_fixed_value(_int32, _msgpack_load32(int32_t,n)); case CS_INT_64: - push_fixed_value(_int64, _msgpack_load64(int64_t,n)); - - //case CS_ - //case CS_ - //case CS_BIG_INT_16: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero); - //case CS_BIG_INT_32: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero); - //case ACS_BIG_INT_VALUE: - //_big_int_zero: - // // FIXME - // push_variable_value(_big_int, data, n, trail); - - //case CS_BIG_FLOAT_16: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero); - //case CS_BIG_FLOAT_32: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero); - //case ACS_BIG_FLOAT_VALUE: - //_big_float_zero: - // // FIXME - // push_variable_value(_big_float, data, n, trail); + push_fixed_value(_int64, _msgpack_load64(int64_t,n)); + + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(_big_int, data, n, trail); + + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(_big_float, data, n, trail); case CS_RAW_16: - again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero); + again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero); case CS_RAW_32: - again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero); + again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero); case ACS_RAW_VALUE: -_raw_zero: - push_variable_value(_raw, data, n, trail); + _raw_zero: + push_variable_value(_raw, data, n, trail); case CS_ARRAY_16: - start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); + start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); case CS_ARRAY_32: - /* FIXME security guard */ - start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM); + /* FIXME security guard */ + start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM); case CS_MAP_16: - start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY); + start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY); case CS_MAP_32: - /* FIXME security guard */ - start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); + /* FIXME security guard */ + start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); default: - goto _failed; + goto _failed; } } _push: - if(top == 0) { goto _finish; } - c = &stack[top-1]; - switch(c->ct) { - case CT_ARRAY_ITEM: - if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } - if(++c->count == c->size) { - obj = c->obj; - if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; } - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - goto _header_again; - case CT_MAP_KEY: - c->map_key = obj; - c->ct = CT_MAP_VALUE; - goto _header_again; - case CT_MAP_VALUE: - if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } - if(++c->count == c->size) { - obj = c->obj; - if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; } - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - c->ct = CT_MAP_KEY; - goto _header_again; - - default: - goto _failed; + if(top == 0) { goto _finish; } + c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } + if(++c->count == c->size) { + obj = c->obj; + if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; } + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + goto _header_again; + case CT_MAP_KEY: + c->map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(++c->count == c->size) { + obj = c->obj; + if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; } + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; } + c->ct = CT_MAP_KEY; + goto _header_again; + + default: + goto _failed; + } _header_again: cs = CS_HEADER; @@ -433,73 +422,53 @@ _end: template <unsigned int fixed_offset, unsigned int var_offset> msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) { - for (;;) { - if (len < *off) { - return 0; - } - uint32_t size; - const unsigned char *const p = (unsigned char*)data + *off; + assert(len >= *off); + uint32_t size; + const unsigned char *const p = (unsigned char*)data + *off; #define inc_offset(inc) \ - if (len - *off < inc) \ + if (len - *off < inc) \ return 0; \ - *off += inc; - - switch (*p) { - case var_offset: - inc_offset(3); - size = _msgpack_load16(uint16_t, p + 1); - break; - case var_offset + 1: - inc_offset(5); - size = _msgpack_load32(uint32_t, p + 1); - break; + *off += inc; + + switch (*p) { + case var_offset: + inc_offset(3); + size = _msgpack_load16(uint16_t, p + 1); + break; + case var_offset + 1: + inc_offset(5); + size = _msgpack_load32(uint32_t, p + 1); + break; #ifdef USE_CASE_RANGE - case fixed_offset + 0x0 ... fixed_offset + 0xf: + case fixed_offset + 0x0 ... fixed_offset + 0xf: #else - case fixed_offset + 0x0: - case fixed_offset + 0x1: - case fixed_offset + 0x2: - case fixed_offset + 0x3: - case fixed_offset + 0x4: - case fixed_offset + 0x5: - case fixed_offset + 0x6: - case fixed_offset + 0x7: - case fixed_offset + 0x8: - case fixed_offset + 0x9: - case fixed_offset + 0xa: - case fixed_offset + 0xb: - case fixed_offset + 0xc: - case fixed_offset + 0xd: - case fixed_offset + 0xe: - case fixed_offset + 0xf: + case fixed_offset + 0x0: + case fixed_offset + 0x1: + case fixed_offset + 0x2: + case fixed_offset + 0x3: + case fixed_offset + 0x4: + case fixed_offset + 0x5: + case fixed_offset + 0x6: + case fixed_offset + 0x7: + case fixed_offset + 0x8: + case fixed_offset + 0x9: + case fixed_offset + 0xa: + case fixed_offset + 0xb: + case fixed_offset + 0xc: + case fixed_offset + 0xd: + case fixed_offset + 0xe: + case fixed_offset + 0xf: #endif - ++*off; - size = ((unsigned int)*p) & 0x0f; - break; - // reserved: - case 0xc1: - case 0xc4: - case 0xc5: - case 0xc6: - case 0xc7: - case 0xc8: - case 0xc9: - case 0xd4: - case 0xd5: - case 0xd6: // big integer 16 - case 0xd7: // big integer 32 - case 0xd8: // big float 16 - case 0xd9: // big float 32 - ++*off; - continue; - default: - PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); - return -1; - } - msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj); - return 1; - } + ++*off; + size = ((unsigned int)*p) & 0x0f; + break; + default: + PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); + return -1; + } + msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj); + return 1; } #undef SWITCH_RANGE_BEGIN diff --git a/test/test_reserved.py b/test/test_reserved.py deleted file mode 100644 index 5094064..0000000 --- a/test/test_reserved.py +++ /dev/null @@ -1,22 +0,0 @@ -import msgpack - -reserved_bytes = [ - b"\xc1", - b"\xc4", - b"\xc5", - b"\xc6", - b"\xc7", - b"\xc8", - b"\xc9", - b"\xd4", - b"\xd5", - b"\xd6", - b"\xd7", - b"\xd8", - b"\xd9", - ] - -def test_skip_reserved(): - packed_list = msgpack.packb([]) - for b in reserved_bytes: - assert msgpack.unpackb(b+packed_list, use_list=1) == [] |