diff options
author | Luke Chen <luke.chen@mongodb.com> | 2019-04-16 14:36:51 +1000 |
---|---|---|
committer | Luke Chen <luke.chen@mongodb.com> | 2019-04-16 14:45:56 +1000 |
commit | a654dcf592ea7ed65426a0de96b4079ff4fc6716 (patch) | |
tree | a5256edad1bb219e6af72fd7e7525f58e235a307 /src/third_party/wiredtiger/lang | |
parent | 19b622ebfb42a525f38e278c09f440eb47b12f1e (diff) | |
download | mongo-a654dcf592ea7ed65426a0de96b4079ff4fc6716.tar.gz |
Import wiredtiger: 9416282c42d40328dfb7ff0f28831f639f98d3cb from branch mongodb-4.2
ref: 1768d66613..9416282c42
for: 4.1.11
WT-4317 Read checksum error in test_wt4156_metadata_salvage
WT-4579 Track the newest durable timestamp for each page
WT-4585 Add WT_WITH_HOTBACKUP_LOCK macro
WT-4598 Enable the assertion that the durable_timestamp is newer than or equals the commit timestamp.
WT-4640 Remove round_to_oldest in favour of roundup_timestamps
WT-4695 Python3: allow most tests to run with Python3 with small changes
WT-4696 Python3: change dist scripts to run under Python3
WT-4698 Python3: fix modify related tests
WT-4699 Python3: fix test_jsondump02.py
WT-4700 Python3: run with same source as Python2
WT-4703 Extend test/checkpoint to do removes and online checking
WT-4704 Add statistic tracking oldest active read timestamp
WT-4705 column-store no longer needs to handle WT_COL page offsets of 0
WT-4707 Failure in verifying cells with copied values
WT-4708 Coverity reported copy-paste error in WiredTiger error message
WT-4711 Python formatting errors reported while running "s_all"
WT-4714 Use the durable timestamp to determine if a page should stay dirty
WT-4724 Syntax error in wtperf_ckpt.sh when running 'dash' as default shell
Diffstat (limited to 'src/third_party/wiredtiger/lang')
-rw-r--r-- | src/third_party/wiredtiger/lang/python/Makefile.am | 23 | ||||
-rw-r--r-- | src/third_party/wiredtiger/lang/python/wiredtiger.i | 185 | ||||
-rwxr-xr-x[-rw-r--r--] | src/third_party/wiredtiger/lang/python/wiredtiger/fpacking.py | 3 | ||||
-rwxr-xr-x[-rw-r--r--] | src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py | 52 | ||||
-rwxr-xr-x[-rw-r--r--] | src/third_party/wiredtiger/lang/python/wiredtiger/packing.py | 88 |
5 files changed, 269 insertions, 82 deletions
diff --git a/src/third_party/wiredtiger/lang/python/Makefile.am b/src/third_party/wiredtiger/lang/python/Makefile.am index d7539c35bdd..3f29ea6aeea 100644 --- a/src/third_party/wiredtiger/lang/python/Makefile.am +++ b/src/third_party/wiredtiger/lang/python/Makefile.am @@ -1,18 +1,33 @@ PYSRC = $(top_srcdir)/lang/python PYDIRS = -t $(abs_builddir) -I $(abs_top_srcdir):$(abs_top_builddir) -L $(abs_top_builddir)/.libs -all-local: _wiredtiger.so +PYDST = $(abs_builddir)/wiredtiger +PYFILES = $(PYDST)/fpacking.py $(PYDST)/intpacking.py $(PYDST)/packing.py \ + $(PYDST)/__init__.py +PY_MAJOR_VERSION := $$($(PYTHON) -c \ + 'import sys; print(int(sys.version_info.major))') + +all-local: _wiredtiger.so pyfiles # We keep generated Python sources under lang/python: that's where they live # in release packages. -$(PYSRC)/wiredtiger_wrap.c: $(top_srcdir)/src/include/wiredtiger.in $(PYSRC)/wiredtiger.i +$(PYSRC)/wiredtiger_wrap.c $(PYSRC)/wiredtiger.py: $(top_srcdir)/src/include/wiredtiger.in $(PYSRC)/wiredtiger.i (cd $(PYSRC) && \ - $(SWIG) -python -threads -O -Wall -nodefaultctor -nodefaultdtor -I$(abs_top_builddir) wiredtiger.i && \ - mv wiredtiger.py wiredtiger/__init__.py) + $(SWIG) -python -DPY_MAJOR_VERSION=$(PY_MAJOR_VERSION) \ + -threads -O -Wall -nodefaultctor -nodefaultdtor \ + -I$(abs_top_builddir) wiredtiger.i) _wiredtiger.so: $(top_builddir)/libwiredtiger.la $(PYSRC)/wiredtiger_wrap.c (cd $(PYSRC) && \ $(PYTHON) setup.py build_ext -f -b $(abs_builddir) $(PYDIRS)) +pyfiles: $(PYFILES) + +$(PYDST)/%: $(PYSRC)/wiredtiger/% + mkdir -p $(PYDST) && cp -f $< $@ + +$(PYDST)/__init__.py: $(PYSRC)/wiredtiger.py + mkdir -p $(PYDST) && cp -f $< $@ + install-exec-local: (cd $(PYSRC) && \ $(PYTHON) setup.py build_py -d $(abs_builddir)/build && \ diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger.i b/src/third_party/wiredtiger/lang/python/wiredtiger.i index 7a9af32a9d1..86182b1716c 100644 --- a/src/third_party/wiredtiger/lang/python/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/python/wiredtiger.i @@ -30,6 +30,8 @@ * wiredtiger.i * The SWIG interface file defining the wiredtiger python API. */ +%include <pybuffer.i> + %define DOCSTRING "Python wrappers around the WiredTiger C API @@ -48,7 +50,7 @@ This provides an API similar to the C API, with the following modifications: %feature("autodoc", "0"); %pythoncode %{ -from packing import pack, unpack +from .packing import pack, unpack ## @endcond %} @@ -161,10 +163,11 @@ from packing import pack, unpack %typemap(argout) (WT_MODIFY *entries, int *nentriesp) { int i; + $result = PyList_New(*$2); for (i = 0; i < *$2; i++) { PyObject *o = SWIG_NewPointerObj(Py_None, SWIGTYPE_p___wt_modify, 0); - PyObject_SetAttrString(o, "data", PyString_FromStringAndSize( + PyObject_SetAttrString(o, "data", PyBytes_FromStringAndSize( $1[i].data.data, $1[i].data.size)); PyObject_SetAttrString(o, "offset", PyInt_FromLong($1[i].offset)); @@ -174,15 +177,30 @@ from packing import pack, unpack } } -%typemap(in) const WT_ITEM * (WT_ITEM val, long sz) { - if (PyString_AsStringAndSize($input, &val.data, &sz) < 0) +%typemap(argout) (WT_MODIFY *entries_string, int *nentriesp) { + int i; + + $result = PyList_New(*$2); + for (i = 0; i < *$2; i++) { + PyObject *o = SWIG_NewPointerObj(Py_None, SWIGTYPE_p___wt_modify, 0); + PyObject_SetAttrString(o, "data", PyUnicode_FromStringAndSize( + $1[i].data.data, $1[i].data.size)); + PyObject_SetAttrString(o, "offset", + PyInt_FromLong($1[i].offset)); + PyObject_SetAttrString(o, "size", + PyInt_FromLong($1[i].size)); + PyList_SetItem($result, i, o); + } + } + +%typemap(in) const WT_ITEM * (WT_ITEM val) { + if (unpackBytesOrString($input, &val.data, &val.size) != 0) SWIG_exception_fail(SWIG_AttributeError, "bad string value for WT_ITEM"); - val.size = (size_t)sz; $1 = &val; } -%typemap(freearg) (WT_MODIFY *entries, int *nentriesp) { +%typemap(freearg) (WT_MODIFY *, int *nentriesp) { __wt_free(NULL, $1); } @@ -198,23 +216,32 @@ from packing import pack, unpack modarray[0].size = (size_t)len; for (i = 1; i <= len; i++) { PyObject *dataobj, *modobj, *offsetobj, *sizeobj; - char *datadata; + void *datadata; long offset, size; - Py_ssize_t datasize; + size_t datasize; - if ((modobj = PySequence_GetItem($input, i - 1)) == NULL) + if ((modobj = PySequence_GetItem($input, i - 1)) == NULL) { + freeModifyArray(modarray); SWIG_exception_fail(SWIG_IndexError, "Modify sequence failed"); + } WT_GETATTR(dataobj, modobj, "data"); - if (PyString_AsStringAndSize(dataobj, &datadata, - &datasize) < 0) { + if (unpackBytesOrString(dataobj, &datadata, &datasize) != 0) { Py_DECREF(dataobj); Py_DECREF(modobj); + freeModifyArray(modarray); SWIG_exception_fail(SWIG_AttributeError, "Modify.data bad value"); } - modarray[i].data.data = malloc(datasize); + if (datasize != 0 && + __wt_malloc(NULL, datasize, &modarray[i].data.data) != 0) { + Py_DECREF(dataobj); + Py_DECREF(modobj); + freeModifyArray(modarray); + SWIG_exception_fail(SWIG_AttributeError, + "Modify.data failed malloc"); + } memcpy(modarray[i].data.data, datadata, datasize); modarray[i].data.size = datasize; Py_DECREF(dataobj); @@ -223,6 +250,7 @@ from packing import pack, unpack if ((offset = PyInt_AsLong(offsetobj)) < 0) { Py_DECREF(offsetobj); Py_DECREF(modobj); + freeModifyArray(modarray); SWIG_exception_fail(SWIG_RuntimeError, "Modify.offset bad value"); } @@ -233,6 +261,7 @@ from packing import pack, unpack if ((size = PyInt_AsLong(sizeobj)) < 0) { Py_DECREF(sizeobj); Py_DECREF(modobj); + freeModifyArray(modarray); SWIG_exception_fail(SWIG_RuntimeError, "Modify.size bad value"); } @@ -244,14 +273,7 @@ from packing import pack, unpack } %typemap(freearg) WT_MODIFY * { - /* The WT_MODIFY arg is in position 2. Is there a better way? */ - WT_MODIFY *modarray = modarray2; - size_t i, len; - - len = modarray[0].size; - for (i = 1; i <= len; i++) - __wt_free(NULL, modarray[i].data.data); - __wt_free(NULL, modarray); + freeModifyArray($1); } /* 64 bit typemaps. */ @@ -262,6 +284,9 @@ from packing import pack, unpack $result = PyLong_FromUnsignedLongLong($1); } +/* Internal _set_key, _set_value methods take a 'bytes' object as parameter. */ +%pybuffer_binary(void *data, int); + /* Throw away references after close. */ %define DESTRUCTOR(class, method) %feature("shadow") class::method %{ @@ -347,6 +372,7 @@ static PyObject *wtError; static int sessionFreeHandler(WT_SESSION *session_arg); static int cursorFreeHandler(WT_CURSOR *cursor_arg); +static int unpackBytesOrString(PyObject *obj, void **data, size_t *size); #define WT_GETATTR(var, parent, name) \ do if ((var = PyObject_GetAttrString(parent, name)) == NULL) { \ @@ -372,6 +398,15 @@ static int cursorFreeHandler(WT_CURSOR *cursor_arg); %pythoncode %{ WiredTigerError = _wiredtiger.WiredTigerError +# Python3 has no explicit long type, recnos work as ints +import sys +if sys.version_info >= (3, 0, 0): + def _wt_recno(i): + return i +else: + def _wt_recno(i): + return long(i) + ## @cond DISABLE # Implements the iterable contract class IterableCursor: @@ -381,10 +416,13 @@ class IterableCursor: def __iter__(self): return self - def next(self): + def __next__(self): if self.cursor.next() == WT_NOTFOUND: raise StopIteration return self.cursor.get_keys() + self.cursor.get_values() + + def next(self): + return self.__next__() ## @endcond # An abstract class, which must be subclassed with notify() overridden. @@ -395,6 +433,12 @@ class AsyncCallback: def notify(self, op, op_ret, flags): raise NotImplementedError +def wiredtiger_calc_modify(session, oldv, newv, maxdiff, nmod): + return _wiredtiger_calc_modify(session, oldv, newv, maxdiff, nmod) + +def wiredtiger_calc_modify_string(session, oldv, newv, maxdiff, nmod): + return _wiredtiger_calc_modify_string(session, oldv, newv, maxdiff, nmod) + %} /* Bail out if arg or arg.this is None, else set res to the C pointer. */ @@ -576,16 +620,26 @@ OVERRIDE_METHOD(__wt_cursor, WT_CURSOR, equals, (self, other)) OVERRIDE_METHOD(__wt_cursor, WT_CURSOR, search_near, (self)) /* SWIG magic to turn Python byte strings into data / size. */ -%apply (char *STRING, int LENGTH) { (char *data, int size) }; +#if PY_MAJOR_VERSION >= 3 + %apply (char *STRING, int LENGTH) { (char *data, int size) }; +#else +%apply (char *STRING, int LENGTH) { (void *data, int size) }; +#endif /* Handle binary data returns from get_key/value -- avoid cstring.i: it creates a list of returns. */ %typemap(in,numinputs=0) (char **datap, int *sizep) (char *data, int size) { $1 = &data; $2 = &size; } +%typemap(in,numinputs=0) (char **charp, int *sizep) (char *data, int size) { $1 = &data; $2 = &size; } %typemap(frearg) (char **datap, int *sizep) ""; -%typemap(argout) (char **datap, int *sizep) { +%typemap(argout) (char **charp, int *sizep) { if (*$1) - $result = SWIG_FromCharPtrAndSize(*$1, *$2); + $result = PyUnicode_FromStringAndSize(*$1, *$2); } +%typemap(argout) (char **datap, int *sizep) { + if (*$1) + $result = PyBytes_FromStringAndSize(*$1, *$2); + } + /* Handle record number returns from get_recno */ %typemap(in,numinputs=0) (uint64_t *recnop) (uint64_t recno) { $1 = &recno; } %typemap(frearg) (uint64_t *recnop) ""; @@ -606,7 +660,7 @@ typedef int int_void; %extend __wt_async_op { /* Get / set keys and values */ - void _set_key(char *data, int size) { + void _set_key(void *data, int size) { WT_ITEM k; k.data = data; k.size = (uint32_t)size; @@ -630,7 +684,7 @@ typedef int int_void; return (ret); } - void _set_value(char *data, int size) { + void _set_value(void *data, int size) { WT_ITEM v; v.data = data; v.size = (uint32_t)size; @@ -714,7 +768,7 @@ typedef int int_void; if len(args) == 1 and type(args[0]) == tuple: args = args[0] if self.is_column: - self._set_recno(long(args[0])) + self._set_recno(_wt_recno(args[0])) else: # Keep the Python string pinned self._key = pack(self.key_format, *args) @@ -747,7 +801,7 @@ typedef int int_void; %extend __wt_cursor { /* Get / set keys and values */ - void _set_key(char *data, int size) { + void _set_key(void *data, int size) { WT_ITEM k; k.data = data; k.size = (uint32_t)size; @@ -776,7 +830,7 @@ typedef int int_void; return (ret); } - void _set_value(char *data, int size) { + void _set_value(void *data, int size) { WT_ITEM v; v.data = data; v.size = (uint32_t)size; @@ -799,11 +853,11 @@ typedef int int_void; return (ret); } - int_void _get_json_key(char **datap, int *sizep) { + int_void _get_json_key(char **charp, int *sizep) { const char *k; int ret = $self->get_key($self, &k); if (ret == 0) { - *datap = (char *)k; + *charp = (char *)k; *sizep = strlen(k); } return (ret); @@ -828,11 +882,11 @@ typedef int int_void; return (ret); } - int_void _get_json_value(char **datap, int *sizep) { + int_void _get_json_value(char **charp, int *sizep) { const char *k; int ret = $self->get_value($self, &k); if (ret == 0) { - *datap = (char *)k; + *charp = (char *)k; *sizep = strlen(k); } return (ret); @@ -953,7 +1007,7 @@ typedef int int_void; if len(args) == 1 and type(args[0]) == tuple: args = args[0] if self.is_column: - self._set_recno(long(args[0])) + self._set_recno(_wt_recno(args[0])) elif self.is_json: self._set_key_str(args[0]) else: @@ -1086,9 +1140,11 @@ OVERRIDE_METHOD(__wt_session, WT_SESSION, log_printf, (self, msg)) %ignore wiredtiger_struct_size; %ignore wiredtiger_struct_unpack; +%ignore wiredtiger_calc_modify; %ignore wiredtiger_extension_init; %ignore wiredtiger_extension_terminate; + /* Convert 'int *' to output args for wiredtiger_version */ %apply int *OUTPUT { int * }; @@ -1100,8 +1156,67 @@ OVERRIDE_METHOD(__wt_session, WT_SESSION, log_printf, (self, msg)) %include "wiredtiger.h" -/* Add event handler support. */ +/* + * The original wiredtiger_calc_modify was ignored, now we define our own. + * Python needs to know whether to return a bytes object or a string. + * Part of the smarts to do that is the output typemap, which matches on + * the naming of the parameter: entries vs. entries_string + */ +extern int _wiredtiger_calc_modify(WT_SESSION *session, + const WT_ITEM *oldv, const WT_ITEM *newv, + size_t maxdiff, WT_MODIFY *entries, int *nentriesp); +extern int _wiredtiger_calc_modify_string(WT_SESSION *session, + const WT_ITEM *oldv, const WT_ITEM *newv, + size_t maxdiff, WT_MODIFY *entries_string, int *nentriesp); %{ +int _wiredtiger_calc_modify(WT_SESSION *session, + const WT_ITEM *oldv, const WT_ITEM *newv, + size_t maxdiff, WT_MODIFY *entries, int *nentriesp) +{ + return (wiredtiger_calc_modify( + session, oldv, newv, maxdiff, entries, nentriesp)); +} + +int _wiredtiger_calc_modify_string(WT_SESSION *session, + const WT_ITEM *oldv, const WT_ITEM *newv, + size_t maxdiff, WT_MODIFY *entries_string, int *nentriesp) +{ + return (wiredtiger_calc_modify( + session, oldv, newv, maxdiff, entries_string, nentriesp)); +} + +/* Add event handler support. */ + +static void +freeModifyArray(WT_MODIFY *modarray) +{ + size_t i, len; + + len = modarray[0].size; + for (i = 1; i <= len; i++) + __wt_free(NULL, modarray[i].data.data); + __wt_free(NULL, modarray); +} + +static int unpackBytesOrString(PyObject *obj, void **datap, size_t *sizep) +{ + void *data; + Py_ssize_t sz; + + if (PyBytes_AsStringAndSize(obj, &data, &sz) < 0) { +#if PY_VERSION_HEX >= 0x03000000 + PyErr_Clear(); + if ((data = PyUnicode_AsUTF8AndSize(obj, &sz)) != 0) + *sizep = strlen((char *)data) + 1; + else +#endif + return (-1); + } + *datap = data; + *sizep = sz; + return (0); +} + /* Write to and flush the stream. */ static int writeToPythonStream(const char *streamname, const char *message) diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/fpacking.py b/src/third_party/wiredtiger/lang/python/wiredtiger/fpacking.py index 5ad37895820..6501783a8e3 100644..100755 --- a/src/third_party/wiredtiger/lang/python/wiredtiger/fpacking.py +++ b/src/third_party/wiredtiger/lang/python/wiredtiger/fpacking.py @@ -30,6 +30,7 @@ # struct library. import struct +from .packing import empty_pack def __wt2struct(fmt): if not fmt: @@ -88,7 +89,7 @@ def unpack(fmt, s): def pack(fmt, *values): pfmt, fmt = __wt2struct(fmt) if not fmt: - return '' + return empty_pack i = sizebytes = 0 for offset, f in enumerate(fmt): if f == 'S': diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py b/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py index baaae58c168..27e756eb87b 100644..100755 --- a/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py +++ b/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py @@ -27,7 +27,9 @@ # OTHER DEALINGS IN THE SOFTWARE. # -import math, struct +from __future__ import print_function +from .packing import _chr, _ord +import math, struct, sys # Variable-length integer packing # need: up to 64 bits, both signed and unsigned @@ -62,58 +64,66 @@ POS_2BYTE_MAX = 2**13 + POS_1BYTE_MAX MINUS_BIT = -1 << 64 UINT64_MASK = 0xffffffffffffffff +_python3 = (sys.version_info >= (3, 0, 0)) +if _python3: + xff_entry = 0xff + x00_entry = 0x00 +else: + xff_entry = '\xff' + x00_entry = '\x00' + def getbits(x, start, end=0): '''return the least significant bits of x, from start to end''' return (x & ((1 << start) - 1)) >> (end) def get_int(b, size): r = 0 - for i in xrange(size): - r = (r << 8) | ord(b[i]) + for i in range(size): + r = (r << 8) | _ord(b[i]) return r def pack_int(x): if x < NEG_2BYTE_MIN: packed = struct.pack('>Q', x & UINT64_MASK) - while packed and packed[0] == '\xff': + while packed and packed[0] == xff_entry: packed = packed[1:] - return chr(NEG_MULTI_MARKER | getbits(8 - len(packed), 4)) + packed + return _chr(NEG_MULTI_MARKER | getbits(8 - len(packed), 4)) + packed elif x < NEG_1BYTE_MIN: x -= NEG_2BYTE_MIN - return chr(NEG_2BYTE_MARKER | getbits(x, 13, 8)) + chr(getbits(x, 8)) + return _chr(NEG_2BYTE_MARKER | getbits(x, 13, 8), getbits(x, 8)) elif x < 0: x -= NEG_1BYTE_MIN - return chr(NEG_1BYTE_MARKER | getbits(x, 6)) + return _chr(NEG_1BYTE_MARKER | getbits(x, 6)) elif x <= POS_1BYTE_MAX: - return chr(POS_1BYTE_MARKER | getbits(x, 6)) + return _chr(POS_1BYTE_MARKER | getbits(x, 6)) elif x <= POS_2BYTE_MAX: x -= (POS_1BYTE_MAX + 1) - return chr(POS_2BYTE_MARKER | getbits(x, 13, 8)) + chr(getbits(x, 8)) + return _chr(POS_2BYTE_MARKER | getbits(x, 13, 8), getbits(x, 8)) elif x == POS_2BYTE_MAX + 1: # This is a special case where we could store the value with # just a single byte, but we append a zero byte so that the # encoding doesn't get shorter for this one value. - return chr(POS_MULTI_MARKER | 0x1) + chr(0) + return _chr(POS_MULTI_MARKER | 0x1, 0) else: packed = struct.pack('>Q', x - (POS_2BYTE_MAX + 1)) - while packed and packed[0] == '\x00': + while packed and packed[0] == x00_entry: packed = packed[1:] - return chr(POS_MULTI_MARKER | getbits(len(packed), 4)) + packed + return _chr(POS_MULTI_MARKER | getbits(len(packed), 4)) + packed def unpack_int(b): - marker = ord(b[0]) + marker = _ord(b[0]) if marker < NEG_2BYTE_MARKER: sz = 8 - getbits(marker, 4) return ((-1 << (sz << 3)) | get_int(b[1:], sz), b[sz+1:]) elif marker < NEG_1BYTE_MARKER: - return (NEG_2BYTE_MIN + ((getbits(marker, 5) << 8) | ord(b[1])), b[2:]) + return (NEG_2BYTE_MIN + ((getbits(marker, 5) << 8) | _ord(b[1])), b[2:]) elif marker < POS_1BYTE_MARKER: return (NEG_1BYTE_MIN + getbits(marker, 6), b[1:]) elif marker < POS_2BYTE_MARKER: return (getbits(marker, 6), b[1:]) elif marker < POS_MULTI_MARKER: return (POS_1BYTE_MAX + 1 + - ((getbits(marker, 5) << 8) | ord(b[1])), b[2:]) + ((getbits(marker, 5) << 8) | _ord(b[1])), b[2:]) else: sz = getbits(marker, 4) return (POS_2BYTE_MAX + 1 + get_int(b[1:], sz), b[sz+1:]) @@ -123,21 +133,21 @@ if __name__ == '__main__': import random for big in (100, 10000, 1 << 40, 1 << 64): - for i in xrange(1000): + for i in range(1000): r = random.randint(-big, big) - print "\rChecking %d" % r, + print("\rChecking %d" % r) if unpack_int(pack_int(r))[0] != r: - print "\nFound a problem with %d" % r + print("\nFound a problem with %d" % r) break print - for i in xrange(1000): + for i in range(1000): r1 = random.randint(-big, big) r2 = random.randint(-big, big) - print "\rChecking %d, %d" % (r1, r2), + print("\rChecking %d, %d" % (r1, r2)) if cmp(r1, r2) != cmp(pack_int(r1), pack_int(r2)): - print "\nFound a problem with %d, %d" % (r1, r2) + print("\nFound a problem with %d, %d" % (r1, r2)) break print diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py b/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py index 438d9c15a08..c1c5a9bc6a3 100644..100755 --- a/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py +++ b/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py @@ -49,7 +49,50 @@ Format Python Notes u str raw byte array """ -from intpacking import pack_int, unpack_int +import sys + +# In the Python3 world, we pack into the bytes type, which like a list of ints. +# In Python2, we pack into a string. Create a set of constants and methods +# to hide the differences from the main code. +if sys.version_info[0] >= 3: + x00 = b'\x00' + xff = b'\xff' + empty_pack = b'' + def _ord(b): + return b + + def _chr(x, y=None): + a = [x] + if y != None: + a.append(y) + return bytes(a) + + def _is_string(s): + return type(s) is str + + def _string_result(s): + return s.decode() + +else: + x00 = '\x00' + xff = '\xff' + empty_pack = '' + def _ord(b): + return ord(b) + + def _chr(x, y=None): + s = chr(x) + if y != None: + s += chr(y) + return s + + def _is_string(s): + return type(s) is unicode + + def _string_result(s): + return s + +from .intpacking import pack_int, unpack_int def __get_type(fmt): if not fmt: @@ -92,36 +135,39 @@ def unpack(fmt, s): if f == 's': pass elif f == 'S': - size = s.find('\0') + size = s.find(x00) elif f == 'u' and offset == len(fmt) - 1: # A WT_ITEM with a NULL data field will be appear as None. if s == None: - s = '' + s = empty_pack size = len(s) else: # Note: 'U' is used internally, and may be exposed to us. # It indicates that the size is always stored unless there # is a size in the format. size, s = unpack_int(s) - result.append(s[:size]) - if f == 'S' and not havesize: - size += 1 + if f in 'Ss': + result.append(_string_result(s[:size])) + if f == 'S' and not havesize: + size += 1 + else: + result.append(s[:size]) s = s[size:] elif f in 't': # bit type, size is number of bits - result.append(ord(s[0:1])) + result.append(_ord(s[0])) s = s[1:] elif f in 'Bb': # byte type - for i in xrange(size): - v = ord(s[0:1]) + for i in range(size): + v = _ord(s[0]) if f != 'B': v -= 0x80 result.append(v) s = s[1:] else: # integral type - for j in xrange(size): + for j in range(size): v, s = unpack_int(s) result.append(v) return result @@ -136,7 +182,7 @@ def __pack_iter_fmt(fmt, values): index += 1 else: # integral type size = size if havesize else 1 - for i in xrange(size): + for i in range(size): value = values[index] yield offset, havesize, 1, char, value index = index + 1 @@ -147,14 +193,14 @@ def pack(fmt, *values): return () if tfmt != '.': raise ValueError('Only variable-length encoding is currently supported') - result = '' + result = empty_pack i = 0 for offset, havesize, size, f, val in __pack_iter_fmt(fmt, values): if f == 'x': if not havesize: - result += '\0' + result += x00 else: - result += '\0' * size + result += x00 * size # Note: no value, don't increment i elif f in 'SsUu': if f == 'S' and '\0' in val: @@ -166,14 +212,14 @@ def pack(fmt, *values): l = size elif (f == 'u' and offset != len(fmt) - 1) or f == 'U': result += pack_int(l) - if type(val) is unicode and f in 'Ss': - result += str(val[:l]) + if _is_string(val) and f in 'Ss': + result += str(val[:l]).encode() else: result += val[:l] if f == 'S' and not havesize: - result += '\0' + result += x00 elif size > l and havesize: - result += '\0' * (size - l) + result += x00 * (size - l) elif f in 't': # bit type, size is number of bits if not havesize: @@ -183,12 +229,12 @@ def pack(fmt, *values): mask = (1 << size) - 1 if (mask & val) != val: raise ValueError("value out of range for 't' encoding") - result += chr(val) + result += _chr(val) elif f in 'Bb': # byte type if not havesize: size = 1 - for i in xrange(size): + for i in range(size): if f == 'B': v = val else: @@ -196,7 +242,7 @@ def pack(fmt, *values): v = val + 0x80 if v > 255 or v < 0: raise ValueError("value out of range for 'B' encoding") - result += chr(v) + result += _chr(v) else: # integral type result += pack_int(val) |