diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2019-04-16 14:36:51 +1000 |
|---|---|---|
| committer | Luke Chen <luke.chen@mongodb.com> | 2019-04-16 14:45:56 +1000 |
| commit | a654dcf592ea7ed65426a0de96b4079ff4fc6716 (patch) | |
| tree | a5256edad1bb219e6af72fd7e7525f58e235a307 | |
| parent | 19b622ebfb42a525f38e278c09f440eb47b12f1e (diff) | |
| download | mongo-a654dcf592ea7ed65426a0de96b4079ff4fc6716.tar.gz | |
Import wiredtiger: 9416282c42d40328dfb7ff0f28831f639f98d3cb from branch mongodb-4.2
ref: 1768d66613..9416282c42
for: 4.1.11
WT-4317 Read checksum error in test_wt4156_metadata_salvage
WT-4579 Track the newest durable timestamp for each page
WT-4585 Add WT_WITH_HOTBACKUP_LOCK macro
WT-4598 Enable the assertion that the durable_timestamp is newer than or equals the commit timestamp.
WT-4640 Remove round_to_oldest in favour of roundup_timestamps
WT-4695 Python3: allow most tests to run with Python3 with small changes
WT-4696 Python3: change dist scripts to run under Python3
WT-4698 Python3: fix modify related tests
WT-4699 Python3: fix test_jsondump02.py
WT-4700 Python3: run with same source as Python2
WT-4703 Extend test/checkpoint to do removes and online checking
WT-4704 Add statistic tracking oldest active read timestamp
WT-4705 column-store no longer needs to handle WT_COL page offsets of 0
WT-4707 Failure in verifying cells with copied values
WT-4708 Coverity reported copy-paste error in WiredTiger error message
WT-4711 Python formatting errors reported while running "s_all"
WT-4714 Use the durable timestamp to determine if a page should stay dirty
WT-4724 Syntax error in wtperf_ckpt.sh when running 'dash' as default shell
171 files changed, 2166 insertions, 1490 deletions
diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index 2646d51378e..20916ac8da0 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -325,6 +325,7 @@ if GetOption("lang-python"): print "The Python Interpreter must be 64-bit in order to build the python bindings" Exit(1) + pythonMajorVersion = sys.version_info.major pythonEnv = env.Clone() pythonEnv.Append(SWIGFLAGS=[ "-python", @@ -332,6 +333,7 @@ if GetOption("lang-python"): "-O", "-nodefaultctor", "-nodefaultdtor", + "-DPY_MAJOR_VERSION=" + str(pythonMajorVersion) ]) # Ignore warnings in swig-generated code. pythonEnv['CFLAGS'].remove("/WX") diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh b/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh index 86793b6bcbd..881ca7c7865 100755 --- a/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh +++ b/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # wtperf_run.sh - run wtperf regression tests on the Jenkins platform. # diff --git a/src/third_party/wiredtiger/dist/api_config.py b/src/third_party/wiredtiger/dist/api_config.py index 78f4f4fa91c..2b13097f108 100644..100755 --- a/src/third_party/wiredtiger/dist/api_config.py +++ b/src/third_party/wiredtiger/dist/api_config.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +from __future__ import print_function import os, re, sys, textwrap import api_data from dist import compare_srcfile @@ -110,7 +111,7 @@ for line in open(f, 'r'): prefix, config_name = m.groups() if config_name not in api_data.methods: - print >>sys.stderr, "Missing configuration for " + config_name + print("Missing configuration for " + config_name, file=sys.stderr) tfile.write(line) continue @@ -261,7 +262,7 @@ for name in sorted(api_data.methods.keys()): # #defines are used to avoid a list search where we know the correct slot). config_defines +=\ '#define\tWT_CONFIG_ENTRY_' + name.replace('.', '_') + '\t' * \ - max(1, 6 - (len('WT_CONFIG_ENTRY_' + name) / 8)) + \ + max(1, 6 - (len('WT_CONFIG_ENTRY_' + name) // 8)) + \ "%2s" % str(slot) + '\n' # Write the method name and base. diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index ba480ba5cb9..39f49fc50e9 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -23,8 +23,24 @@ class Config: self.subconfig = subconfig self.flags = flags - def __cmp__(self, other): - return cmp(self.name, other.name) + # Comparators for sorting. + def __eq__(self, other): + return self.name == other.name + + def __ne__(self, other): + return self.name != other.name + + def __lt__(self, other): + return self.name < other.name + + def __le__(self, other): + return self.name <= other.name + + def __gt__(self, other): + return self.name > other.name + + def __ge__(self, other): + return self.name >= other.name common_runtime_config = [ Config('app_metadata', '', r''' @@ -1282,10 +1298,6 @@ methods = { read using the specified timestamp. The supplied value must not be older than the current oldest timestamp. See @ref transaction_timestamps'''), - Config('round_to_oldest', 'false', r''' - if read timestamp is earlier than oldest timestamp, - read timestamp will be rounded to oldest timestamp''', - type='boolean'), Config('roundup_timestamps', '', r''' round up timestamps of the transaction. This setting alters the visibility expected in a transaction. See @ref @@ -1365,10 +1377,6 @@ methods = { read using the specified timestamp. The supplied value must not be older than the current oldest timestamp. This can only be set once for a transaction. See @ref transaction_timestamps'''), - Config('round_to_oldest', 'false', r''' - if read timestamp is earlier than oldest timestamp, - read timestamp will be rounded to oldest timestamp''', - type='boolean'), ]), 'WT_SESSION.rollback_transaction' : Method([]), diff --git a/src/third_party/wiredtiger/dist/dist.py b/src/third_party/wiredtiger/dist/dist.py index bd3f1b2b7e9..987b43d0098 100644 --- a/src/third_party/wiredtiger/dist/dist.py +++ b/src/third_party/wiredtiger/dist/dist.py @@ -1,3 +1,4 @@ +from __future__ import print_function import filecmp, fnmatch, glob, os, re, shutil # source_files -- @@ -50,12 +51,12 @@ def source_dirs(): def print_source_dirs(): for d in source_dirs(): - print d + print(d) # compare_srcfile -- # Compare two files, and if they differ, update the source file. def compare_srcfile(tmp, src): if not os.path.isfile(src) or not filecmp.cmp(tmp, src, shallow=False): - print('Updating ' + src) + print(('Updating ' + src)) shutil.copyfile(tmp, src) os.remove(tmp) diff --git a/src/third_party/wiredtiger/dist/flags.py b/src/third_party/wiredtiger/dist/flags.py index 0380cde1781..0f960e3f8b7 100644..100755 --- a/src/third_party/wiredtiger/dist/flags.py +++ b/src/third_party/wiredtiger/dist/flags.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +from __future__ import print_function import re, sys from dist import all_c_files, all_h_files, compare_srcfile @@ -26,8 +27,8 @@ def flag_declare(name): elif line.find('AUTOMATIC FLAG VALUE GENERATION STOP') != -1: # We only support 64 bits. if len(defines) > 64: - print >>sys.stderr, name + ": line " +\ - str(lcnt) + ": exceeds maximum 64 bit flags" + print(name + ": line " + str(lcnt) +\ + ": exceeds maximum 64 bit flags", file=sys.stderr) sys.exit(1) # Calculate number of hex bytes, create format string @@ -42,8 +43,8 @@ def flag_declare(name): parsing = False elif parsing and line.find('#define') == -1: - print >>sys.stderr, name + ": line " +\ - str(lcnt) + ": unexpected flag line, no #define" + print(name + ": line " + str(lcnt) +\ + ": unexpected flag line, no #define", file=sys.stderr) sys.exit(1) elif parsing: defines.append(line) diff --git a/src/third_party/wiredtiger/dist/function.py b/src/third_party/wiredtiger/dist/function.py index 22c1d2928b9..fb4bb532f90 100644 --- a/src/third_party/wiredtiger/dist/function.py +++ b/src/third_party/wiredtiger/dist/function.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # Check the style of WiredTiger C code. +from __future__ import print_function import fnmatch, os, re, sys from dist import all_c_files, compare_srcfile, source_files @@ -16,8 +17,8 @@ def missing_comment(): for m in func_re.finditer(s): if not m.group(1) or \ not m.group(1).startswith('/*\n * %s --\n' % m.group(2)): - print "%s:%d: missing or malformed comment for %s" % \ - (f, s[:m.start(2)].count('\n'), m.group(2)) + print("%s:%d: missing or malformed comment for %s" % \ + (f, s[:m.start(2)].count('\n'), m.group(2))) # Sort helper function, discard * operators so a pointer doesn't necessarily # sort before non-pointers, ignore const/static/volatile keywords. @@ -105,7 +106,7 @@ def function_args(name, line): # Check for illegal types. for m in illegal_types: if re.search('^' + m + "\s*[\w(*]", line): - print >>sys.stderr, name + ": illegal type: " + line.strip() + print(name + ": illegal type: " + line.strip(), file=sys.stderr) sys.exit(1) # Check for matching types. @@ -152,8 +153,8 @@ def function_declaration(): # initializers (and we've already skipped statics, which # are also typically initialized in the declaration). if re.search("\s=\s[-\w]", line): - print >>sys.stderr, \ - name + ": assignment in string: " + line.strip() + print(name + ": assignment in string: " + line.strip(),\ + file=sys.stderr) sys.exit(1); list[n].append(line) diff --git a/src/third_party/wiredtiger/dist/java_doc.py b/src/third_party/wiredtiger/dist/java_doc.py index 71dfd93d3a8..b08bad42bd4 100644 --- a/src/third_party/wiredtiger/dist/java_doc.py +++ b/src/third_party/wiredtiger/dist/java_doc.py @@ -3,6 +3,7 @@ # This program pulls the function names from wiredtiger.in and generates # an input file for Java SWIG that adds doxygen copydoc comments to functions. +from __future__ import print_function import os, re, sys import api_data from dist import compare_srcfile diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index d585c1e268d..e82ef2ad31f 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -47,6 +47,9 @@ WT_PREPARE_INIT WT_READ_BARRIER WT_REF_SIZE WT_SESSION_LOCKED_CHECKPOINT +WT_SESSION_LOCKED_HOTBACKUP +WT_SESSION_LOCKED_HOTBACKUP_READ +WT_SESSION_LOCKED_HOTBACKUP_WRITE WT_SESSION_LOCKED_TABLE_READ WT_SESSION_LOCKED_TABLE_WRITE WT_SESSION_LOCKED_TURTLE diff --git a/src/third_party/wiredtiger/dist/s_docs b/src/third_party/wiredtiger/dist/s_docs index 6ebffb947ec..d23e37ed9fa 100755 --- a/src/third_party/wiredtiger/dist/s_docs +++ b/src/third_party/wiredtiger/dist/s_docs @@ -146,7 +146,7 @@ EOF } # Add optional extras - EXTRAS="../lang/java/src/com/wiredtiger/db ../lang/python/wiredtiger.py" + EXTRAS="../lang/java/src/com/wiredtiger/db" EXTRA_INPUT="" for f in $EXTRAS ; do [ -e "$f" ] && EXTRA_INPUT="$EXTRA_INPUT ../$f" diff --git a/src/third_party/wiredtiger/dist/s_python b/src/third_party/wiredtiger/dist/s_python index bfe3ba57783..7ecb97059b5 100755 --- a/src/third_party/wiredtiger/dist/s_python +++ b/src/third_party/wiredtiger/dist/s_python @@ -10,6 +10,7 @@ cd .. egrep ' ' `find . -name '*.py'` | sed -e 's/:.*//' \ -e '/__init__.py/d' \ + -e '/\/wiredtiger.py/d' \ -e '/src\/docs\/tools\/doxypy.py/d' | sort -u | sed 's/^/ /' > $t diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index bc265efb34f..c1781aa8425 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -172,6 +172,7 @@ HHHH HHHHLL HHHLL HILQr +HOTBACKUP Hendrik HyperLevelDB ID's @@ -394,6 +395,7 @@ UIDs UINT ULINE UNC +UNCOND URI URIs UTF @@ -1197,6 +1199,7 @@ sizep sizev skiplist skiplists +skipp slotsp slvg snaplen diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 4fda3e681b7..2f056cb81dd 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -559,6 +559,7 @@ connection_stats = [ TxnStat('txn_pinned_timestamp', 'transaction range of timestamps currently pinned', 'no_clear,no_scale'), TxnStat('txn_pinned_timestamp_checkpoint', 'transaction range of timestamps pinned by a checkpoint', 'no_clear,no_scale'), TxnStat('txn_pinned_timestamp_oldest', 'transaction range of timestamps pinned by the oldest timestamp', 'no_clear,no_scale'), + TxnStat('txn_pinned_timestamp_reader', 'transaction range of timestamps pinned by the oldest active read timestamp', 'no_clear,no_scale'), TxnStat('txn_prepare', 'prepared transactions'), TxnStat('txn_prepare_active', 'prepared transactions currently active'), TxnStat('txn_prepare_commit', 'prepared transactions committed'), @@ -586,6 +587,7 @@ connection_stats = [ TxnStat('txn_snapshots_created', 'number of named snapshots created'), TxnStat('txn_snapshots_dropped', 'number of named snapshots dropped'), TxnStat('txn_sync', 'transaction sync calls'), + TxnStat('txn_timestamp_oldest_active_read', 'transaction read timestamp of the oldest active reader', 'no_clear,no_scale'), TxnStat('txn_update_conflict', 'update conflicts'), ########################################## diff --git a/src/third_party/wiredtiger/dist/style.py b/src/third_party/wiredtiger/dist/style.py index 9336333cb67..bfd1694cbbc 100755 --- a/src/third_party/wiredtiger/dist/style.py +++ b/src/third_party/wiredtiger/dist/style.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # Check the style of WiredTiger C code. +from __future__ import print_function import re, sys from dist import source_files @@ -16,9 +17,9 @@ def lines_could_join(): for m in match_re.finditer(s): if len(m.group(1).expandtabs()) + \ len(m.group(2).expandtabs()) < 80: - print f + ': lines may be combined: ' - print '\t' + m.group(1).lstrip() + m.group(2) - print + print(f + ': lines may be combined: ') + print('\t' + m.group(1).lstrip() + m.group(2)) + print() # Don't display lines that could be joined by default; in some cases, the code # isn't maintained by WiredTiger, or the line splitting enhances readability. diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 1738efa6d8a..96009a78193 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "1768d66613fc32b664ac3608a4e740d5e8c6fd0f", + "commit": "9416282c42d40328dfb7ff0f28831f639f98d3cb", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.2" diff --git a/src/third_party/wiredtiger/lang/python/Makefile.am b/src/third_party/wiredtiger/lang/python/Makefile.am index d7539c35bdd..3f29ea6aeea 100644 --- a/src/third_party/wiredtiger/lang/python/Makefile.am +++ b/src/third_party/wiredtiger/lang/python/Makefile.am @@ -1,18 +1,33 @@ PYSRC = $(top_srcdir)/lang/python PYDIRS = -t $(abs_builddir) -I $(abs_top_srcdir):$(abs_top_builddir) -L $(abs_top_builddir)/.libs -all-local: _wiredtiger.so +PYDST = $(abs_builddir)/wiredtiger +PYFILES = $(PYDST)/fpacking.py $(PYDST)/intpacking.py $(PYDST)/packing.py \ + $(PYDST)/__init__.py +PY_MAJOR_VERSION := $$($(PYTHON) -c \ + 'import sys; print(int(sys.version_info.major))') + +all-local: _wiredtiger.so pyfiles # We keep generated Python sources under lang/python: that's where they live # in release packages. -$(PYSRC)/wiredtiger_wrap.c: $(top_srcdir)/src/include/wiredtiger.in $(PYSRC)/wiredtiger.i +$(PYSRC)/wiredtiger_wrap.c $(PYSRC)/wiredtiger.py: $(top_srcdir)/src/include/wiredtiger.in $(PYSRC)/wiredtiger.i (cd $(PYSRC) && \ - $(SWIG) -python -threads -O -Wall -nodefaultctor -nodefaultdtor -I$(abs_top_builddir) wiredtiger.i && \ - mv wiredtiger.py wiredtiger/__init__.py) + $(SWIG) -python -DPY_MAJOR_VERSION=$(PY_MAJOR_VERSION) \ + -threads -O -Wall -nodefaultctor -nodefaultdtor \ + -I$(abs_top_builddir) wiredtiger.i) _wiredtiger.so: $(top_builddir)/libwiredtiger.la $(PYSRC)/wiredtiger_wrap.c (cd $(PYSRC) && \ $(PYTHON) setup.py build_ext -f -b $(abs_builddir) $(PYDIRS)) +pyfiles: $(PYFILES) + +$(PYDST)/%: $(PYSRC)/wiredtiger/% + mkdir -p $(PYDST) && cp -f $< $@ + +$(PYDST)/__init__.py: $(PYSRC)/wiredtiger.py + mkdir -p $(PYDST) && cp -f $< $@ + install-exec-local: (cd $(PYSRC) && \ $(PYTHON) setup.py build_py -d $(abs_builddir)/build && \ diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger.i b/src/third_party/wiredtiger/lang/python/wiredtiger.i index 7a9af32a9d1..86182b1716c 100644 --- a/src/third_party/wiredtiger/lang/python/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/python/wiredtiger.i @@ -30,6 +30,8 @@ * wiredtiger.i * The SWIG interface file defining the wiredtiger python API. */ +%include <pybuffer.i> + %define DOCSTRING "Python wrappers around the WiredTiger C API @@ -48,7 +50,7 @@ This provides an API similar to the C API, with the following modifications: %feature("autodoc", "0"); %pythoncode %{ -from packing import pack, unpack +from .packing import pack, unpack ## @endcond %} @@ -161,10 +163,11 @@ from packing import pack, unpack %typemap(argout) (WT_MODIFY *entries, int *nentriesp) { int i; + $result = PyList_New(*$2); for (i = 0; i < *$2; i++) { PyObject *o = SWIG_NewPointerObj(Py_None, SWIGTYPE_p___wt_modify, 0); - PyObject_SetAttrString(o, "data", PyString_FromStringAndSize( + PyObject_SetAttrString(o, "data", PyBytes_FromStringAndSize( $1[i].data.data, $1[i].data.size)); PyObject_SetAttrString(o, "offset", PyInt_FromLong($1[i].offset)); @@ -174,15 +177,30 @@ from packing import pack, unpack } } -%typemap(in) const WT_ITEM * (WT_ITEM val, long sz) { - if (PyString_AsStringAndSize($input, &val.data, &sz) < 0) +%typemap(argout) (WT_MODIFY *entries_string, int *nentriesp) { + int i; + + $result = PyList_New(*$2); + for (i = 0; i < *$2; i++) { + PyObject *o = SWIG_NewPointerObj(Py_None, SWIGTYPE_p___wt_modify, 0); + PyObject_SetAttrString(o, "data", PyUnicode_FromStringAndSize( + $1[i].data.data, $1[i].data.size)); + PyObject_SetAttrString(o, "offset", + PyInt_FromLong($1[i].offset)); + PyObject_SetAttrString(o, "size", + PyInt_FromLong($1[i].size)); + PyList_SetItem($result, i, o); + } + } + +%typemap(in) const WT_ITEM * (WT_ITEM val) { + if (unpackBytesOrString($input, &val.data, &val.size) != 0) SWIG_exception_fail(SWIG_AttributeError, "bad string value for WT_ITEM"); - val.size = (size_t)sz; $1 = &val; } -%typemap(freearg) (WT_MODIFY *entries, int *nentriesp) { +%typemap(freearg) (WT_MODIFY *, int *nentriesp) { __wt_free(NULL, $1); } @@ -198,23 +216,32 @@ from packing import pack, unpack modarray[0].size = (size_t)len; for (i = 1; i <= len; i++) { PyObject *dataobj, *modobj, *offsetobj, *sizeobj; - char *datadata; + void *datadata; long offset, size; - Py_ssize_t datasize; + size_t datasize; - if ((modobj = PySequence_GetItem($input, i - 1)) == NULL) + if ((modobj = PySequence_GetItem($input, i - 1)) == NULL) { + freeModifyArray(modarray); SWIG_exception_fail(SWIG_IndexError, "Modify sequence failed"); + } WT_GETATTR(dataobj, modobj, "data"); - if (PyString_AsStringAndSize(dataobj, &datadata, - &datasize) < 0) { + if (unpackBytesOrString(dataobj, &datadata, &datasize) != 0) { Py_DECREF(dataobj); Py_DECREF(modobj); + freeModifyArray(modarray); SWIG_exception_fail(SWIG_AttributeError, "Modify.data bad value"); } - modarray[i].data.data = malloc(datasize); + if (datasize != 0 && + __wt_malloc(NULL, datasize, &modarray[i].data.data) != 0) { + Py_DECREF(dataobj); + Py_DECREF(modobj); + freeModifyArray(modarray); + SWIG_exception_fail(SWIG_AttributeError, + "Modify.data failed malloc"); + } memcpy(modarray[i].data.data, datadata, datasize); modarray[i].data.size = datasize; Py_DECREF(dataobj); @@ -223,6 +250,7 @@ from packing import pack, unpack if ((offset = PyInt_AsLong(offsetobj)) < 0) { Py_DECREF(offsetobj); Py_DECREF(modobj); + freeModifyArray(modarray); SWIG_exception_fail(SWIG_RuntimeError, "Modify.offset bad value"); } @@ -233,6 +261,7 @@ from packing import pack, unpack if ((size = PyInt_AsLong(sizeobj)) < 0) { Py_DECREF(sizeobj); Py_DECREF(modobj); + freeModifyArray(modarray); SWIG_exception_fail(SWIG_RuntimeError, "Modify.size bad value"); } @@ -244,14 +273,7 @@ from packing import pack, unpack } %typemap(freearg) WT_MODIFY * { - /* The WT_MODIFY arg is in position 2. Is there a better way? */ - WT_MODIFY *modarray = modarray2; - size_t i, len; - - len = modarray[0].size; - for (i = 1; i <= len; i++) - __wt_free(NULL, modarray[i].data.data); - __wt_free(NULL, modarray); + freeModifyArray($1); } /* 64 bit typemaps. */ @@ -262,6 +284,9 @@ from packing import pack, unpack $result = PyLong_FromUnsignedLongLong($1); } +/* Internal _set_key, _set_value methods take a 'bytes' object as parameter. */ +%pybuffer_binary(void *data, int); + /* Throw away references after close. */ %define DESTRUCTOR(class, method) %feature("shadow") class::method %{ @@ -347,6 +372,7 @@ static PyObject *wtError; static int sessionFreeHandler(WT_SESSION *session_arg); static int cursorFreeHandler(WT_CURSOR *cursor_arg); +static int unpackBytesOrString(PyObject *obj, void **data, size_t *size); #define WT_GETATTR(var, parent, name) \ do if ((var = PyObject_GetAttrString(parent, name)) == NULL) { \ @@ -372,6 +398,15 @@ static int cursorFreeHandler(WT_CURSOR *cursor_arg); %pythoncode %{ WiredTigerError = _wiredtiger.WiredTigerError +# Python3 has no explicit long type, recnos work as ints +import sys +if sys.version_info >= (3, 0, 0): + def _wt_recno(i): + return i +else: + def _wt_recno(i): + return long(i) + ## @cond DISABLE # Implements the iterable contract class IterableCursor: @@ -381,10 +416,13 @@ class IterableCursor: def __iter__(self): return self - def next(self): + def __next__(self): if self.cursor.next() == WT_NOTFOUND: raise StopIteration return self.cursor.get_keys() + self.cursor.get_values() + + def next(self): + return self.__next__() ## @endcond # An abstract class, which must be subclassed with notify() overridden. @@ -395,6 +433,12 @@ class AsyncCallback: def notify(self, op, op_ret, flags): raise NotImplementedError +def wiredtiger_calc_modify(session, oldv, newv, maxdiff, nmod): + return _wiredtiger_calc_modify(session, oldv, newv, maxdiff, nmod) + +def wiredtiger_calc_modify_string(session, oldv, newv, maxdiff, nmod): + return _wiredtiger_calc_modify_string(session, oldv, newv, maxdiff, nmod) + %} /* Bail out if arg or arg.this is None, else set res to the C pointer. */ @@ -576,16 +620,26 @@ OVERRIDE_METHOD(__wt_cursor, WT_CURSOR, equals, (self, other)) OVERRIDE_METHOD(__wt_cursor, WT_CURSOR, search_near, (self)) /* SWIG magic to turn Python byte strings into data / size. */ -%apply (char *STRING, int LENGTH) { (char *data, int size) }; +#if PY_MAJOR_VERSION >= 3 + %apply (char *STRING, int LENGTH) { (char *data, int size) }; +#else +%apply (char *STRING, int LENGTH) { (void *data, int size) }; +#endif /* Handle binary data returns from get_key/value -- avoid cstring.i: it creates a list of returns. */ %typemap(in,numinputs=0) (char **datap, int *sizep) (char *data, int size) { $1 = &data; $2 = &size; } +%typemap(in,numinputs=0) (char **charp, int *sizep) (char *data, int size) { $1 = &data; $2 = &size; } %typemap(frearg) (char **datap, int *sizep) ""; -%typemap(argout) (char **datap, int *sizep) { +%typemap(argout) (char **charp, int *sizep) { if (*$1) - $result = SWIG_FromCharPtrAndSize(*$1, *$2); + $result = PyUnicode_FromStringAndSize(*$1, *$2); } +%typemap(argout) (char **datap, int *sizep) { + if (*$1) + $result = PyBytes_FromStringAndSize(*$1, *$2); + } + /* Handle record number returns from get_recno */ %typemap(in,numinputs=0) (uint64_t *recnop) (uint64_t recno) { $1 = &recno; } %typemap(frearg) (uint64_t *recnop) ""; @@ -606,7 +660,7 @@ typedef int int_void; %extend __wt_async_op { /* Get / set keys and values */ - void _set_key(char *data, int size) { + void _set_key(void *data, int size) { WT_ITEM k; k.data = data; k.size = (uint32_t)size; @@ -630,7 +684,7 @@ typedef int int_void; return (ret); } - void _set_value(char *data, int size) { + void _set_value(void *data, int size) { WT_ITEM v; v.data = data; v.size = (uint32_t)size; @@ -714,7 +768,7 @@ typedef int int_void; if len(args) == 1 and type(args[0]) == tuple: args = args[0] if self.is_column: - self._set_recno(long(args[0])) + self._set_recno(_wt_recno(args[0])) else: # Keep the Python string pinned self._key = pack(self.key_format, *args) @@ -747,7 +801,7 @@ typedef int int_void; %extend __wt_cursor { /* Get / set keys and values */ - void _set_key(char *data, int size) { + void _set_key(void *data, int size) { WT_ITEM k; k.data = data; k.size = (uint32_t)size; @@ -776,7 +830,7 @@ typedef int int_void; return (ret); } - void _set_value(char *data, int size) { + void _set_value(void *data, int size) { WT_ITEM v; v.data = data; v.size = (uint32_t)size; @@ -799,11 +853,11 @@ typedef int int_void; return (ret); } - int_void _get_json_key(char **datap, int *sizep) { + int_void _get_json_key(char **charp, int *sizep) { const char *k; int ret = $self->get_key($self, &k); if (ret == 0) { - *datap = (char *)k; + *charp = (char *)k; *sizep = strlen(k); } return (ret); @@ -828,11 +882,11 @@ typedef int int_void; return (ret); } - int_void _get_json_value(char **datap, int *sizep) { + int_void _get_json_value(char **charp, int *sizep) { const char *k; int ret = $self->get_value($self, &k); if (ret == 0) { - *datap = (char *)k; + *charp = (char *)k; *sizep = strlen(k); } return (ret); @@ -953,7 +1007,7 @@ typedef int int_void; if len(args) == 1 and type(args[0]) == tuple: args = args[0] if self.is_column: - self._set_recno(long(args[0])) + self._set_recno(_wt_recno(args[0])) elif self.is_json: self._set_key_str(args[0]) else: @@ -1086,9 +1140,11 @@ OVERRIDE_METHOD(__wt_session, WT_SESSION, log_printf, (self, msg)) %ignore wiredtiger_struct_size; %ignore wiredtiger_struct_unpack; +%ignore wiredtiger_calc_modify; %ignore wiredtiger_extension_init; %ignore wiredtiger_extension_terminate; + /* Convert 'int *' to output args for wiredtiger_version */ %apply int *OUTPUT { int * }; @@ -1100,8 +1156,67 @@ OVERRIDE_METHOD(__wt_session, WT_SESSION, log_printf, (self, msg)) %include "wiredtiger.h" -/* Add event handler support. */ +/* + * The original wiredtiger_calc_modify was ignored, now we define our own. + * Python needs to know whether to return a bytes object or a string. + * Part of the smarts to do that is the output typemap, which matches on + * the naming of the parameter: entries vs. entries_string + */ +extern int _wiredtiger_calc_modify(WT_SESSION *session, + const WT_ITEM *oldv, const WT_ITEM *newv, + size_t maxdiff, WT_MODIFY *entries, int *nentriesp); +extern int _wiredtiger_calc_modify_string(WT_SESSION *session, + const WT_ITEM *oldv, const WT_ITEM *newv, + size_t maxdiff, WT_MODIFY *entries_string, int *nentriesp); %{ +int _wiredtiger_calc_modify(WT_SESSION *session, + const WT_ITEM *oldv, const WT_ITEM *newv, + size_t maxdiff, WT_MODIFY *entries, int *nentriesp) +{ + return (wiredtiger_calc_modify( + session, oldv, newv, maxdiff, entries, nentriesp)); +} + +int _wiredtiger_calc_modify_string(WT_SESSION *session, + const WT_ITEM *oldv, const WT_ITEM *newv, + size_t maxdiff, WT_MODIFY *entries_string, int *nentriesp) +{ + return (wiredtiger_calc_modify( + session, oldv, newv, maxdiff, entries_string, nentriesp)); +} + +/* Add event handler support. */ + +static void +freeModifyArray(WT_MODIFY *modarray) +{ + size_t i, len; + + len = modarray[0].size; + for (i = 1; i <= len; i++) + __wt_free(NULL, modarray[i].data.data); + __wt_free(NULL, modarray); +} + +static int unpackBytesOrString(PyObject *obj, void **datap, size_t *sizep) +{ + void *data; + Py_ssize_t sz; + + if (PyBytes_AsStringAndSize(obj, &data, &sz) < 0) { +#if PY_VERSION_HEX >= 0x03000000 + PyErr_Clear(); + if ((data = PyUnicode_AsUTF8AndSize(obj, &sz)) != 0) + *sizep = strlen((char *)data) + 1; + else +#endif + return (-1); + } + *datap = data; + *sizep = sz; + return (0); +} + /* Write to and flush the stream. */ static int writeToPythonStream(const char *streamname, const char *message) diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/fpacking.py b/src/third_party/wiredtiger/lang/python/wiredtiger/fpacking.py index 5ad37895820..6501783a8e3 100644..100755 --- a/src/third_party/wiredtiger/lang/python/wiredtiger/fpacking.py +++ b/src/third_party/wiredtiger/lang/python/wiredtiger/fpacking.py @@ -30,6 +30,7 @@ # struct library. import struct +from .packing import empty_pack def __wt2struct(fmt): if not fmt: @@ -88,7 +89,7 @@ def unpack(fmt, s): def pack(fmt, *values): pfmt, fmt = __wt2struct(fmt) if not fmt: - return '' + return empty_pack i = sizebytes = 0 for offset, f in enumerate(fmt): if f == 'S': diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py b/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py index baaae58c168..27e756eb87b 100644..100755 --- a/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py +++ b/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py @@ -27,7 +27,9 @@ # OTHER DEALINGS IN THE SOFTWARE. # -import math, struct +from __future__ import print_function +from .packing import _chr, _ord +import math, struct, sys # Variable-length integer packing # need: up to 64 bits, both signed and unsigned @@ -62,58 +64,66 @@ POS_2BYTE_MAX = 2**13 + POS_1BYTE_MAX MINUS_BIT = -1 << 64 UINT64_MASK = 0xffffffffffffffff +_python3 = (sys.version_info >= (3, 0, 0)) +if _python3: + xff_entry = 0xff + x00_entry = 0x00 +else: + xff_entry = '\xff' + x00_entry = '\x00' + def getbits(x, start, end=0): '''return the least significant bits of x, from start to end''' return (x & ((1 << start) - 1)) >> (end) def get_int(b, size): r = 0 - for i in xrange(size): - r = (r << 8) | ord(b[i]) + for i in range(size): + r = (r << 8) | _ord(b[i]) return r def pack_int(x): if x < NEG_2BYTE_MIN: packed = struct.pack('>Q', x & UINT64_MASK) - while packed and packed[0] == '\xff': + while packed and packed[0] == xff_entry: packed = packed[1:] - return chr(NEG_MULTI_MARKER | getbits(8 - len(packed), 4)) + packed + return _chr(NEG_MULTI_MARKER | getbits(8 - len(packed), 4)) + packed elif x < NEG_1BYTE_MIN: x -= NEG_2BYTE_MIN - return chr(NEG_2BYTE_MARKER | getbits(x, 13, 8)) + chr(getbits(x, 8)) + return _chr(NEG_2BYTE_MARKER | getbits(x, 13, 8), getbits(x, 8)) elif x < 0: x -= NEG_1BYTE_MIN - return chr(NEG_1BYTE_MARKER | getbits(x, 6)) + return _chr(NEG_1BYTE_MARKER | getbits(x, 6)) elif x <= POS_1BYTE_MAX: - return chr(POS_1BYTE_MARKER | getbits(x, 6)) + return _chr(POS_1BYTE_MARKER | getbits(x, 6)) elif x <= POS_2BYTE_MAX: x -= (POS_1BYTE_MAX + 1) - return chr(POS_2BYTE_MARKER | getbits(x, 13, 8)) + chr(getbits(x, 8)) + return _chr(POS_2BYTE_MARKER | getbits(x, 13, 8), getbits(x, 8)) elif x == POS_2BYTE_MAX + 1: # This is a special case where we could store the value with # just a single byte, but we append a zero byte so that the # encoding doesn't get shorter for this one value. - return chr(POS_MULTI_MARKER | 0x1) + chr(0) + return _chr(POS_MULTI_MARKER | 0x1, 0) else: packed = struct.pack('>Q', x - (POS_2BYTE_MAX + 1)) - while packed and packed[0] == '\x00': + while packed and packed[0] == x00_entry: packed = packed[1:] - return chr(POS_MULTI_MARKER | getbits(len(packed), 4)) + packed + return _chr(POS_MULTI_MARKER | getbits(len(packed), 4)) + packed def unpack_int(b): - marker = ord(b[0]) + marker = _ord(b[0]) if marker < NEG_2BYTE_MARKER: sz = 8 - getbits(marker, 4) return ((-1 << (sz << 3)) | get_int(b[1:], sz), b[sz+1:]) elif marker < NEG_1BYTE_MARKER: - return (NEG_2BYTE_MIN + ((getbits(marker, 5) << 8) | ord(b[1])), b[2:]) + return (NEG_2BYTE_MIN + ((getbits(marker, 5) << 8) | _ord(b[1])), b[2:]) elif marker < POS_1BYTE_MARKER: return (NEG_1BYTE_MIN + getbits(marker, 6), b[1:]) elif marker < POS_2BYTE_MARKER: return (getbits(marker, 6), b[1:]) elif marker < POS_MULTI_MARKER: return (POS_1BYTE_MAX + 1 + - ((getbits(marker, 5) << 8) | ord(b[1])), b[2:]) + ((getbits(marker, 5) << 8) | _ord(b[1])), b[2:]) else: sz = getbits(marker, 4) return (POS_2BYTE_MAX + 1 + get_int(b[1:], sz), b[sz+1:]) @@ -123,21 +133,21 @@ if __name__ == '__main__': import random for big in (100, 10000, 1 << 40, 1 << 64): - for i in xrange(1000): + for i in range(1000): r = random.randint(-big, big) - print "\rChecking %d" % r, + print("\rChecking %d" % r) if unpack_int(pack_int(r))[0] != r: - print "\nFound a problem with %d" % r + print("\nFound a problem with %d" % r) break print - for i in xrange(1000): + for i in range(1000): r1 = random.randint(-big, big) r2 = random.randint(-big, big) - print "\rChecking %d, %d" % (r1, r2), + print("\rChecking %d, %d" % (r1, r2)) if cmp(r1, r2) != cmp(pack_int(r1), pack_int(r2)): - print "\nFound a problem with %d, %d" % (r1, r2) + print("\nFound a problem with %d, %d" % (r1, r2)) break print diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py b/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py index 438d9c15a08..c1c5a9bc6a3 100644..100755 --- a/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py +++ b/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py @@ -49,7 +49,50 @@ Format Python Notes u str raw byte array """ -from intpacking import pack_int, unpack_int +import sys + +# In the Python3 world, we pack into the bytes type, which like a list of ints. +# In Python2, we pack into a string. Create a set of constants and methods +# to hide the differences from the main code. +if sys.version_info[0] >= 3: + x00 = b'\x00' + xff = b'\xff' + empty_pack = b'' + def _ord(b): + return b + + def _chr(x, y=None): + a = [x] + if y != None: + a.append(y) + return bytes(a) + + def _is_string(s): + return type(s) is str + + def _string_result(s): + return s.decode() + +else: + x00 = '\x00' + xff = '\xff' + empty_pack = '' + def _ord(b): + return ord(b) + + def _chr(x, y=None): + s = chr(x) + if y != None: + s += chr(y) + return s + + def _is_string(s): + return type(s) is unicode + + def _string_result(s): + return s + +from .intpacking import pack_int, unpack_int def __get_type(fmt): if not fmt: @@ -92,36 +135,39 @@ def unpack(fmt, s): if f == 's': pass elif f == 'S': - size = s.find('\0') + size = s.find(x00) elif f == 'u' and offset == len(fmt) - 1: # A WT_ITEM with a NULL data field will be appear as None. if s == None: - s = '' + s = empty_pack size = len(s) else: # Note: 'U' is used internally, and may be exposed to us. # It indicates that the size is always stored unless there # is a size in the format. size, s = unpack_int(s) - result.append(s[:size]) - if f == 'S' and not havesize: - size += 1 + if f in 'Ss': + result.append(_string_result(s[:size])) + if f == 'S' and not havesize: + size += 1 + else: + result.append(s[:size]) s = s[size:] elif f in 't': # bit type, size is number of bits - result.append(ord(s[0:1])) + result.append(_ord(s[0])) s = s[1:] elif f in 'Bb': # byte type - for i in xrange(size): - v = ord(s[0:1]) + for i in range(size): + v = _ord(s[0]) if f != 'B': v -= 0x80 result.append(v) s = s[1:] else: # integral type - for j in xrange(size): + for j in range(size): v, s = unpack_int(s) result.append(v) return result @@ -136,7 +182,7 @@ def __pack_iter_fmt(fmt, values): index += 1 else: # integral type size = size if havesize else 1 - for i in xrange(size): + for i in range(size): value = values[index] yield offset, havesize, 1, char, value index = index + 1 @@ -147,14 +193,14 @@ def pack(fmt, *values): return () if tfmt != '.': raise ValueError('Only variable-length encoding is currently supported') - result = '' + result = empty_pack i = 0 for offset, havesize, size, f, val in __pack_iter_fmt(fmt, values): if f == 'x': if not havesize: - result += '\0' + result += x00 else: - result += '\0' * size + result += x00 * size # Note: no value, don't increment i elif f in 'SsUu': if f == 'S' and '\0' in val: @@ -166,14 +212,14 @@ def pack(fmt, *values): l = size elif (f == 'u' and offset != len(fmt) - 1) or f == 'U': result += pack_int(l) - if type(val) is unicode and f in 'Ss': - result += str(val[:l]) + if _is_string(val) and f in 'Ss': + result += str(val[:l]).encode() else: result += val[:l] if f == 'S' and not havesize: - result += '\0' + result += x00 elif size > l and havesize: - result += '\0' * (size - l) + result += x00 * (size - l) elif f in 't': # bit type, size is number of bits if not havesize: @@ -183,12 +229,12 @@ def pack(fmt, *values): mask = (1 << size) - 1 if (mask & val) != val: raise ValueError("value out of range for 't' encoding") - result += chr(val) + result += _chr(val) elif f in 'Bb': # byte type if not havesize: size = 1 - for i in xrange(size): + for i in range(size): if f == 'B': v = val else: @@ -196,7 +242,7 @@ def pack(fmt, *values): v = val + 0x80 if v > 255 or v < 0: raise ValueError("value out of range for 'B' encoding") - result += chr(v) + result += _chr(v) else: # integral type result += pack_int(val) diff --git a/src/third_party/wiredtiger/src/block/block_write.c b/src/third_party/wiredtiger/src/block/block_write.c index 9edc4e0108b..55f9d4ca57c 100644 --- a/src/third_party/wiredtiger/src/block/block_write.c +++ b/src/third_party/wiredtiger/src/block/block_write.c @@ -43,10 +43,8 @@ __wt_block_truncate(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t len) * more targeted solution at some point. */ if (!conn->hot_backup) { - __wt_readlock(session, &conn->hot_backup_lock); - if (!conn->hot_backup) - ret = __wt_ftruncate(session, block->fh, len); - __wt_readunlock(session, &conn->hot_backup_lock); + WT_WITH_HOTBACKUP_READ_LOCK(session, + ret = __wt_ftruncate(session, block->fh, len), NULL); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index f504bdeddf4..6a85ccf6c17 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -244,8 +244,7 @@ restart_read: /* Find the matching WT_COL slot. */ * information. */ if (cbt->cip_saved != cip) { - if ((cell = WT_COL_PTR(page, cip)) == NULL) - continue; + cell = WT_COL_PTR(page, cip); __wt_cell_unpack(session, page, cell, &unpack); if (unpack.type == WT_CELL_DEL) { if ((rle = __wt_cell_rle(&unpack)) == 1) diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index 22effc47553..1b8df0008b9 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -390,8 +390,7 @@ restart_read: /* Find the matching WT_COL slot. */ * information. */ if (cbt->cip_saved != cip) { - if ((cell = WT_COL_PTR(page, cip)) == NULL) - continue; + cell = WT_COL_PTR(page, cip); __wt_cell_unpack(session, page, cell, &unpack); if (unpack.type == WT_CELL_DEL) { if (__wt_cell_rle(&unpack) == 1) diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index a6645608150..e75432f7836 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -328,8 +328,8 @@ __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp, bool *valid) * when read. */ cip = &page->pg_var[cbt->slot]; - if ((cell = WT_COL_PTR(page, cip)) == NULL || - __wt_cell_type(cell) == WT_CELL_DEL) + cell = WT_COL_PTR(page, cip); + if (__wt_cell_type(cell) == WT_CELL_DEL) return (0); break; case BTREE_ROW: diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index 8d1ed01377c..86d00c18300 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -993,13 +993,9 @@ __debug_page_col_var(WT_DBG *ds, WT_REF *ref) recno = ref->ref_recno; WT_COL_FOREACH(page, cip, i) { - if ((cell = WT_COL_PTR(page, cip)) == NULL) { - unpack = NULL; - rle = 1; - } else { - __wt_cell_unpack(ds->session, page, cell, unpack); - rle = __wt_cell_rle(unpack); - } + cell = WT_COL_PTR(page, cip); + __wt_cell_unpack(ds->session, page, cell, unpack); + rle = __wt_cell_rle(unpack); WT_RET(__wt_snprintf( tag, sizeof(tag), "%" PRIu64 " %" PRIu64, recno, rle)); WT_RET( @@ -1339,7 +1335,8 @@ __debug_cell(WT_DBG *ds, const WT_PAGE_HEADER *dsk, WT_CELL_UNPACK *unpack) case WT_CELL_ADDR_LEAF: case WT_CELL_ADDR_LEAF_NO: __wt_timestamp_to_string(unpack->oldest_start_ts, ts_string[0]); - __wt_timestamp_to_string(unpack->newest_start_ts, ts_string[1]); + __wt_timestamp_to_string( + unpack->newest_durable_ts, ts_string[1]); __wt_timestamp_to_string(unpack->newest_stop_ts, ts_string[2]); WT_RET(ds->f(ds, ", ts %s,%s,%s", ts_string[0], ts_string[1], ts_string[2])); diff --git a/src/third_party/wiredtiger/src/btree/bt_rebalance.c b/src/third_party/wiredtiger/src/btree/bt_rebalance.c index 46dc96aedce..c04135ee82d 100644 --- a/src/third_party/wiredtiger/src/btree/bt_rebalance.c +++ b/src/third_party/wiredtiger/src/btree/bt_rebalance.c @@ -57,9 +57,9 @@ __rebalance_discard(WT_SESSION_IMPL *session, WT_REBALANCE_STUFF *rs) * Add a new entry to the list of leaf pages. */ static int -__rebalance_leaf_append(WT_SESSION_IMPL *session, - const uint8_t *key, size_t key_len, - WT_CELL_UNPACK *unpack, WT_REBALANCE_STUFF *rs) +__rebalance_leaf_append(WT_SESSION_IMPL *session, wt_timestamp_t durable_ts, + const uint8_t *key, size_t key_len, WT_CELL_UNPACK *unpack, + WT_REBALANCE_STUFF *rs) { WT_ADDR *copy_addr; WT_REF *copy; @@ -80,7 +80,7 @@ __rebalance_leaf_append(WT_SESSION_IMPL *session, WT_RET(__wt_calloc_one(session, ©_addr)); copy->addr = copy_addr; copy_addr->oldest_start_ts = unpack->oldest_start_ts; - copy_addr->newest_start_ts = unpack->newest_start_ts; + copy_addr->newest_durable_ts = durable_ts; copy_addr->newest_stop_ts = unpack->newest_stop_ts; WT_RET(__wt_memdup( session, unpack->data, unpack->size, ©_addr->addr)); @@ -194,8 +194,8 @@ __rebalance_free_original(WT_SESSION_IMPL *session, WT_REBALANCE_STUFF *rs) * Walk a column-store page and its descendants. */ static int -__rebalance_col_walk( - WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, WT_REBALANCE_STUFF *rs) +__rebalance_col_walk(WT_SESSION_IMPL *session, wt_timestamp_t durable_ts, + const WT_PAGE_HEADER *dsk, WT_REBALANCE_STUFF *rs) { WT_BTREE *btree; WT_CELL_UNPACK unpack; @@ -221,7 +221,8 @@ __rebalance_col_walk( /* An internal page: read it and recursively walk it. */ WT_ERR(__wt_bt_read( session, buf, unpack.data, unpack.size)); - WT_ERR(__rebalance_col_walk(session, buf->data, rs)); + WT_ERR(__rebalance_col_walk( + session, unpack.newest_durable_ts, buf->data, rs)); __wt_verbose(session, WT_VERB_REBALANCE, "free-list append internal page: %s", __wt_addr_string( @@ -232,7 +233,7 @@ __rebalance_col_walk( case WT_CELL_ADDR_LEAF: case WT_CELL_ADDR_LEAF_NO: WT_ERR(__rebalance_leaf_append( - session, NULL, 0, &unpack, rs)); + session, durable_ts, NULL, 0, &unpack, rs)); break; WT_ILLEGAL_VALUE_ERR(session, unpack.type); } @@ -273,8 +274,8 @@ __rebalance_row_leaf_key(WT_SESSION_IMPL *session, * Walk a row-store page and its descendants. */ static int -__rebalance_row_walk( - WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, WT_REBALANCE_STUFF *rs) +__rebalance_row_walk(WT_SESSION_IMPL *session, wt_timestamp_t durable_ts, + const WT_PAGE_HEADER *dsk, WT_REBALANCE_STUFF *rs) { WT_BTREE *btree; WT_CELL_UNPACK key, unpack; @@ -347,7 +348,8 @@ __rebalance_row_walk( /* Read and recursively walk the page. */ WT_ERR(__wt_bt_read( session, buf, unpack.data, unpack.size)); - WT_ERR(__rebalance_row_walk(session, buf->data, rs)); + WT_ERR(__rebalance_row_walk( + session, unpack.newest_durable_ts, buf->data, rs)); break; case WT_CELL_ADDR_LEAF: case WT_CELL_ADDR_LEAF_NO: @@ -376,7 +378,7 @@ __rebalance_row_walk( len = key.size; } WT_ERR(__rebalance_leaf_append( - session, p, len, &unpack, rs)); + session, durable_ts, p, len, &unpack, rs)); first_cell = false; break; @@ -399,17 +401,19 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_BTREE *btree; WT_DECL_RET; WT_REBALANCE_STUFF *rs, _rstuff; + WT_REF *ref; WT_UNUSED(cfg); btree = S2BT(session); + ref = &btree->root; /* * If the tree has never been written to disk, we're done, rebalance * walks disk images, not in-memory pages. For the same reason, the * tree has to be clean. */ - if (btree->root.page->dsk == NULL) + if (ref->page->dsk == NULL) return (0); if (btree->modified) WT_RET_MSG(session, EINVAL, @@ -422,17 +426,22 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_scr_alloc(session, 0, &rs->tmp2)); /* Set the internal page tree type. */ - rs->type = btree->root.page->type; + rs->type = ref->page->type; - /* Recursively walk the tree. */ + /* + * Recursively walk the tree. We start with a durable timestamp, but + * it should never be used (we'll accumulate durable timestamps from + * all the internal pages in our final write), so set it to something + * impossible. + */ switch (rs->type) { case WT_PAGE_ROW_INT: - WT_ERR( - __rebalance_row_walk(session, btree->root.page->dsk, rs)); + WT_ERR(__rebalance_row_walk( + session, WT_TS_MAX, ref->page->dsk, rs)); break; case WT_PAGE_COL_INT: - WT_ERR( - __rebalance_col_walk(session, btree->root.page->dsk, rs)); + WT_ERR(__rebalance_col_walk( + session, WT_TS_MAX, ref->page->dsk, rs)); break; WT_ILLEGAL_VALUE_ERR(session, rs->type); } @@ -450,8 +459,8 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) * Swap the old root page for our newly built root page, writing the new * root page as part of a checkpoint will finish the rebalance. */ - __wt_page_out(session, &btree->root.page); - btree->root.page = rs->root; + __wt_page_out(session, &ref->page); + ref->page = rs->root; rs->root = NULL; err: /* Discard any leftover root page we created. */ diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index a03cfb6405d..08f7c424d6c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -75,7 +75,6 @@ struct __wt_track { #define trk_addr shared->addr.addr #define trk_addr_size shared->addr.size #define trk_oldest_start_ts shared->addr.oldest_start_ts -#define trk_newest_start_ts shared->addr.newest_start_ts #define trk_newest_stop_ts shared->addr.newest_stop_ts #define trk_gen shared->gen #define trk_ovfl_addr shared->ovfl_addr @@ -505,10 +504,9 @@ __slvg_trk_init(WT_SESSION_IMPL *session, trk->trk_addr_size = (uint8_t)addr_size; trk->trk_size = dsk->mem_size; trk->trk_oldest_start_ts = WT_TS_MAX; - trk->trk_newest_start_ts = trk->trk_newest_stop_ts = WT_TS_NONE; + trk->trk_newest_stop_ts = WT_TS_NONE; if (!__wt_process.page_version_ts || dsk->type == WT_PAGE_COL_FIX) { - trk->trk_oldest_start_ts = - trk->trk_newest_start_ts = WT_TS_NONE; + trk->trk_oldest_start_ts = WT_TS_NONE; trk->trk_newest_stop_ts = WT_TS_MAX; } trk->trk_gen = dsk->write_gen; @@ -665,8 +663,6 @@ __slvg_trk_leaf_ts(WT_TRACK *trk, WT_CELL_UNPACK *unpack) { trk->trk_oldest_start_ts = WT_MIN(unpack->start_ts, trk->trk_oldest_start_ts); - trk->trk_newest_start_ts = - WT_MAX(unpack->start_ts, trk->trk_newest_start_ts); trk->trk_newest_stop_ts = WT_MAX(unpack->stop_ts, trk->trk_newest_stop_ts); } @@ -1070,8 +1066,6 @@ merge: */ a_trk->trk_oldest_start_ts = b_trk->trk_oldest_start_ts = WT_MIN(a_trk->trk_oldest_start_ts, b_trk->trk_oldest_start_ts); - a_trk->trk_newest_start_ts = b_trk->trk_newest_start_ts = - WT_MAX(a_trk->trk_newest_start_ts, b_trk->trk_newest_start_ts); a_trk->trk_newest_stop_ts = b_trk->trk_newest_stop_ts = WT_MAX(a_trk->trk_newest_stop_ts, b_trk->trk_newest_stop_ts); __wt_verbose(session, WT_VERB_SALVAGE, @@ -1203,9 +1197,13 @@ __slvg_col_build_internal( ref->home = page; ref->page = NULL; + /* + * Salvage doesn't read tree internal pages, so all pages are + * immediately durable, regardless of the leaf page timestamps. + */ WT_ERR(__wt_calloc_one(session, &addr)); addr->oldest_start_ts = trk->trk_oldest_start_ts; - addr->newest_start_ts = trk->trk_newest_start_ts; + addr->newest_durable_ts = WT_TS_NONE; addr->newest_stop_ts = trk->trk_newest_stop_ts; WT_ERR(__wt_memdup( session, trk->trk_addr, trk->trk_addr_size, &addr->addr)); @@ -1726,8 +1724,6 @@ merge: */ a_trk->trk_oldest_start_ts = b_trk->trk_oldest_start_ts = WT_MIN(a_trk->trk_oldest_start_ts, b_trk->trk_oldest_start_ts); - a_trk->trk_newest_start_ts = b_trk->trk_newest_start_ts = - WT_MAX(a_trk->trk_newest_start_ts, b_trk->trk_newest_start_ts); a_trk->trk_newest_stop_ts = b_trk->trk_newest_stop_ts = WT_MAX(a_trk->trk_newest_stop_ts, b_trk->trk_newest_stop_ts); __wt_verbose(session, WT_VERB_SALVAGE, @@ -1875,9 +1871,13 @@ __slvg_row_build_internal( ref->home = page; ref->page = NULL; + /* + * Salvage doesn't read tree internal pages, so all pages are + * immediately durable, regardless of the leaf page timestamps. + */ WT_ERR(__wt_calloc_one(session, &addr)); addr->oldest_start_ts = trk->trk_oldest_start_ts; - addr->newest_start_ts = trk->trk_newest_start_ts; + addr->newest_durable_ts = WT_TS_NONE; addr->newest_stop_ts = trk->trk_newest_stop_ts; WT_ERR(__wt_memdup( session, trk->trk_addr, trk->trk_addr_size, &addr->addr)); diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index f0407ce71b1..127c307b9ab 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -264,7 +264,7 @@ __split_ref_move(WT_SESSION_IMPL *session, WT_PAGE *from_home, session, from_home, (WT_CELL *)ref_addr, &unpack); WT_RET(__wt_calloc_one(session, &addr)); addr->oldest_start_ts = unpack.oldest_start_ts; - addr->newest_start_ts = unpack.newest_start_ts; + addr->newest_durable_ts = unpack.newest_durable_ts; addr->newest_stop_ts = unpack.newest_stop_ts; WT_ERR(__wt_memdup( session, unpack.data, unpack.size, &addr->addr)); @@ -1675,7 +1675,7 @@ __wt_multi_to_ref(WT_SESSION_IMPL *session, WT_RET(__wt_calloc_one(session, &addr)); ref->addr = addr; addr->oldest_start_ts = multi->addr.oldest_start_ts; - addr->newest_start_ts = multi->addr.newest_start_ts; + addr->newest_durable_ts = multi->addr.newest_durable_ts; addr->newest_stop_ts = multi->addr.newest_stop_ts; WT_RET(__wt_memdup(session, multi->addr.addr, multi->addr.size, &addr->addr)); diff --git a/src/third_party/wiredtiger/src/btree/bt_stat.c b/src/third_party/wiredtiger/src/btree/bt_stat.c index c201d9af73a..976a771a233 100644 --- a/src/third_party/wiredtiger/src/btree/bt_stat.c +++ b/src/third_party/wiredtiger/src/btree/bt_stat.c @@ -160,21 +160,18 @@ __stat_page_col_var( * we see. */ WT_COL_FOREACH(page, cip, i) { - if ((cell = WT_COL_PTR(page, cip)) == NULL) { + cell = WT_COL_PTR(page, cip); + __wt_cell_unpack(session, page, cell, unpack); + if (unpack->type == WT_CELL_DEL) { orig_deleted = true; - ++deleted_cnt; + deleted_cnt += __wt_cell_rle(unpack); } else { orig_deleted = false; - __wt_cell_unpack(session, page, cell, unpack); - if (unpack->type == WT_CELL_DEL) - orig_deleted = true; - else { - entry_cnt += __wt_cell_rle(unpack); - rle_cnt += __wt_cell_rle(unpack) - 1; - } - if (unpack->ovfl) - ++ovfl_cnt; + entry_cnt += __wt_cell_rle(unpack); } + rle_cnt += __wt_cell_rle(unpack) - 1; + if (unpack->ovfl) + ++ovfl_cnt; /* * Walk the insert list, checking for changes. For each insert diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy.c b/src/third_party/wiredtiger/src/btree/bt_vrfy.c index 1a412ace8f9..f85389bbe81 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy.c @@ -237,7 +237,7 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) */ memset(&addr_unpack, 0, sizeof(addr_unpack)); addr_unpack.oldest_start_ts = ckpt->oldest_start_ts; - addr_unpack.newest_start_ts = ckpt->newest_start_ts; + addr_unpack.newest_durable_ts = ckpt->newest_durable_ts; addr_unpack.newest_stop_ts = ckpt->newest_stop_ts; addr_unpack.raw = WT_CELL_ADDR_INT; @@ -331,14 +331,9 @@ __verify_addr_ts(WT_SESSION_IMPL *session, "internal page reference at %s has a newest stop " "timestamp of 0", __wt_page_addr_string(session, ref, vs->tmp1)); - if (unpack->oldest_start_ts > unpack->newest_start_ts) + if (unpack->oldest_start_ts > unpack->newest_stop_ts) WT_RET_MSG(session, WT_ERROR, "internal page reference at %s has an oldest start " - "timestamp newer than its newest start timestamp", - __wt_page_addr_string(session, ref, vs->tmp1)); - if (unpack->newest_start_ts > unpack->newest_stop_ts) - WT_RET_MSG(session, WT_ERROR, - "internal page reference at %s has a newest start " "timestamp newer than its newest stop timestamp", __wt_page_addr_string(session, ref, vs->tmp1)); return (0); @@ -448,13 +443,11 @@ recno_chk: if (recno != vs->record_total + 1) break; case WT_PAGE_COL_VAR: recno = 0; - WT_COL_FOREACH(page, cip, i) - if ((cell = WT_COL_PTR(page, cip)) == NULL) - ++recno; - else { - __wt_cell_unpack(session, page, cell, unpack); - recno += __wt_cell_rle(unpack); - } + WT_COL_FOREACH(page, cip, i) { + cell = WT_COL_PTR(page, cip); + __wt_cell_unpack(session, page, cell, unpack); + recno += __wt_cell_rle(unpack); + } vs->record_total += recno; break; } @@ -745,7 +738,7 @@ __verify_ts_addr_cmp(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t cell_num, bool gt, WT_VSTUFF *vs) { const char *ts1_bp, *ts2_bp; - char ts1_buf[32], ts2_buf[32]; + char ts_string[2][WT_TS_INT_STRING_SIZE]; if (gt && ts1 >= ts2) return (0); @@ -760,9 +753,8 @@ __verify_ts_addr_cmp(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t cell_num, ts1_bp = "WT_TS_NONE"; break; default: - WT_RET( - __wt_snprintf(ts1_buf, sizeof(ts1_buf), "%" PRIu64, ts1)); - ts1_bp = ts1_buf; + __wt_timestamp_to_string(ts1, ts_string[0]); + ts1_bp = ts_string[0]; break; } switch (ts2) { @@ -773,14 +765,13 @@ __verify_ts_addr_cmp(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t cell_num, ts2_bp = "WT_TS_NONE"; break; default: - WT_RET( - __wt_snprintf(ts2_buf, sizeof(ts2_buf), "%" PRIu64, ts2)); - ts2_bp = ts2_buf; + __wt_timestamp_to_string(ts2, ts_string[1]); + ts2_bp = ts_string[1]; break; } WT_RET_MSG(session, WT_ERROR, "cell %" PRIu32 " on page at %s failed verification with %s " - "time of %s, %s the parent's %s time of %s", + "timestamp of %s, %s the parent's %s timestamp of %s", cell_num, __wt_page_addr_string(session, ref, vs->tmp1), ts1_name, ts1_bp, @@ -801,6 +792,7 @@ __verify_page_cell(WT_SESSION_IMPL *session, WT_DECL_RET; const WT_PAGE_HEADER *dsk; uint32_t cell_num; + char ts_string[2][WT_TS_INT_STRING_SIZE]; bool found_ovfl; /* @@ -851,30 +843,27 @@ __verify_page_cell(WT_SESSION_IMPL *session, cell_num - 1, __wt_page_addr_string( session, ref, vs->tmp1)); - if (unpack.oldest_start_ts > unpack.newest_start_ts) + if (unpack.oldest_start_ts > unpack.newest_stop_ts) { + __wt_timestamp_to_string( + unpack.oldest_start_ts, ts_string[0]); + __wt_timestamp_to_string( + unpack.newest_stop_ts, ts_string[1]); WT_RET_MSG(session, WT_ERROR, "cell %" PRIu32 " on page at %s has an " - "oldest start timestamp newer than its " - "newest start timestamp", - cell_num - 1, - __wt_page_addr_string( - session, ref, vs->tmp1)); - if (unpack.newest_start_ts > unpack.newest_stop_ts) - WT_RET_MSG(session, WT_ERROR, - "cell %" PRIu32 " on page at %s has a " - "newest start timestamp newer than its " - "newest stop timestamp", + "oldest start timestamp %s newer than " + "its newest stop timestamp %s", cell_num - 1, - __wt_page_addr_string( - session, ref, vs->tmp1)); + __wt_page_addr_string(session, + ref, vs->tmp1), ts_string[0], ts_string[1]); + } WT_RET(__verify_ts_addr_cmp(session, ref, cell_num - 1, "oldest start", unpack.oldest_start_ts, "oldest start", addr_unpack->oldest_start_ts, true, vs)); WT_RET(__verify_ts_addr_cmp(session, ref, cell_num - 1, - "newest start", unpack.newest_start_ts, - "newest start", addr_unpack->newest_start_ts, + "newest durable", unpack.newest_durable_ts, + "newest durable", addr_unpack->newest_durable_ts, false, vs)); WT_RET(__verify_ts_addr_cmp(session, ref, cell_num - 1, "newest stop", unpack.newest_stop_ts, @@ -893,24 +882,25 @@ __verify_page_cell(WT_SESSION_IMPL *session, cell_num - 1, __wt_page_addr_string( session, ref, vs->tmp1)); - if (unpack.start_ts > unpack.stop_ts) + if (unpack.start_ts > unpack.stop_ts) { + __wt_timestamp_to_string( + unpack.start_ts, ts_string[0]); + __wt_timestamp_to_string( + unpack.stop_ts, ts_string[1]); WT_RET_MSG(session, WT_ERROR, "cell %" PRIu32 " on page at %s has a " - "start timestamp newer than its stop " - "timestamp ", + "start timestamp %s newer than its stop " + "timestamp %s", cell_num - 1, - __wt_page_addr_string( - session, ref, vs->tmp1)); + __wt_page_addr_string(session, + ref, vs->tmp1), ts_string[0], ts_string[1]); + } WT_RET(__verify_ts_addr_cmp(session, ref, cell_num - 1, "start", unpack.start_ts, "oldest start", addr_unpack->oldest_start_ts, true, vs)); WT_RET(__verify_ts_addr_cmp(session, ref, cell_num - 1, - "start", unpack.start_ts, - "newest start", addr_unpack->newest_start_ts, - false, vs)); - WT_RET(__verify_ts_addr_cmp(session, ref, cell_num - 1, "stop", unpack.stop_ts, "newest stop", addr_unpack->newest_stop_ts, false, vs)); diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c index ee6cd904aec..24d6d22f1ef 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c @@ -222,7 +222,7 @@ __verify_dsk_ts_addr_cmp(WT_SESSION_IMPL *session, uint32_t cell_num, bool gt, const char *tag) { const char *ts1_bp, *ts2_bp; - char ts1_buf[32], ts2_buf[32]; + char ts_string[2][WT_TS_INT_STRING_SIZE]; if (gt && ts1 >= ts2) return (0); @@ -237,9 +237,8 @@ __verify_dsk_ts_addr_cmp(WT_SESSION_IMPL *session, uint32_t cell_num, ts1_bp = "WT_TS_NONE"; break; default: - WT_RET( - __wt_snprintf(ts1_buf, sizeof(ts1_buf), "%" PRIu64, ts1)); - ts1_bp = ts1_buf; + __wt_timestamp_to_string(ts1, ts_string[0]); + ts1_bp = ts_string[0]; break; } switch (ts2) { @@ -250,14 +249,13 @@ __verify_dsk_ts_addr_cmp(WT_SESSION_IMPL *session, uint32_t cell_num, ts2_bp = "WT_TS_NONE"; break; default: - WT_RET( - __wt_snprintf(ts2_buf, sizeof(ts2_buf), "%" PRIu64, ts2)); - ts2_bp = ts2_buf; + __wt_timestamp_to_string(ts2, ts_string[1]); + ts2_bp = ts_string[1]; break; } WT_RET_MSG(session, WT_ERROR, "cell %" PRIu32 " on page at %s failed verification with %s " - "time of %s, %s the parent's %s time of %s", + "timestamp of %s, %s the parent's %s timestamp of %s", cell_num, tag, ts1_name, ts1_bp, gt ? "less than" : "greater than", @@ -272,6 +270,8 @@ static int __verify_dsk_ts(WT_SESSION_IMPL *session, WT_CELL_UNPACK *unpack, uint32_t cell_num, WT_ADDR *addr, const char *tag) { + char ts_string[2][WT_TS_INT_STRING_SIZE]; + /* * Check timestamp order, and optionally, against a parent address. * Timestamps in the parent address aren't necessarily an exact match, @@ -291,17 +291,17 @@ __verify_dsk_ts(WT_SESSION_IMPL *session, "cell %" PRIu32 " on page at %s has a newest stop " "timestamp of 0", cell_num - 1, tag); - if (unpack->oldest_start_ts > unpack->newest_start_ts) + if (unpack->oldest_start_ts > unpack->newest_stop_ts) { + __wt_timestamp_to_string( + unpack->oldest_start_ts, ts_string[0]); + __wt_timestamp_to_string( + unpack->newest_stop_ts, ts_string[1]); WT_RET_VRFY(session, "cell %" PRIu32 " on page at %s has an oldest " - "start timestamp newer than its newest start " - "timestamp", - cell_num - 1, tag); - if (unpack->newest_start_ts > unpack->newest_stop_ts) - WT_RET_VRFY(session, - "cell %" PRIu32 " on page at %s has a newest start " - "timestamp newer than its newest stop timestamp", - cell_num - 1, tag); + "start timestamp %s newer than its newest stop " + "timestamp %s", + cell_num - 1, tag, ts_string[0], ts_string[1]); + } if (addr == NULL) break; @@ -310,8 +310,8 @@ __verify_dsk_ts(WT_SESSION_IMPL *session, "oldest start", addr->oldest_start_ts, true, tag)); WT_RET(__verify_dsk_ts_addr_cmp(session, cell_num - 1, - "newest start", unpack->newest_start_ts, - "newest start", addr->newest_start_ts, + "newest durable", unpack->newest_durable_ts, + "newest durable", addr->newest_durable_ts, false, tag)); WT_RET(__verify_dsk_ts_addr_cmp(session, cell_num - 1, "newest stop", unpack->newest_stop_ts, @@ -329,11 +329,16 @@ __verify_dsk_ts(WT_SESSION_IMPL *session, "cell %" PRIu32 " on page at %s has a stop " "timestamp of 0", cell_num - 1, tag); - if (unpack->start_ts > unpack->stop_ts) + if (unpack->start_ts > unpack->stop_ts) { + __wt_timestamp_to_string( + unpack->start_ts, ts_string[0]); + __wt_timestamp_to_string( + unpack->stop_ts, ts_string[0]); WT_RET_VRFY(session, "cell %" PRIu32 " on page at %s has a start " - "timestamp newer than its stop timestamp ", - cell_num - 1, tag); + "timestamp %s newer than its stop timestamp %s", + cell_num - 1, tag, ts_string[0], ts_string[1]); + } if (addr == NULL) break; @@ -342,10 +347,6 @@ __verify_dsk_ts(WT_SESSION_IMPL *session, "oldest start", addr->oldest_start_ts, true, tag)); WT_RET(__verify_dsk_ts_addr_cmp(session, cell_num - 1, - "start", unpack->start_ts, - "newest start", addr->newest_start_ts, - false, tag)); - WT_RET(__verify_dsk_ts_addr_cmp(session, cell_num - 1, "stop", unpack->stop_ts, "newest stop", addr->newest_stop_ts, false, tag)); diff --git a/src/third_party/wiredtiger/src/cache/cache_las.c b/src/third_party/wiredtiger/src/cache/cache_las.c index 905c24dceae..06e0056613c 100644 --- a/src/third_party/wiredtiger/src/cache/cache_las.c +++ b/src/third_party/wiredtiger/src/cache/cache_las.c @@ -1156,12 +1156,9 @@ __wt_las_sweep(WT_SESSION_IMPL *session) (prepare_state != WT_PREPARE_INPROGRESS || durable_timestamp == 0)); - /* - * FIXME Disable this assertion until fixed by WT-4598. - * WT_ASSERT(session, - * (prepare_state == WT_PREPARE_INPROGRESS || - * durable_timestamp >= las_timestamp)); - */ + WT_ASSERT(session, + (prepare_state == WT_PREPARE_INPROGRESS || + durable_timestamp >= las_timestamp)); /* * There are several conditions that need to be met diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 09a52300f3b..e56e7f29004 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -287,7 +287,6 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_begin_transaction[] = { { "name", "string", NULL, NULL, NULL, 0 }, { "priority", "int", NULL, "min=-100,max=100", NULL, 0 }, { "read_timestamp", "string", NULL, NULL, NULL, 0 }, - { "round_to_oldest", "boolean", NULL, NULL, NULL, 0 }, { "roundup_timestamps", "category", NULL, NULL, confchk_WT_SESSION_begin_transaction_roundup_timestamps_subconfigs, 2 }, @@ -533,7 +532,6 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_timestamp_transaction[] = { { "durable_timestamp", "string", NULL, NULL, NULL, 0 }, { "prepare_timestamp", "string", NULL, NULL, NULL, 0 }, { "read_timestamp", "string", NULL, NULL, NULL, 0 }, - { "round_to_oldest", "boolean", NULL, NULL, NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -1371,9 +1369,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { }, { "WT_SESSION.begin_transaction", "ignore_prepare=false,isolation=,name=,priority=0,read_timestamp=" - ",round_to_oldest=false,roundup_timestamps=(prepared=false," - "read=false),snapshot=,sync=", - confchk_WT_SESSION_begin_transaction, 9 + ",roundup_timestamps=(prepared=false,read=false),snapshot=,sync=", + confchk_WT_SESSION_begin_transaction, 8 }, { "WT_SESSION.checkpoint", "drop=,force=false,name=,target=,use_timestamp=true", @@ -1482,8 +1479,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { }, { "WT_SESSION.timestamp_transaction", "commit_timestamp=,durable_timestamp=,prepare_timestamp=," - "read_timestamp=,round_to_oldest=false", - confchk_WT_SESSION_timestamp_transaction, 5 + "read_timestamp=", + confchk_WT_SESSION_timestamp_transaction, 4 }, { "WT_SESSION.transaction_sync", "timeout_ms=1200000", diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index 8bc111346c5..81f5724663f 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -343,6 +343,28 @@ __wt_logmgr_reconfig(WT_SESSION_IMPL *session, const char **cfg) } /* + * __log_archive_once_int -- + * Helper for __log_archive_once. Intended to be called while holding the + * hot backup read lock. + */ +static int +__log_archive_once_int(WT_SESSION_IMPL *session, + char **logfiles, u_int logcount, uint32_t min_lognum) +{ + uint32_t lognum; + u_int i; + + for (i = 0; i < logcount; i++) { + WT_RET(__wt_log_extract_lognum(session, logfiles[i], &lognum)); + if (lognum < min_lognum) + WT_RET(__wt_log_remove( + session, WT_LOG_FILENAME, lognum)); + } + + return (0); +} + +/* * __log_archive_once -- * Perform one iteration of log archiving. Must be called with the * log archive lock held. @@ -353,15 +375,13 @@ __log_archive_once(WT_SESSION_IMPL *session, uint32_t backup_file) WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_LOG *log; - uint32_t lognum, min_lognum; - u_int i, logcount; + uint32_t min_lognum; + u_int logcount; char **logfiles; - bool locked; conn = S2C(session); log = conn->log; logcount = 0; - locked = false; logfiles = NULL; /* @@ -386,22 +406,18 @@ __log_archive_once(WT_SESSION_IMPL *session, uint32_t backup_file) session, conn->log_path, WT_LOG_FILENAME, &logfiles, &logcount)); /* - * We can only archive files if a hot backup is not in progress or - * if we are the backup. + * If backup_file is non-zero we know we're coming from an incremental + * backup cursor. In that case just perform the archive operation + * without the lock. */ - __wt_readlock(session, &conn->hot_backup_lock); - locked = true; - if (!conn->hot_backup || backup_file != 0) { - for (i = 0; i < logcount; i++) { - WT_ERR(__wt_log_extract_lognum( - session, logfiles[i], &lognum)); - if (lognum < min_lognum) - WT_ERR(__wt_log_remove( - session, WT_LOG_FILENAME, lognum)); - } - } - __wt_readunlock(session, &conn->hot_backup_lock); - locked = false; + if (backup_file != 0) + ret = __log_archive_once_int( + session, logfiles, logcount, min_lognum); + else + WT_WITH_HOTBACKUP_READ_LOCK(session, + ret = __log_archive_once_int( + session, logfiles, logcount, min_lognum), NULL); + WT_ERR(ret); /* * Indicate what is our new earliest LSN. It is the start @@ -411,8 +427,6 @@ __log_archive_once(WT_SESSION_IMPL *session, uint32_t backup_file) if (0) err: __wt_err(session, ret, "log archive server error"); - if (locked) - __wt_readunlock(session, &conn->hot_backup_lock); WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); return (ret); } @@ -594,18 +608,15 @@ __log_file_server(void *arg) * truncate: both are OK, it's just more work * during cursor traversal. */ - if (!conn->hot_backup) { - __wt_readlock( - session, &conn->hot_backup_lock); - if (!conn->hot_backup && - conn->log_cursors == 0) - WT_ERR_ERROR_OK( - __wt_ftruncate(session, + if (!conn->hot_backup && + conn->log_cursors == 0) { + WT_WITH_HOTBACKUP_READ_LOCK(session, + WT_ERR_ERROR_OK( + __wt_ftruncate( + session, close_fh, close_end_lsn.l.offset), - ENOTSUP); - __wt_readunlock( - session, &conn->hot_backup_lock); + ENOTSUP), NULL); } WT_SET_LSN(&close_end_lsn, close_end_lsn.l.file + 1, 0); @@ -976,11 +987,8 @@ __log_server(void *arg) * agreed not to rename or remove any files in * the database directory. */ - __wt_readlock(session, &conn->hot_backup_lock); - if (!conn->hot_backup) - ret = __log_prealloc_once(session); - __wt_readunlock( - session, &conn->hot_backup_lock); + WT_WITH_HOTBACKUP_READ_LOCK(session, + ret = __log_prealloc_once(session), NULL); WT_ERR(ret); } diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index 04882e527ce..9a279ca3970 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -265,10 +265,8 @@ __backup_start(WT_SESSION_IMPL *session, * operations will not see the backup file list until it is * complete and valid. */ - __wt_writelock(session, &conn->hot_backup_lock); - conn->hot_backup = true; - conn->hot_backup_list = NULL; - __wt_writeunlock(session, &conn->hot_backup_lock); + WT_WITH_HOTBACKUP_WRITE_LOCK(session, + WT_CONN_HOTBACKUP_START(conn)); /* We're the lock holder, we own cleanup. */ F_SET(cb, WT_CURBACKUP_LOCKER); @@ -368,9 +366,8 @@ err: /* Close the hot backup file. */ ret = __wt_sync_and_rename(session, &cb->bfs, WT_BACKUP_TMP, dest); if (ret == 0) { - __wt_writelock(session, &conn->hot_backup_lock); - conn->hot_backup_list = cb->list; - __wt_writeunlock(session, &conn->hot_backup_lock); + WT_WITH_HOTBACKUP_WRITE_LOCK(session, + conn->hot_backup_list = cb->list); F_SET(session, WT_SESSION_BACKUP_CURSOR); } /* @@ -399,18 +396,14 @@ __backup_stop(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) WT_ASSERT(session, !F_ISSET(cb, WT_CURBACKUP_DUP)); /* If it's not a dup backup cursor, make sure one isn't open. */ WT_ASSERT(session, !F_ISSET(session, WT_SESSION_BACKUP_DUP)); - __wt_writelock(session, &conn->hot_backup_lock); - conn->hot_backup_list = NULL; - __wt_writeunlock(session, &conn->hot_backup_lock); + WT_WITH_HOTBACKUP_WRITE_LOCK(session, conn->hot_backup_list = NULL); __backup_free(session, cb); /* Remove any backup specific file. */ WT_TRET(__wt_backup_file_remove(session)); /* Checkpoint deletion and next hot backup can proceed. */ - __wt_writelock(session, &conn->hot_backup_lock); - conn->hot_backup = false; - __wt_writeunlock(session, &conn->hot_backup_lock); + WT_WITH_HOTBACKUP_WRITE_LOCK(session, conn->hot_backup = false); F_CLR(session, WT_SESSION_BACKUP_CURSOR); return (ret); diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index baabcd0182c..ee0d57037eb 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -544,8 +544,8 @@ __wt_curindex_open(WT_SESSION_IMPL *session, WT_ERR(__curindex_open_colgroups(session, cindex, cfg)); if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON)) - __wt_json_column_init(cursor, uri, table->key_format, - &idx->colconf, &table->colconf); + WT_ERR(__wt_json_column_init(cursor, uri, table->key_format, + &idx->colconf, &table->colconf)); if (0) { err: WT_TRET(__curindex_close(cursor)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_json.c b/src/third_party/wiredtiger/src/cursor/cur_json.c index 21716005c27..f540775180e 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_json.c +++ b/src/third_party/wiredtiger/src/cursor/cur_json.c @@ -309,6 +309,8 @@ __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor) if ((json = (WT_CURSOR_JSON *)cursor->json_private) != NULL) { __wt_free(session, json->key_buf); __wt_free(session, json->value_buf); + __wt_free(session, json->key_names.str); + __wt_free(session, json->value_names.str); __wt_free(session, json); } } @@ -373,21 +375,26 @@ __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) * Set json_key_names, json_value_names to comma separated lists * of column names. */ -void +int __wt_json_column_init(WT_CURSOR *cursor, const char *uri, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf) { WT_CURSOR_JSON *json; + WT_SESSION_IMPL *session; + size_t len; uint32_t keycnt, nkeys; const char *beginkey, *end, *lparen, *p; json = (WT_CURSOR_JSON *)cursor->json_private; + session = (WT_SESSION_IMPL *)cursor->session; beginkey = colconf->str; end = beginkey + colconf->len; if (idxconf != NULL) { - json->key_names.str = idxconf->str; - json->key_names.len = idxconf->len; + len = idxconf->len; + WT_RET(__wt_strndup(session, idxconf->str, len, + &json->key_names.str)); + json->key_names.len = len; } else if (colconf->len > 0 && *beginkey == '(') { beginkey++; if (end[-1] == ')') @@ -407,20 +414,25 @@ __wt_json_column_init(WT_CURSOR *cursor, const char *uri, const char *keyformat, } if ((lparen = strchr(uri, '(')) != NULL) { /* This cursor is a projection. */ - json->value_names.str = lparen; - json->value_names.len = strlen(lparen) - 1; - WT_ASSERT((WT_SESSION_IMPL *)cursor->session, - json->value_names.str[json->value_names.len] == ')'); + len = strlen(lparen) - 1; + WT_ASSERT(session, lparen[len] == ')'); + WT_RET(__wt_strndup(session, lparen, len, + &json->value_names.str)); + json->value_names.len = len; } else { - json->value_names.str = p; - json->value_names.len = WT_PTRDIFF(end, p); + len = WT_PTRDIFF(end, p); + WT_RET(__wt_strndup(session, p, len, &json->value_names.str)); + json->value_names.len = len; } if (idxconf == NULL) { if (p > beginkey) p--; - json->key_names.str = beginkey; - json->key_names.len = WT_PTRDIFF(p, beginkey); + len = WT_PTRDIFF(p, beginkey); + WT_RET(__wt_strndup(session, beginkey, len, + &json->key_names.str)); + json->key_names.len = len; } + return (0); } #define MATCH_KEYWORD(session, in, result, keyword, matchval) do { \ diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c index 77c6018778c..3198a15bd13 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_table.c +++ b/src/third_party/wiredtiger/src/cursor/cur_table.c @@ -1050,8 +1050,8 @@ __wt_curtable_open(WT_SESSION_IMPL *session, cursor, cursor->internal_uri, owner, cfg, cursorp)); if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON)) - __wt_json_column_init( - cursor, uri, table->key_format, NULL, &table->colconf); + WT_ERR(__wt_json_column_init( + cursor, uri, table->key_format, NULL, &table->colconf)); /* * Open the colgroup cursors immediately: we're going to need them for diff --git a/src/third_party/wiredtiger/src/docs/tools/doxypy.py b/src/third_party/wiredtiger/src/docs/tools/doxypy.py index 54fef5f03a5..f05a597ed6e 100755 --- a/src/third_party/wiredtiger/src/docs/tools/doxypy.py +++ b/src/third_party/wiredtiger/src/docs/tools/doxypy.py @@ -1,5 +1,7 @@ #!/usr/bin/env python +from __future__ import print_function + __applicationName__ = "doxypy" __blurb__ = """ doxypy is an input filter for Doxygen. It preprocesses python @@ -86,7 +88,7 @@ class FSM(object): self.current_input = input self.current_transition = transition if options.debug: - print >>sys.stderr, "# FSM: executing (%s -> %s) for line '%s'" % (from_state, to_state, input) + print("# FSM: executing (%s -> %s) for line '%s'" % (from_state, to_state, input), file=sys.stderr) callback(match) return @@ -208,8 +210,8 @@ class Doxypy(object): if self.output: try: if options.debug: - print >>sys.stderr, "# OUTPUT: ", self.output - print >>self.outstream, "\n".join(self.output) + print("# OUTPUT: ", self.output, file=sys.stderr) + print("\n".join(self.output), file=self.outstream) self.outstream.flush() except IOError: # Fix for FS#33. Catches "broken pipe" when doxygen closes @@ -228,7 +230,7 @@ class Doxypy(object): Closes the current commentblock and starts a new comment search. """ if options.debug: - print >>sys.stderr, "# CALLBACK: resetCommentSearch" + print("# CALLBACK: resetCommentSearch" , file=sys.stderr) self.__closeComment() self.startCommentSearch(match) @@ -239,7 +241,7 @@ class Doxypy(object): the current indentation. """ if options.debug: - print >>sys.stderr, "# CALLBACK: startCommentSearch" + print("# CALLBACK: startCommentSearch", file=sys.stderr) self.defclass = [self.fsm.current_input] self.comment = [] self.indent = match.group(1) @@ -251,7 +253,7 @@ class Doxypy(object): appends the current line to the output. """ if options.debug: - print >>sys.stderr, "# CALLBACK: stopCommentSearch" + print("# CALLBACK: stopCommentSearch" , file=sys.stderr) self.__closeComment() self.defclass = [] @@ -263,7 +265,7 @@ class Doxypy(object): Closes the open comment block, resets it and appends the current line. """ if options.debug: - print >>sys.stderr, "# CALLBACK: appendFileheadLine" + print("# CALLBACK: appendFileheadLine" , file=sys.stderr) self.__closeComment() self.comment = [] self.output.append(self.fsm.current_input) @@ -275,7 +277,7 @@ class Doxypy(object): well as singleline comments. """ if options.debug: - print >>sys.stderr, "# CALLBACK: appendCommentLine" + print("# CALLBACK: appendCommentLine" , file=sys.stderr) (from_state, to_state, condition, callback) = self.fsm.current_transition # single line comment @@ -312,13 +314,13 @@ class Doxypy(object): def appendNormalLine(self, match): """Appends a line to the output.""" if options.debug: - print >>sys.stderr, "# CALLBACK: appendNormalLine" + print("# CALLBACK: appendNormalLine" , file=sys.stderr) self.output.append(self.fsm.current_input) def appendDefclassLine(self, match): """Appends a line to the triggering block.""" if options.debug: - print >>sys.stderr, "# CALLBACK: appendDefclassLine" + print("# CALLBACK: appendDefclassLine" , file=sys.stderr) self.defclass.append(self.fsm.current_input) def makeCommentBlock(self): @@ -397,7 +399,7 @@ def optParse(): (options, filename) = parser.parse_args() if not filename: - print >>sys.stderr, "No filename given." + print("No filename given.", file=sys.stderr) sys.exit(-1) return filename[0] diff --git a/src/third_party/wiredtiger/src/docs/tools/fixlinks.py b/src/third_party/wiredtiger/src/docs/tools/fixlinks.py index 59f8494ada3..1532118cd21 100755 --- a/src/third_party/wiredtiger/src/docs/tools/fixlinks.py +++ b/src/third_party/wiredtiger/src/docs/tools/fixlinks.py @@ -59,8 +59,8 @@ def process(source): (m.group(0), m.group(1), m.group(1), m.group(2))), source) # Replace "self, handle" with "self" -- these are typedef'ed away - source = re.sub(r'(\s+#.*self), - (?:connection|cursor|session)', r'\1', source) + source = re.sub(r'(\s+#.*self),' + + r'(?:connection|cursor|session)', r'\1', source) return source if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index dc1bdc07419..c8f2221daa2 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -133,7 +133,7 @@ __wt_page_header_byteswap(WT_PAGE_HEADER *dsk) */ struct __wt_addr { wt_timestamp_t oldest_start_ts; /* Aggregated timestamp information */ - wt_timestamp_t newest_start_ts; + wt_timestamp_t newest_durable_ts; wt_timestamp_t newest_stop_ts; uint8_t *addr; /* Block-manager's cookie */ @@ -990,8 +990,6 @@ struct __wt_col { * of a base pointer. The on-page data is a WT_CELL (same as row-store * pages). * - * If the value is 0, it's a single, deleted record. - * * Obscure the field name, code shouldn't use WT_COL->__col_value, the * public interface is WT_COL_PTR and WT_COL_PTR_SET. */ @@ -1004,8 +1002,7 @@ struct __wt_col { * not exist on the page, return a NULL.) */ #define WT_COL_PTR(page, cip) \ - ((cip)->__col_value == 0 ? \ - NULL : WT_PAGE_REF_OFFSET(page, (cip)->__col_value)) + WT_PAGE_REF_OFFSET(page, (cip)->__col_value) #define WT_COL_PTR_SET(cip, value) \ (cip)->__col_value = (value) diff --git a/src/third_party/wiredtiger/src/include/cell.i b/src/third_party/wiredtiger/src/include/cell.i index 0bbe3283dee..260e2304034 100644 --- a/src/third_party/wiredtiger/src/include/cell.i +++ b/src/third_party/wiredtiger/src/include/cell.i @@ -147,7 +147,7 @@ struct __wt_cell_unpack { /* Start/stop timestamps for a value */ wt_timestamp_t start_ts, stop_ts; /* Aggregated timestamp information */ - wt_timestamp_t oldest_start_ts, newest_start_ts, newest_stop_ts; + wt_timestamp_t oldest_start_ts, newest_durable_ts, newest_stop_ts; /* * !!! @@ -219,37 +219,50 @@ __cell_pack_timestamp_value(WT_SESSION_IMPL *session, */ static inline void __wt_timestamp_addr_check(WT_SESSION_IMPL *session, - wt_timestamp_t oldest_start_ts, - wt_timestamp_t newest_start_ts, wt_timestamp_t newest_stop_ts) + wt_timestamp_t oldest_start_ts, wt_timestamp_t newest_stop_ts) { +#ifdef HAVE_DIAGNOSTIC + char ts_string[2][WT_TS_INT_STRING_SIZE]; + + if (newest_stop_ts == WT_TS_NONE) { + __wt_errx(session, "newest stop timestamp of 0"); + WT_ASSERT(session, newest_stop_ts != WT_TS_NONE); + } + if (oldest_start_ts > newest_stop_ts) { + __wt_timestamp_to_string(oldest_start_ts, ts_string[0]); + __wt_timestamp_to_string(newest_stop_ts, ts_string[1]); + __wt_errx(session, + "an oldest start timestamp %s newer than its newest " + "stop timestamp %s", + ts_string[0], ts_string[1]); + WT_ASSERT(session, oldest_start_ts <= newest_stop_ts); + } +#else + WT_UNUSED(session); WT_UNUSED(oldest_start_ts); - WT_UNUSED(newest_start_ts); WT_UNUSED(newest_stop_ts); - - WT_ASSERT(session, newest_stop_ts != WT_TS_NONE); - WT_ASSERT(session, oldest_start_ts <= newest_start_ts); - WT_ASSERT(session, newest_start_ts <= newest_stop_ts); +#endif } /* * __cell_pack_timestamp_addr -- - * Pack a oldest_start, newest_start, newest_stop timestamp triplet for an - * address. + * Pack a oldest_start, newest_durable_ts, newest_stop timestamp triplet + * for an address. */ static inline void __cell_pack_timestamp_addr(WT_SESSION_IMPL *session, uint8_t **pp, wt_timestamp_t oldest_start_ts, - wt_timestamp_t newest_start_ts, wt_timestamp_t newest_stop_ts) + wt_timestamp_t newest_durable_ts, wt_timestamp_t newest_stop_ts) { - __wt_timestamp_addr_check(session, - oldest_start_ts, newest_start_ts, newest_stop_ts); + __wt_timestamp_addr_check(session, oldest_start_ts, newest_stop_ts); ++*pp; if (__wt_process.page_version_ts) { /* Store differences, not absolutes. */ (void)__wt_vpack_uint(pp, 0, oldest_start_ts); - (void)__wt_vpack_uint(pp, 0, newest_start_ts - oldest_start_ts); - (void)__wt_vpack_uint(pp, 0, newest_stop_ts - newest_start_ts); + (void)__wt_vpack_uint( + pp, 0, newest_durable_ts - oldest_start_ts); + (void)__wt_vpack_uint(pp, 0, newest_stop_ts - oldest_start_ts); } } @@ -260,8 +273,8 @@ __cell_pack_timestamp_addr(WT_SESSION_IMPL *session, static inline size_t __wt_cell_pack_addr(WT_SESSION_IMPL *session, WT_CELL *cell, u_int cell_type, uint64_t recno, - wt_timestamp_t oldest_start_ts, - wt_timestamp_t newest_start_ts, wt_timestamp_t newest_stop_ts, size_t size) + wt_timestamp_t oldest_start_ts, wt_timestamp_t newest_durable_ts, + wt_timestamp_t newest_stop_ts, size_t size) { uint8_t *p; @@ -270,7 +283,7 @@ __wt_cell_pack_addr(WT_SESSION_IMPL *session, *p = '\0'; __cell_pack_timestamp_addr(session, - &p, oldest_start_ts, newest_start_ts, newest_stop_ts); + &p, oldest_start_ts, newest_durable_ts, newest_stop_ts); if (recno == WT_RECNO_OOB) cell->__chunk[0] = (uint8_t)cell_type; /* Type */ @@ -728,10 +741,11 @@ __wt_cell_unpack_safe(WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, * the copied cell must be available from unpack after we return, as our * caller has no way to find the copied cell). */ - WT_CELL_LEN_CHK(cell, 0); unpack->cell = cell; restart: + WT_CELL_LEN_CHK(cell, 0); + /* * This path is performance critical for read-only trees, we're parsing * on-page structures. For that reason we don't clear the unpacked cell @@ -742,7 +756,7 @@ restart: unpack->v = 0; unpack->start_ts = WT_TS_NONE; unpack->stop_ts = WT_TS_MAX; - unpack->oldest_start_ts = unpack->newest_start_ts = WT_TS_NONE; + unpack->oldest_start_ts = unpack->newest_durable_ts = WT_TS_NONE; unpack->newest_stop_ts = WT_TS_MAX; unpack->raw = (uint8_t)__wt_cell_type_raw(cell); unpack->type = (uint8_t)__wt_cell_type(cell); @@ -798,15 +812,14 @@ restart: WT_RET(__wt_vunpack_uint(&p, end == NULL ? 0 : WT_PTRDIFF(end, p), &unpack->oldest_start_ts)); WT_RET(__wt_vunpack_uint(&p, end == NULL ? 0 : - WT_PTRDIFF(end, p), &unpack->newest_start_ts)); - unpack->newest_start_ts += unpack->oldest_start_ts; + WT_PTRDIFF(end, p), &unpack->newest_durable_ts)); + unpack->newest_durable_ts += unpack->oldest_start_ts; WT_RET(__wt_vunpack_uint(&p, end == NULL ? 0 : WT_PTRDIFF(end, p), &unpack->newest_stop_ts)); - unpack->newest_stop_ts += unpack->newest_start_ts; + unpack->newest_stop_ts += unpack->oldest_start_ts; __wt_timestamp_addr_check(session, - unpack->oldest_start_ts, - unpack->newest_start_ts, unpack->newest_stop_ts); + unpack->oldest_start_ts, unpack->newest_stop_ts); break; case WT_CELL_DEL: case WT_CELL_VALUE: @@ -950,7 +963,7 @@ __wt_cell_unpack_dsk(WT_SESSION_IMPL *session, * somewhere. * unpack->oldest_start_ts - unpack->newest_start_ts + unpack->newest_durable_ts unpack->newest_stop_ts */ unpack->data = ""; diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index b6100ae134d..1f461a06137 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -148,6 +148,16 @@ struct __wt_named_extractor { } while (0) /* + * WT_CONN_HOTBACKUP_START -- + * Macro to set connection data appropriately for when we commence hot + * backup. + */ +#define WT_CONN_HOTBACKUP_START(conn) do { \ + conn->hot_backup = true; \ + conn->hot_backup_list = NULL; \ +} while (0) + +/* * WT_CONNECTION_IMPL -- * Implementation of WT_CONNECTION */ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 1ca81b0b4d9..d93deb0a361 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -318,7 +318,7 @@ extern int __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT extern int __wt_json_alloc_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, WT_CURSOR_JSON *json, bool iskey, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor); extern size_t __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern void __wt_json_column_init(WT_CURSOR *cursor, const char *uri, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf); +extern int __wt_json_column_init(WT_CURSOR *cursor, const char *uri, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, const char **tokstart, size_t *toklen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_json_tokname(int toktype) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_json_to_item(WT_SESSION_IMPL *session, const char *jstr, const char *format, WT_CURSOR_JSON *json, bool iskey, WT_ITEM *item) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -572,6 +572,7 @@ extern int __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp) WT_GCC_FUNC_DECL_AT extern bool __wt_fsync_background_chk(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_fsync_background(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_close_connection_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_file_zero(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t start_off, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_os_inmemory(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_fopen(WT_SESSION_IMPL *session, const char *name, uint32_t open_flags, uint32_t flags, WT_FSTREAM **fstrp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_os_stdio(WT_SESSION_IMPL *session); @@ -828,7 +829,7 @@ extern int __wt_verbose_dump_txn(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTR extern int __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_checkpoint_progress(WT_SESSION_IMPL *session, bool closing); extern int __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[], bool waiting) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_checkpoint_tree_reconcile_update(WT_SESSION_IMPL *session, wt_timestamp_t oldest_start_ts, wt_timestamp_t newest_start_ts, wt_timestamp_t newest_stop_ts); +extern void __wt_checkpoint_tree_reconcile_update(WT_SESSION_IMPL *session, wt_timestamp_t oldest_start_ts, wt_timestamp_t newest_durable_ts, wt_timestamp_t newest_stop_ts); extern int __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint_sync(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -857,6 +858,7 @@ extern void __wt_timestamp_to_hex_string(wt_timestamp_t ts, char *hex_timestamp) extern void __wt_verbose_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t ts, const char *msg); extern int __wt_txn_parse_timestamp_raw(WT_SESSION_IMPL *session, const char *name, wt_timestamp_t *timestamp, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_parse_timestamp(WT_SESSION_IMPL *session, const char *name, wt_timestamp_t *timestamp, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_get_pinned_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t *tsp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_query_timestamp(WT_SESSION_IMPL *session, char *hex_timestamp, const char *cfg[], bool global_txn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index b1f9a557934..23c30e9a031 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -77,7 +77,7 @@ struct __wt_ckpt { uint64_t write_gen; /* Write generation */ wt_timestamp_t oldest_start_ts; /* Aggregated timestamp information */ - wt_timestamp_t newest_start_ts; + wt_timestamp_t newest_durable_ts; wt_timestamp_t newest_stop_ts; void *bpriv; /* Block manager private */ diff --git a/src/third_party/wiredtiger/src/include/schema.h b/src/third_party/wiredtiger/src/include/schema.h index 74af41132f2..cd217fe9c51 100644 --- a/src/third_party/wiredtiger/src/include/schema.h +++ b/src/third_party/wiredtiger/src/include/schema.h @@ -83,6 +83,9 @@ struct __wt_table { #define WT_SESSION_LOCKED_TABLE \ (WT_SESSION_LOCKED_TABLE_READ | \ WT_SESSION_LOCKED_TABLE_WRITE) +#define WT_SESSION_LOCKED_HOTBACKUP \ + (WT_SESSION_LOCKED_HOTBACKUP_READ | \ + WT_SESSION_LOCKED_HOTBACKUP_WRITE) /* * WT_WITH_LOCK_WAIT -- @@ -257,6 +260,76 @@ struct __wt_table { } while (0) /* + * WT_WITH_HOTBACKUP_READ_LOCK -- + * Acquire the hot backup read lock and perform an operation provided that + * there is no hot backup in progress. The skipp parameter can be used to + * check whether the operation got skipped or not. + */ +#define WT_WITH_HOTBACKUP_READ_LOCK(session, op, skipp) do { \ + WT_CONNECTION_IMPL *__conn = S2C(session); \ + if ((skipp) != (bool *)NULL) \ + *(bool *)(skipp) = true; \ + if (F_ISSET(session, WT_SESSION_LOCKED_HOTBACKUP)) { \ + if (!__conn->hot_backup) { \ + if ((skipp) != (bool *)NULL) \ + *(bool *)(skipp) = false; \ + op; \ + } \ + } else { \ + __wt_readlock(session, &__conn->hot_backup_lock); \ + F_SET(session, WT_SESSION_LOCKED_HOTBACKUP_READ); \ + if (!__conn->hot_backup) { \ + if ((skipp) != (bool *)NULL) \ + *(bool *)(skipp) = false; \ + op; \ + } \ + F_CLR(session, WT_SESSION_LOCKED_HOTBACKUP_READ); \ + __wt_readunlock(session, &__conn->hot_backup_lock); \ + } \ +} while (0) + +/* + * WT_WITH_HOTBACKUP_WRITE_LOCK -- + * Acquire the hot backup write lock and perform an operation. + */ +#define WT_WITH_HOTBACKUP_WRITE_LOCK(session, op) do { \ + WT_CONNECTION_IMPL *__conn = S2C(session); \ + if (F_ISSET(session, WT_SESSION_LOCKED_HOTBACKUP_WRITE)) { \ + op; \ + } else { \ + WT_ASSERT(session, \ + !F_ISSET( \ + session, WT_SESSION_LOCKED_HOTBACKUP_READ)); \ + __wt_writelock(session, &__conn->hot_backup_lock); \ + F_SET(session, WT_SESSION_LOCKED_HOTBACKUP_WRITE); \ + op; \ + F_CLR(session, WT_SESSION_LOCKED_HOTBACKUP_WRITE); \ + __wt_writeunlock(session, &__conn->hot_backup_lock); \ + } \ +} while (0) + +/* + * WT_WITH_HOTBACKUP_READ_LOCK_UNCOND -- + * Acquire the hot backup read lock and perform an operation + * unconditionally. This is a specialized macro for a few isolated cases. + * Code that wishes to acquire the read lock should default to using + * WT_WITH_HOTBACKUP_READ_LOCK which checks that there is no hot backup in + * progress. + */ +#define WT_WITH_HOTBACKUP_READ_LOCK_UNCOND(session, op) do { \ + WT_CONNECTION_IMPL *__conn = S2C(session); \ + if (F_ISSET(session, WT_SESSION_LOCKED_HOTBACKUP)) { \ + op; \ + } else { \ + __wt_readlock(session, &__conn->hot_backup_lock); \ + F_SET(session, WT_SESSION_LOCKED_HOTBACKUP_READ); \ + op; \ + F_CLR(session, WT_SESSION_LOCKED_HOTBACKUP_READ); \ + __wt_readunlock(session, &__conn->hot_backup_lock); \ + } \ +} while (0) + +/* * WT_WITHOUT_LOCKS -- * Drop the handle, table and/or schema locks, perform an operation, * re-acquire the lock(s). diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 0d99b4cc6e0..c7ae31b4e54 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -172,23 +172,25 @@ struct __wt_session_impl { #define WT_SESSION_LOCKED_CHECKPOINT 0x0000040u #define WT_SESSION_LOCKED_HANDLE_LIST_READ 0x0000080u #define WT_SESSION_LOCKED_HANDLE_LIST_WRITE 0x0000100u -#define WT_SESSION_LOCKED_METADATA 0x0000200u -#define WT_SESSION_LOCKED_PASS 0x0000400u -#define WT_SESSION_LOCKED_SCHEMA 0x0000800u -#define WT_SESSION_LOCKED_SLOT 0x0001000u -#define WT_SESSION_LOCKED_TABLE_READ 0x0002000u -#define WT_SESSION_LOCKED_TABLE_WRITE 0x0004000u -#define WT_SESSION_LOCKED_TURTLE 0x0008000u -#define WT_SESSION_LOGGING_INMEM 0x0010000u -#define WT_SESSION_LOOKASIDE_CURSOR 0x0020000u -#define WT_SESSION_NO_DATA_HANDLES 0x0040000u -#define WT_SESSION_NO_LOGGING 0x0080000u -#define WT_SESSION_NO_RECONCILE 0x0100000u -#define WT_SESSION_NO_SCHEMA_LOCK 0x0200000u -#define WT_SESSION_QUIET_CORRUPT_FILE 0x0400000u -#define WT_SESSION_READ_WONT_NEED 0x0800000u -#define WT_SESSION_SCHEMA_TXN 0x1000000u -#define WT_SESSION_SERVER_ASYNC 0x2000000u +#define WT_SESSION_LOCKED_HOTBACKUP_READ 0x0000200u +#define WT_SESSION_LOCKED_HOTBACKUP_WRITE 0x0000400u +#define WT_SESSION_LOCKED_METADATA 0x0000800u +#define WT_SESSION_LOCKED_PASS 0x0001000u +#define WT_SESSION_LOCKED_SCHEMA 0x0002000u +#define WT_SESSION_LOCKED_SLOT 0x0004000u +#define WT_SESSION_LOCKED_TABLE_READ 0x0008000u +#define WT_SESSION_LOCKED_TABLE_WRITE 0x0010000u +#define WT_SESSION_LOCKED_TURTLE 0x0020000u +#define WT_SESSION_LOGGING_INMEM 0x0040000u +#define WT_SESSION_LOOKASIDE_CURSOR 0x0080000u +#define WT_SESSION_NO_DATA_HANDLES 0x0100000u +#define WT_SESSION_NO_LOGGING 0x0200000u +#define WT_SESSION_NO_RECONCILE 0x0400000u +#define WT_SESSION_NO_SCHEMA_LOCK 0x0800000u +#define WT_SESSION_QUIET_CORRUPT_FILE 0x1000000u +#define WT_SESSION_READ_WONT_NEED 0x2000000u +#define WT_SESSION_SCHEMA_TXN 0x4000000u +#define WT_SESSION_SERVER_ASYNC 0x8000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index f5d1075581b..e2b2aae3d33 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -708,7 +708,9 @@ struct __wt_connection_stats { int64_t txn_pinned_snapshot_range; int64_t txn_pinned_timestamp; int64_t txn_pinned_timestamp_checkpoint; + int64_t txn_pinned_timestamp_reader; int64_t txn_pinned_timestamp_oldest; + int64_t txn_timestamp_oldest_active_read; int64_t txn_sync; int64_t txn_commit; int64_t txn_rollback; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 7fed51cc76b..c60b1772fe9 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -23,6 +23,12 @@ #define WT_TXN_OLDEST_WAIT 0x2u /* AUTOMATIC FLAG VALUE GENERATION STOP */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_TXN_TS_ALREADY_LOCKED 0x1u +#define WT_TXN_TS_INCLUDE_CKPT 0x2u +#define WT_TXN_TS_INCLUDE_OLDEST 0x4u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + /* * Transaction ID comparison dealing with edge cases. * diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 5a091db45a0..890fbb26a74 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1791,9 +1791,6 @@ struct __wt_session { * @config{read_timestamp, read using the specified timestamp. The * supplied value must not be older than the current oldest timestamp. * See @ref transaction_timestamps., a string; default empty.} - * @config{round_to_oldest, if read timestamp is earlier than oldest - * timestamp\, read timestamp will be rounded to oldest timestamp., a - * boolean flag; default \c false.} * @config{roundup_timestamps = (, round up timestamps of the * transaction. This setting alters the visibility expected in a * transaction. See @ref transaction_timestamps., a set of related @@ -1929,9 +1926,6 @@ struct __wt_session { * supplied value must not be older than the current oldest timestamp. * This can only be set once for a transaction. See @ref * transaction_timestamps., a string; default empty.} - * @config{round_to_oldest, if read timestamp is earlier than oldest - * timestamp\, read timestamp will be rounded to oldest timestamp., a - * boolean flag; default \c false.} * @configend * @errors */ @@ -5805,17 +5799,24 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1383 /*! * transaction: transaction range of timestamps pinned by the oldest + * active read timestamp + */ +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1384 +/*! + * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1384 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1385 +/*! transaction: transaction read timestamp of the oldest active reader */ +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1386 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1385 +#define WT_STAT_CONN_TXN_SYNC 1387 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1386 +#define WT_STAT_CONN_TXN_COMMIT 1388 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1387 +#define WT_STAT_CONN_TXN_ROLLBACK 1389 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1388 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1390 /*! * @} diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 10b52246987..1dc6c60a137 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -629,68 +629,6 @@ err: WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); } /* - * __log_zero -- - * Zero a log file. - */ -static int -__log_zero(WT_SESSION_IMPL *session, - WT_FH *fh, wt_off_t start_off, wt_off_t len) -{ - WT_CONNECTION_IMPL *conn; - WT_DECL_ITEM(zerobuf); - WT_DECL_RET; - WT_LOG *log; - uint32_t allocsize, bufsz, off, partial, wrlen; - - conn = S2C(session); - log = conn->log; - allocsize = log->allocsize; - zerobuf = NULL; - if (allocsize < WT_MEGABYTE) - bufsz = WT_MEGABYTE; - else - bufsz = allocsize; - /* - * If they're using smaller log files, cap it at the file size. - */ - if (conn->log_file_max < bufsz) - bufsz = (uint32_t)conn->log_file_max; - WT_RET(__wt_scr_alloc(session, bufsz, &zerobuf)); - memset(zerobuf->mem, 0, zerobuf->memsize); - WT_STAT_CONN_INCR(session, log_zero_fills); - - /* - * Read in a chunk starting at the end of the file. Keep going until - * we reach the beginning or we find a chunk that contains any non-zero - * bytes. Compare against a known zero byte chunk. - */ - off = (uint32_t)start_off; - while (off < (uint32_t)len) { - /* - * Typically we start to zero the file after the log header - * and the bufsz is a sector-aligned size. So we want to - * align our writes when we can. - */ - partial = off % bufsz; - if (partial != 0) - wrlen = bufsz - partial; - else - wrlen = bufsz; - /* - * Check if we're writing a partial amount at the end too. - */ - if ((uint32_t)len - off < bufsz) - wrlen = (uint32_t)len - off; - __wt_capacity_throttle(session, wrlen, WT_THROTTLE_LOG); - WT_ERR(__wt_write(session, - fh, (wt_off_t)off, wrlen, zerobuf->mem)); - off += wrlen; - } -err: __wt_scr_free(session, &zerobuf); - return (ret); -} - -/* * __log_prealloc -- * Pre-allocate a log file. */ @@ -710,7 +648,7 @@ __log_prealloc(WT_SESSION_IMPL *session, WT_FH *fh) * and zero the log file based on what is available. */ if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ZERO_FILL)) - return (__log_zero(session, fh, + return (__wt_file_zero(session, fh, log->first_record, conn->log_file_max)); /* If configured to not extend the file, we're done. */ @@ -1235,7 +1173,7 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) WT_LOG *log; WT_LSN end_lsn, logrec_lsn; u_int yield_cnt; - bool create_log; + bool create_log, skipp; conn = S2C(session); log = conn->log; @@ -1284,13 +1222,11 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) */ create_log = true; if (conn->log_prealloc > 0 && !conn->hot_backup) { - __wt_readlock(session, &conn->hot_backup_lock); - if (conn->hot_backup) - __wt_readunlock(session, &conn->hot_backup_lock); - else { - ret = __log_alloc_prealloc(session, log->fileid); - __wt_readunlock(session, &conn->hot_backup_lock); + WT_WITH_HOTBACKUP_READ_LOCK(session, + ret = __log_alloc_prealloc(session, log->fileid), + &skipp); + if (!skipp) { /* * If ret is 0 it means we found a pre-allocated file. * If ret is WT_NOTFOUND, create the new log file and @@ -1517,24 +1453,23 @@ __log_truncate_file(WT_SESSION_IMPL *session, WT_FH *log_fh, wt_off_t offset) WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_LOG *log; + bool skipp; conn = S2C(session); log = conn->log; if (!F_ISSET(log, WT_LOG_TRUNCATE_NOTSUP) && !conn->hot_backup) { - __wt_readlock(session, &conn->hot_backup_lock); - if (conn->hot_backup) - __wt_readunlock(session, &conn->hot_backup_lock); - else { - ret = __wt_ftruncate(session, log_fh, offset); - __wt_readunlock(session, &conn->hot_backup_lock); + WT_WITH_HOTBACKUP_READ_LOCK(session, + ret = __wt_ftruncate( + session, log_fh, offset), &skipp); + if (!skipp) { if (ret != ENOTSUP) return (ret); F_SET(log, WT_LOG_TRUNCATE_NOTSUP); } } - return (__log_zero(session, log_fh, offset, conn->log_file_max)); + return (__wt_file_zero(session, log_fh, offset, conn->log_file_max)); } /* diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index 58711cc4e92..c4eb01b2d39 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -352,16 +352,16 @@ __ckpt_load(WT_SESSION_IMPL *session, WT_RET_NOTFOUND_OK(ret); ckpt->oldest_start_ts = ret == WT_NOTFOUND || a.len == 0 ? WT_TS_NONE : (uint64_t)a.val; - ret = __wt_config_subgets(session, v, "newest_start_ts", &a); + ret = __wt_config_subgets(session, v, "newest_durable_ts", &a); WT_RET_NOTFOUND_OK(ret); - ckpt->newest_start_ts = + ckpt->newest_durable_ts = ret == WT_NOTFOUND || a.len == 0 ? WT_TS_NONE : (uint64_t)a.val; ret = __wt_config_subgets(session, v, "newest_stop_ts", &a); WT_RET_NOTFOUND_OK(ret); ckpt->newest_stop_ts = ret == WT_NOTFOUND || a.len == 0 ? WT_TS_MAX : (uint64_t)a.val; __wt_timestamp_addr_check(session, - ckpt->oldest_start_ts, ckpt->newest_start_ts, ckpt->newest_stop_ts); + ckpt->oldest_start_ts, ckpt->newest_stop_ts); WT_RET(__wt_config_subgets(session, v, "write_gen", &a)); if (a.len == 0) @@ -433,8 +433,8 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, __wt_seconds(session, &ckpt->sec); } - __wt_timestamp_addr_check(session, ckpt->oldest_start_ts, - ckpt->newest_start_ts, ckpt->newest_stop_ts); + __wt_timestamp_addr_check(session, + ckpt->oldest_start_ts, ckpt->newest_stop_ts); WT_ERR(__wt_buf_catfmt(session, buf, "%s%s", sep, ckpt->name)); sep = ","; @@ -452,7 +452,7 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, ",time=%" PRIu64 ",size=%" PRId64 ",oldest_start_ts=%" PRId64 - ",newest_start_ts=%" PRId64 + ",newest_durable_ts=%" PRId64 ",newest_stop_ts=%" PRId64 ",write_gen=%" PRId64 ")", (int)ckpt->addr.size, (char *)ckpt->addr.data, @@ -460,7 +460,7 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, ckpt->sec, (int64_t)ckpt->size, (int64_t)ckpt->oldest_start_ts, - (int64_t)ckpt->newest_start_ts, + (int64_t)ckpt->newest_durable_ts, (int64_t)ckpt->newest_stop_ts, (int64_t)ckpt->write_gen)); } diff --git a/src/third_party/wiredtiger/src/os_common/os_fhandle.c b/src/third_party/wiredtiger/src/os_common/os_fhandle.c index df67508c4fe..ca2fe730444 100644 --- a/src/third_party/wiredtiger/src/os_common/os_fhandle.c +++ b/src/third_party/wiredtiger/src/os_common/os_fhandle.c @@ -500,3 +500,53 @@ __wt_close_connection_close(WT_SESSION_IMPL *session) } WT_TAILQ_SAFE_REMOVE_END return (ret); } + +/* + * __wt_file_zero -- + * Zero out the file from offset for size bytes. + */ +int +__wt_file_zero(WT_SESSION_IMPL *session, + WT_FH *fh, wt_off_t start_off, wt_off_t size) +{ + WT_DECL_ITEM(zerobuf); + WT_DECL_RET; + WT_THROTTLE_TYPE type; + uint64_t bufsz, off, partial, wrlen; + + zerobuf = NULL; + bufsz = WT_MIN((uint64_t)size, WT_MEGABYTE); + /* + * For now logging is the only type and statistic. This needs + * updating if block manager decides to use this function. + */ + type = WT_THROTTLE_LOG; + WT_STAT_CONN_INCR(session, log_zero_fills); + WT_RET(__wt_scr_alloc(session, bufsz, &zerobuf)); + memset(zerobuf->mem, 0, zerobuf->memsize); + off = (uint64_t)start_off; + while (off < (uint64_t)size) { + /* + * We benefit from aligning our writes when we can. Log files + * will typically want to start to zero after the log header + * and the bufsz is a sector-aligned size. So align when + * we can. + */ + partial = off % bufsz; + if (partial != 0) + wrlen = bufsz - partial; + else + wrlen = bufsz; + /* + * Check if we're writing a partial amount at the end too. + */ + if ((uint64_t)size - off < bufsz) + wrlen = (uint64_t)size - off; + __wt_capacity_throttle(session, wrlen, type); + WT_ERR(__wt_write(session, + fh, (wt_off_t)off, (size_t)wrlen, zerobuf->mem)); + off += wrlen; + } +err: __wt_scr_free(session, &zerobuf); + return (ret); +} diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index f25ada93885..90db828b1a5 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -151,7 +151,7 @@ typedef struct { uint64_t recno; WT_ITEM key; wt_timestamp_t oldest_start_ts; - wt_timestamp_t newest_start_ts; + wt_timestamp_t newest_durable_ts; wt_timestamp_t newest_stop_ts; /* Saved minimum split-size boundary information. */ @@ -159,7 +159,7 @@ typedef struct { uint64_t min_recno; WT_ITEM min_key; wt_timestamp_t min_oldest_start_ts; - wt_timestamp_t min_newest_start_ts; + wt_timestamp_t min_newest_durable_ts; wt_timestamp_t min_newest_stop_ts; size_t min_offset; /* byte offset */ @@ -279,7 +279,7 @@ typedef struct { WT_UPDATE *upd; /* Update to write (or NULL) */ uint64_t txnid; /* Transaction ID, timestamps */ - wt_timestamp_t start_ts, stop_ts; + wt_timestamp_t start_ts, durable_ts, stop_ts; bool upd_saved; /* Updates saved to list */ @@ -318,7 +318,7 @@ static int __rec_las_wrapup_err(WT_SESSION_IMPL *, WT_RECONCILE *); static int __rec_root_write(WT_SESSION_IMPL *, WT_PAGE *, uint32_t); static int __rec_row_int(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); static int __rec_row_leaf(WT_SESSION_IMPL *, - WT_RECONCILE *, WT_PAGE *, WT_SALVAGE_COOKIE *); + WT_RECONCILE *, WT_REF *, WT_SALVAGE_COOKIE *); static int __rec_row_leaf_insert( WT_SESSION_IMPL *, WT_RECONCILE *, WT_INSERT *); static int __rec_row_merge(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); @@ -469,7 +469,7 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, ret = __rec_row_int(session, r, page)); break; case WT_PAGE_ROW_LEAF: - ret = __rec_row_leaf(session, r, page, salvage); + ret = __rec_row_leaf(session, r, ref, salvage); break; default: ret = __wt_illegal_value(session, page->type); @@ -1186,6 +1186,7 @@ __rec_append_orig_value(WT_SESSION_IMPL *session, append->txnid = upd->txnid; append->start_ts = upd->start_ts; append->durable_ts = upd->durable_ts; + append->stop_ts = upd->stop_ts; append->next = upd->next; } @@ -1396,15 +1397,16 @@ __rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, * TIMESTAMP-FIXME * This is waiting on the WT_UPDATE structure's start/stop * timestamp work. For now, if we don't have a timestamp, - * just pretend it's durable, otherwise pretend the start - * and stop timestamps are the same. + * just pretend it's durable, otherwise pretend the durable, + * start and stop timestamps are all the same. * */ if (upd_select->upd->start_ts == WT_TS_NONE) { - upd_select->start_ts = WT_TS_NONE; + upd_select->start_ts = + upd_select->durable_ts = WT_TS_NONE; upd_select->stop_ts = WT_TS_MAX; } else - upd_select->start_ts = + upd_select->start_ts = upd_select->durable_ts = upd_select->stop_ts = upd_select->upd->start_ts; /* @@ -1453,7 +1455,7 @@ __rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, * order), so we track the maximum transaction ID and the newest update * with a timestamp (if any). */ - timestamp = first_ts_upd == NULL ? 0 : first_ts_upd->start_ts; + timestamp = first_ts_upd == NULL ? 0 : first_ts_upd->durable_ts; all_visible = upd == first_txn_upd && !(uncommitted || prepared) && (F_ISSET(r, WT_REC_VISIBLE_ALL) ? __wt_txn_visible_all(session, max_txn, timestamp) : @@ -1515,14 +1517,11 @@ __rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, if (WT_TXNID_LT(r->unstable_txn, first_upd->txnid)) r->unstable_txn = first_upd->txnid; if (first_ts_upd != NULL) { - /* - * FIXME Disable this assertion until fixed by WT-4598. - * WT_ASSERT(session, - * first_ts_upd->prepare_state == - * WT_PREPARE_INPROGRESS || - * first_ts_upd->start_ts <= - * first_ts_upd->durable_ts); - */ + WT_ASSERT(session, + first_ts_upd->prepare_state == + WT_PREPARE_INPROGRESS || + first_ts_upd->start_ts <= first_ts_upd->durable_ts); + if (r->unstable_timestamp < first_ts_upd->start_ts) r->unstable_timestamp = first_ts_upd->start_ts; @@ -1545,12 +1544,10 @@ __rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, * to use the two independently and be confident both * will be set. */ - /* - * FIXME Disable this assertion until fixed by WT-4598. - * WT_ASSERT(session, - * upd->prepare_state == WT_PREPARE_INPROGRESS || - * upd->durable_ts >= upd->start_ts); - */ + WT_ASSERT(session, + upd->prepare_state == WT_PREPARE_INPROGRESS || + upd->durable_ts >= upd->start_ts); + if (upd->start_ts < r->unstable_timestamp) r->unstable_timestamp = upd->start_ts; /* @@ -2244,7 +2241,7 @@ __wt_split_page_size(int split_pct, uint32_t maxpagesize, uint32_t allocsize) */ static void __rec_addr_ts_init(WT_RECONCILE *r, wt_timestamp_t *oldest_start_tsp, - wt_timestamp_t *newest_start_tsp, wt_timestamp_t *newest_stop_tsp) + wt_timestamp_t *newest_durable_ts, wt_timestamp_t *newest_stop_tsp) { /* * If the page format supports address timestamps (and not fixed-length @@ -2254,9 +2251,9 @@ __rec_addr_ts_init(WT_RECONCILE *r, wt_timestamp_t *oldest_start_tsp, * the oldest/newest timestamps to simple durability. */ *oldest_start_tsp = WT_TS_MAX; - *newest_start_tsp = *newest_stop_tsp = WT_TS_NONE; + *newest_durable_ts = *newest_stop_tsp = WT_TS_NONE; if (!__wt_process.page_version_ts || r->page->type == WT_PAGE_COL_FIX) { - *oldest_start_tsp = *newest_start_tsp = WT_TS_NONE; + *oldest_start_tsp = *newest_durable_ts = WT_TS_NONE; *newest_stop_tsp = WT_TS_MAX; } } @@ -2267,12 +2264,12 @@ __rec_addr_ts_init(WT_RECONCILE *r, wt_timestamp_t *oldest_start_tsp, */ static inline void __rec_addr_ts_update(WT_RECONCILE *r, wt_timestamp_t oldest_start_ts, - wt_timestamp_t newest_start_ts, wt_timestamp_t newest_stop_ts) + wt_timestamp_t newest_durable_ts, wt_timestamp_t newest_stop_ts) { r->cur_ptr->oldest_start_ts = WT_MIN(oldest_start_ts, r->cur_ptr->oldest_start_ts); - r->cur_ptr->newest_start_ts = - WT_MAX(newest_start_ts, r->cur_ptr->newest_start_ts); + r->cur_ptr->newest_durable_ts = + WT_MAX(newest_durable_ts, r->cur_ptr->newest_durable_ts); r->cur_ptr->newest_stop_ts = WT_MAX(newest_stop_ts, r->cur_ptr->newest_stop_ts); } @@ -2290,14 +2287,14 @@ __rec_split_chunk_init( chunk->key.size = 0; chunk->entries = 0; __rec_addr_ts_init(r, &chunk->oldest_start_ts, - &chunk->newest_start_ts, &chunk->newest_stop_ts); + &chunk->newest_durable_ts, &chunk->newest_stop_ts); chunk->min_recno = WT_RECNO_OOB; /* Don't touch the key item memory, that memory is reused. */ chunk->min_key.size = 0; chunk->min_entries = 0; __rec_addr_ts_init(r, &chunk->min_oldest_start_ts, - &chunk->min_newest_start_ts, &chunk->min_newest_stop_ts); + &chunk->min_newest_durable_ts, &chunk->min_newest_stop_ts); chunk->min_offset = 0; /* @@ -2760,7 +2757,8 @@ __rec_split_crossing_bnd( WT_RET(__rec_split_row_promote( session, r, &r->cur_ptr->min_key, r->page->type)); r->cur_ptr->min_oldest_start_ts = r->cur_ptr->oldest_start_ts; - r->cur_ptr->min_newest_start_ts = r->cur_ptr->newest_start_ts; + r->cur_ptr->min_newest_durable_ts = + r->cur_ptr->newest_durable_ts; r->cur_ptr->min_newest_stop_ts = r->cur_ptr->newest_stop_ts; /* Assert we're not re-entering this code. */ @@ -2818,8 +2816,9 @@ __rec_split_finish_process_prev(WT_SESSION_IMPL *session, WT_RECONCILE *r) prev_ptr->entries += cur_ptr->entries; prev_ptr->oldest_start_ts = WT_MIN(prev_ptr->oldest_start_ts, cur_ptr->oldest_start_ts); - prev_ptr->newest_start_ts = - WT_MAX(prev_ptr->newest_start_ts, cur_ptr->newest_start_ts); + prev_ptr->newest_durable_ts = + WT_MAX(prev_ptr->newest_durable_ts, + cur_ptr->newest_durable_ts); prev_ptr->newest_stop_ts = WT_MAX(prev_ptr->newest_stop_ts, cur_ptr->newest_stop_ts); dsk = r->cur_ptr->image.mem; @@ -2873,15 +2872,16 @@ __rec_split_finish_process_prev(WT_SESSION_IMPL *session, WT_RECONCILE *r) prev_ptr->min_key.data, prev_ptr->min_key.size)); cur_ptr->oldest_start_ts = WT_MIN(prev_ptr->oldest_start_ts, cur_ptr->oldest_start_ts); - cur_ptr->newest_start_ts = - WT_MAX(prev_ptr->newest_start_ts, cur_ptr->newest_start_ts); + cur_ptr->newest_durable_ts = + WT_MAX(prev_ptr->newest_durable_ts, + cur_ptr->newest_durable_ts); cur_ptr->newest_stop_ts = WT_MAX(prev_ptr->newest_stop_ts, cur_ptr->newest_stop_ts); cur_ptr->image.size += len_to_move; prev_ptr->entries = prev_ptr->min_entries; prev_ptr->oldest_start_ts = prev_ptr->min_oldest_start_ts; - prev_ptr->newest_start_ts = prev_ptr->min_newest_start_ts; + prev_ptr->newest_durable_ts = prev_ptr->min_newest_durable_ts; prev_ptr->newest_stop_ts = prev_ptr->min_newest_stop_ts; prev_ptr->image.size -= len_to_move; } @@ -3040,11 +3040,10 @@ done: if (F_ISSET(r, WT_REC_LOOKASIDE)) { multi->page_las.unstable_txn = r->unstable_txn; WT_ASSERT(session, r->unstable_txn != WT_TXN_NONE); multi->page_las.max_timestamp = r->max_timestamp; - /* - * FIXME Disable this assertion until fixed by WT-4598. - * WT_ASSERT(session, r->all_upd_prepare_in_prog == true || - * r->unstable_durable_timestamp >= r->unstable_timestamp); - */ + + WT_ASSERT(session, r->all_upd_prepare_in_prog == true || + r->unstable_durable_timestamp >= r->unstable_timestamp); + multi->page_las.unstable_timestamp = r->unstable_timestamp; multi->page_las.unstable_durable_timestamp = r->unstable_durable_timestamp; @@ -3297,7 +3296,7 @@ __rec_split_write(WT_SESSION_IMPL *session, WT_RECONCILE *r, /* Initialize the address (set the addr type for the parent). */ multi->addr.oldest_start_ts = chunk->oldest_start_ts; - multi->addr.newest_start_ts = chunk->newest_start_ts; + multi->addr.newest_durable_ts = chunk->newest_durable_ts; multi->addr.newest_stop_ts = chunk->newest_stop_ts; switch (page->type) { @@ -3765,7 +3764,7 @@ __rec_col_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) WT_KV *val; WT_PAGE *child, *page; WT_REF *ref; - wt_timestamp_t oldest_start_ts, newest_start_ts, newest_stop_ts; + wt_timestamp_t oldest_start_ts, newest_durable_ts, newest_stop_ts; bool hazard; btree = S2BT(session); @@ -3854,13 +3853,13 @@ __rec_col_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) val->cell_len = 0; val->len = val->buf.size; oldest_start_ts = vpack->oldest_start_ts; - newest_start_ts = vpack->newest_start_ts; + newest_durable_ts = vpack->newest_durable_ts; newest_stop_ts = vpack->newest_stop_ts; } else { __rec_cell_build_addr( session, r, addr, false, ref->ref_recno); oldest_start_ts = addr->oldest_start_ts; - newest_start_ts = addr->newest_start_ts; + newest_durable_ts = addr->newest_durable_ts; newest_stop_ts = addr->newest_stop_ts; } WT_CHILD_RELEASE_ERR(session, hazard, ref); @@ -3871,8 +3870,8 @@ __rec_col_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) /* Copy the value onto the page. */ __rec_image_copy(session, r, val); - __rec_addr_ts_update( - r, oldest_start_ts, newest_start_ts, newest_stop_ts); + __rec_addr_ts_update(r, + oldest_start_ts, newest_durable_ts, newest_stop_ts); } WT_INTL_FOREACH_END; /* Write the remnant page. */ @@ -3916,7 +3915,7 @@ __rec_col_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) /* Copy the value onto the page. */ __rec_image_copy(session, r, val); __rec_addr_ts_update(r, addr->oldest_start_ts, - addr->newest_start_ts, addr->newest_stop_ts); + addr->newest_durable_ts, addr->newest_stop_ts); } return (0); } @@ -4128,7 +4127,7 @@ __rec_col_fix_slvg(WT_SESSION_IMPL *session, static int __rec_col_var_helper(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_SALVAGE_COOKIE *salvage, WT_ITEM *value, - wt_timestamp_t start_ts, wt_timestamp_t stop_ts, + wt_timestamp_t start_ts, wt_timestamp_t durable_ts ,wt_timestamp_t stop_ts, uint64_t rle, bool deleted, bool overflow_type) { WT_BTREE *btree; @@ -4193,7 +4192,7 @@ __rec_col_var_helper(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_RET(__rec_dict_replace( session, r, start_ts, stop_ts, rle, val)); __rec_image_copy(session, r, val); - __rec_addr_ts_update(r, start_ts, start_ts, stop_ts); + __rec_addr_ts_update(r, start_ts, durable_ts, stop_ts); /* Update the starting record number in case we split. */ r->recno += rle; @@ -4215,6 +4214,7 @@ __rec_col_var(WT_SESSION_IMPL *session, wt_timestamp_t start_ts, stop_ts; /* Timestamps */ bool deleted; /* If deleted */ } last; + WT_ADDR *addr; WT_BTREE *btree; WT_CELL *cell; WT_CELL_UNPACK *vpack, _vpack; @@ -4226,7 +4226,7 @@ __rec_col_var(WT_SESSION_IMPL *session, WT_PAGE *page; WT_UPDATE *upd; WT_UPDATE_SELECT upd_select; - wt_timestamp_t start_ts, stop_ts; + wt_timestamp_t start_ts, durable_ts, newest_durable_ts, stop_ts; uint64_t n, nrepeat, repeat_count, rle, skip, src_recno; uint32_t i, size; bool deleted, orig_deleted, update_no_copy; @@ -4240,9 +4240,24 @@ __rec_col_var(WT_SESSION_IMPL *session, size = 0; data = NULL; + /* + * Acquire the newest-durable timestamp for this page so we can roll it + * forward. If it exists, it's in the WT_REF structure or the parent's + * disk image. + */ + if ((addr = pageref->addr) == NULL) + newest_durable_ts = WT_TS_NONE; + else if (__wt_off_page(pageref->home, addr)) + newest_durable_ts = addr->newest_durable_ts; + else { + __wt_cell_unpack(session, pageref->home, pageref->addr, vpack); + newest_durable_ts = vpack->newest_durable_ts; + } + /* Set the "last" values to cause failure if they're not set. */ last.value = r->last; - last.start_ts = last.stop_ts = WT_TS_NONE; + last.start_ts = WT_TS_MAX; + last.stop_ts = WT_TS_NONE; last.deleted = false; /* @@ -4250,7 +4265,8 @@ __rec_col_var(WT_SESSION_IMPL *session, * [-Werror=maybe-uninitialized] */ /* NOLINTNEXTLINE(clang-analyzer-deadcode.DeadStores) */ - start_ts = stop_ts = WT_TS_NONE; + start_ts = WT_TS_MAX; + durable_ts = stop_ts = WT_TS_NONE; WT_RET(__rec_split_init(session, r, page, pageref->ref_recno, btree->maxleafpage_precomp)); @@ -4283,7 +4299,7 @@ __rec_col_var(WT_SESSION_IMPL *session, salvage->take += salvage->missing; } else WT_ERR(__rec_col_var_helper(session, r, - NULL, NULL, WT_TS_NONE, WT_TS_MAX, + NULL, NULL, WT_TS_NONE, WT_TS_NONE, WT_TS_MAX, salvage->missing, true, false)); } @@ -4303,58 +4319,53 @@ __rec_col_var(WT_SESSION_IMPL *session, /* For each entry in the in-memory page... */ WT_COL_FOREACH(page, cip, i) { ovfl_state = OVFL_IGNORE; - if ((cell = WT_COL_PTR(page, cip)) == NULL) { - nrepeat = 1; - ins = NULL; - orig_deleted = true; - } else { - __wt_cell_unpack(session, page, cell, vpack); - nrepeat = __wt_cell_rle(vpack); - ins = WT_SKIP_FIRST(WT_COL_UPDATE(page, cip)); - - /* - * If the original value is "deleted", there's no value - * to compare, we're done. - */ - orig_deleted = vpack->type == WT_CELL_DEL; - if (orig_deleted) - goto record_loop; + cell = WT_COL_PTR(page, cip); + __wt_cell_unpack(session, page, cell, vpack); + nrepeat = __wt_cell_rle(vpack); + ins = WT_SKIP_FIRST(WT_COL_UPDATE(page, cip)); - /* - * Overflow items are tricky: we don't know until we're - * finished processing the set of values if we need the - * overflow value or not. If we don't use the overflow - * item at all, we have to discard it from the backing - * file, otherwise we'll leak blocks on the checkpoint. - * That's safe because if the backing overflow value is - * still needed by any running transaction, we'll cache - * a copy in the update list. - * - * Regardless, we avoid copying in overflow records: if - * there's a WT_INSERT entry that modifies a reference - * counted overflow record, we may have to write copies - * of the overflow record, and in that case we'll do the - * comparisons, but we don't read overflow items just to - * see if they match records on either side. - */ - if (vpack->ovfl) { - ovfl_state = OVFL_UNUSED; - goto record_loop; - } + /* + * If the original value is "deleted", there's no value + * to compare, we're done. + */ + orig_deleted = vpack->type == WT_CELL_DEL; + if (orig_deleted) + goto record_loop; - /* - * If data is Huffman encoded, we have to decode it in - * order to compare it with the last item we saw, which - * may have been an update string. This guarantees we - * find every single pair of objects we can RLE encode, - * including applications updating an existing record - * where the new value happens (?) to match a Huffman- - * encoded value in a previous or next record. - */ - WT_ERR(__wt_dsk_cell_data_ref( - session, WT_PAGE_COL_VAR, vpack, orig)); + /* + * Overflow items are tricky: we don't know until we're + * finished processing the set of values if we need the + * overflow value or not. If we don't use the overflow + * item at all, we have to discard it from the backing + * file, otherwise we'll leak blocks on the checkpoint. + * That's safe because if the backing overflow value is + * still needed by any running transaction, we'll cache + * a copy in the update list. + * + * Regardless, we avoid copying in overflow records: if + * there's a WT_INSERT entry that modifies a reference + * counted overflow record, we may have to write copies + * of the overflow record, and in that case we'll do the + * comparisons, but we don't read overflow items just to + * see if they match records on either side. + */ + if (vpack->ovfl) { + ovfl_state = OVFL_UNUSED; + goto record_loop; } + /* + * If data is Huffman encoded, we have to decode it in + * order to compare it with the last item we saw, which + * may have been an update string. This guarantees we + * find every single pair of objects we can RLE encode, + * including applications updating an existing record + * where the new value happens (?) to match a Huffman- + * encoded value in a previous or next record. + */ + WT_ERR(__wt_dsk_cell_data_ref( + session, WT_PAGE_COL_VAR, vpack, orig)); + record_loop: /* * Generate on-page entries: loop repeat records, looking for * WT_INSERT entries matching the record number. The WT_INSERT @@ -4363,6 +4374,7 @@ record_loop: /* for (n = 0; n < nrepeat; n += repeat_count, src_recno += repeat_count) { start_ts = vpack->start_ts; + durable_ts = newest_durable_ts; stop_ts = vpack->stop_ts; upd = NULL; if (ins != NULL && WT_INSERT_RECNO(ins) == src_recno) { @@ -4378,9 +4390,11 @@ record_loop: /* * the page. */ start_ts = WT_TS_NONE; + durable_ts = WT_TS_NONE; stop_ts = WT_TS_MAX; } else { start_ts = upd_select.start_ts; + durable_ts = upd_select.durable_ts; stop_ts = upd_select.stop_ts; } ins = WT_SKIP_NEXT(ins); @@ -4471,8 +4485,8 @@ record_loop: /* if (rle != 0) { WT_ERR(__rec_col_var_helper( session, r, salvage, - last.value, - last.start_ts, last.stop_ts, + last.value, last.start_ts, + durable_ts, last.stop_ts, rle, last.deleted, false)); rle = 0; } @@ -4480,8 +4494,8 @@ record_loop: /* last.value->data = vpack->data; last.value->size = vpack->size; WT_ERR(__rec_col_var_helper(session, r, - salvage, - last.value, start_ts, stop_ts, + salvage, last.value, + start_ts, durable_ts, stop_ts, repeat_count, false, true)); /* Track if page has overflow items. */ @@ -4534,8 +4548,8 @@ compare: /* continue; } WT_ERR(__rec_col_var_helper(session, r, salvage, - last.value, last.start_ts, last.stop_ts, - rle, last.deleted, false)); + last.value, last.start_ts, durable_ts, + last.stop_ts, rle, last.deleted, false)); } /* @@ -4623,9 +4637,11 @@ compare: /* * tombstone on the page. */ start_ts = WT_TS_NONE; + durable_ts = WT_TS_NONE; stop_ts = WT_TS_MAX; } else { start_ts = upd_select.start_ts; + durable_ts = upd_select.durable_ts; stop_ts = upd_select.stop_ts; } while (src_recno <= n) { @@ -4665,11 +4681,13 @@ compare: /* * the page. */ start_ts = WT_TS_NONE; + durable_ts = WT_TS_NONE; stop_ts = WT_TS_MAX; deleted = true; } else { start_ts = upd_select.start_ts; + durable_ts = upd_select.durable_ts; stop_ts = upd_select.stop_ts; switch (upd->type) { @@ -4714,8 +4732,8 @@ compare: /* goto next; } WT_ERR(__rec_col_var_helper(session, r, salvage, - last.value, last.start_ts, last.stop_ts, - rle, last.deleted, false)); + last.value, last.start_ts, durable_ts, + last.stop_ts, rle, last.deleted, false)); } /* @@ -4762,8 +4780,9 @@ next: if (src_recno == UINT64_MAX) /* If we were tracking a record, write it. */ if (rle != 0) - WT_ERR(__rec_col_var_helper(session, r, salvage, last.value, - last.start_ts, last.stop_ts, rle, last.deleted, false)); + WT_ERR(__rec_col_var_helper(session, r, salvage, + last.value, last.start_ts, durable_ts, last.stop_ts, + rle, last.deleted, false)); /* Write the remnant page. */ ret = __rec_split_finish(session, r); @@ -4789,7 +4808,7 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) WT_KV *key, *val; WT_PAGE *child; WT_REF *ref; - wt_timestamp_t oldest_start_ts, newest_start_ts, newest_stop_ts; + wt_timestamp_t oldest_start_ts, newest_durable_ts, newest_stop_ts; size_t size; bool hazard, key_onpage_ovfl, ovfl_key; const void *p; @@ -4943,7 +4962,7 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) __rec_cell_build_addr(session, r, addr, state == WT_CHILD_PROXY, WT_RECNO_OOB); oldest_start_ts = addr->oldest_start_ts; - newest_start_ts = addr->newest_start_ts; + newest_durable_ts = addr->newest_durable_ts; newest_stop_ts = addr->newest_stop_ts; } else { __wt_cell_unpack(session, page, ref->addr, vpack); @@ -4959,7 +4978,7 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) val->cell_len = 0; val->len = val->buf.size; oldest_start_ts = vpack->oldest_start_ts; - newest_start_ts = vpack->newest_start_ts; + newest_durable_ts = vpack->newest_durable_ts; newest_stop_ts = vpack->newest_stop_ts; } WT_CHILD_RELEASE_ERR(session, hazard, ref); @@ -5002,8 +5021,8 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) /* Copy the key and value onto the page. */ __rec_image_copy(session, r, key); __rec_image_copy(session, r, val); - __rec_addr_ts_update( - r, oldest_start_ts, newest_start_ts, newest_stop_ts); + __rec_addr_ts_update(r, + oldest_start_ts, newest_durable_ts, newest_stop_ts); /* Update compression state. */ __rec_key_state_update(r, ovfl_key); @@ -5056,7 +5075,7 @@ __rec_row_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) __rec_image_copy(session, r, key); __rec_image_copy(session, r, val); __rec_addr_ts_update(r, addr->oldest_start_ts, - addr->newest_start_ts, addr->newest_stop_ts); + addr->newest_durable_ts, addr->newest_stop_ts); /* Update compression state. */ __rec_key_state_update(r, ovfl_key); @@ -5091,8 +5110,9 @@ __rec_row_zero_len(WT_SESSION_IMPL *session, */ static int __rec_row_leaf(WT_SESSION_IMPL *session, - WT_RECONCILE *r, WT_PAGE *page, WT_SALVAGE_COOKIE *salvage) + WT_RECONCILE *r, WT_REF *pageref, WT_SALVAGE_COOKIE *salvage) { + WT_ADDR *addr; WT_BTREE *btree; WT_CELL *cell; WT_CELL_UNPACK *kpack, _kpack, *vpack, _vpack; @@ -5103,10 +5123,11 @@ __rec_row_leaf(WT_SESSION_IMPL *session, WT_IKEY *ikey; WT_INSERT *ins; WT_KV *key, *val; + WT_PAGE *page; WT_ROW *rip; WT_UPDATE *upd; WT_UPDATE_SELECT upd_select; - wt_timestamp_t start_ts, stop_ts; + wt_timestamp_t start_ts, durable_ts, newest_durable_ts, stop_ts; size_t size; uint64_t slvg_skip, txnid; uint32_t i; @@ -5116,12 +5137,27 @@ __rec_row_leaf(WT_SESSION_IMPL *session, btree = S2BT(session); cbt = &r->update_modify_cbt; + page = pageref->page; slvg_skip = salvage == NULL ? 0 : salvage->skip; key = &r->k; val = &r->v; vpack = &_vpack; + /* + * Acquire the newest-durable timestamp for this page so we can roll it + * forward. If it exists, it's in the WT_REF structure or the parent's + * disk image. + */ + if ((addr = pageref->addr) == NULL) + newest_durable_ts = WT_TS_NONE; + else if (__wt_off_page(pageref->home, addr)) + newest_durable_ts = addr->newest_durable_ts; + else { + __wt_cell_unpack(session, pageref->home, pageref->addr, vpack); + newest_durable_ts = vpack->newest_durable_ts; + } + WT_RET(__rec_split_init( session, r, page, 0, btree->maxleafpage_precomp)); @@ -5173,6 +5209,7 @@ __rec_row_leaf(WT_SESSION_IMPL *session, /* Unpack the on-page value cell, set the default timestamps. */ __wt_row_leaf_value_cell(session, page, rip, NULL, vpack); start_ts = vpack->start_ts; + durable_ts = newest_durable_ts; stop_ts = vpack->stop_ts; txnid = WT_TXN_NONE; @@ -5180,9 +5217,10 @@ __rec_row_leaf(WT_SESSION_IMPL *session, WT_ERR(__rec_upd_select( session, r, NULL, rip, vpack, &upd_select)); if ((upd = upd_select.upd) != NULL) { - txnid = upd_select.txnid; start_ts = upd_select.start_ts; + durable_ts = upd_select.durable_ts; stop_ts = upd_select.stop_ts; + txnid = upd_select.txnid; } /* Build value cell. */ @@ -5450,7 +5488,7 @@ build: session, r, start_ts, stop_ts, 0, val)); __rec_image_copy(session, r, val); } - __rec_addr_ts_update(r, start_ts, start_ts, stop_ts); + __rec_addr_ts_update(r, start_ts, durable_ts, stop_ts); /* Update compression state. */ __rec_key_state_update(r, ovfl_key); @@ -5480,7 +5518,7 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) WT_KV *key, *val; WT_UPDATE *upd; WT_UPDATE_SELECT upd_select; - wt_timestamp_t start_ts, stop_ts; + wt_timestamp_t start_ts, durable_ts, stop_ts; uint64_t txnid; bool ovfl_key, upd_saved; @@ -5494,9 +5532,10 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) WT_RET(__rec_upd_select( session, r, ins, NULL, NULL, &upd_select)); upd = upd_select.upd; - txnid = upd_select.txnid; start_ts = upd_select.start_ts; + durable_ts = upd_select.durable_ts; stop_ts = upd_select.stop_ts; + txnid = upd_select.txnid; upd_saved = upd_select.upd_saved; if (upd == NULL) { @@ -5580,7 +5619,7 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) session, r, start_ts, stop_ts, 0, val)); __rec_image_copy(session, r, val); } - __rec_addr_ts_update(r, start_ts, start_ts, stop_ts); + __rec_addr_ts_update(r, start_ts, durable_ts, stop_ts); /* Update compression state. */ __rec_key_state_update(r, ovfl_key); @@ -5837,7 +5876,7 @@ __rec_write_wrapup(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) r->wrapup_checkpoint_compressed)); __wt_checkpoint_tree_reconcile_update(session, r->multi->addr.oldest_start_ts, - r->multi->addr.newest_start_ts, + r->multi->addr.newest_durable_ts, r->multi->addr.newest_stop_ts); } @@ -6179,10 +6218,9 @@ __rec_cell_build_addr(WT_SESSION_IMPL *session, */ val->buf.data = addr->addr; val->buf.size = addr->size; - val->cell_len = __wt_cell_pack_addr(session, - &val->cell, cell_type, recno, - addr->oldest_start_ts, addr->newest_start_ts, addr->newest_stop_ts, - val->buf.size); + val->cell_len = __wt_cell_pack_addr( + session, &val->cell, cell_type, recno, addr->oldest_start_ts, + addr->newest_durable_ts, addr->newest_stop_ts, val->buf.size); val->len = val->cell_len + val->buf.size; } diff --git a/src/third_party/wiredtiger/src/schema/schema_util.c b/src/third_party/wiredtiger/src/schema/schema_util.c index 9626cf51d13..f3ad28708c9 100644 --- a/src/third_party/wiredtiger/src/schema/schema_util.c +++ b/src/third_party/wiredtiger/src/schema/schema_util.c @@ -9,24 +9,19 @@ #include "wt_internal.h" /* - * __wt_schema_backup_check -- - * Check if a backup cursor is open and give an error if the schema - * operation will conflict. This is called after the schema operations - * have taken the schema lock so no hot backup cursor can be created until - * this is done. + * __schema_backup_check_int -- + * Helper for __wt_schema_backup_check. Intended to be called while + * holding the hot backup read lock. */ -int -__wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) +static int +__schema_backup_check_int(WT_SESSION_IMPL *session, const char *name) { WT_CONNECTION_IMPL *conn; - WT_DECL_RET; int i; char **backup_list; conn = S2C(session); - if (!conn->hot_backup) - return (0); - __wt_readlock(session, &conn->hot_backup_lock); + /* * There is a window at the end of a backup where the list has been * cleared from the connection but the flag is still set. It is safe @@ -34,16 +29,34 @@ __wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) */ if (!conn->hot_backup || (backup_list = conn->hot_backup_list) == NULL) { - __wt_readunlock(session, &conn->hot_backup_lock); return (0); } for (i = 0; backup_list[i] != NULL; ++i) { - if (strcmp(backup_list[i], name) == 0) { - ret = __wt_set_return(session, EBUSY); - break; - } + if (strcmp(backup_list[i], name) == 0) + return __wt_set_return(session, EBUSY); } - __wt_readunlock(session, &conn->hot_backup_lock); + + return (0); +} + +/* + * __wt_schema_backup_check -- + * Check if a backup cursor is open and give an error if the schema + * operation will conflict. This is called after the schema operations + * have taken the schema lock so no hot backup cursor can be created until + * this is done. + */ +int +__wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + + conn = S2C(session); + if (!conn->hot_backup) + return (0); + WT_WITH_HOTBACKUP_READ_LOCK_UNCOND(session, + ret = __schema_backup_check_int(session, name)); return (ret); } diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 4a7b50d72e1..3a21171d781 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -1137,7 +1137,9 @@ static const char * const __stats_connection_desc[] = { "transaction: transaction range of IDs currently pinned by named snapshots", "transaction: transaction range of timestamps currently pinned", "transaction: transaction range of timestamps pinned by a checkpoint", + "transaction: transaction range of timestamps pinned by the oldest active read timestamp", "transaction: transaction range of timestamps pinned by the oldest timestamp", + "transaction: transaction read timestamp of the oldest active reader", "transaction: transaction sync calls", "transaction: transactions committed", "transaction: transactions rolled back", @@ -1568,7 +1570,9 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing txn_pinned_snapshot_range */ /* not clearing txn_pinned_timestamp */ /* not clearing txn_pinned_timestamp_checkpoint */ + /* not clearing txn_pinned_timestamp_reader */ /* not clearing txn_pinned_timestamp_oldest */ + /* not clearing txn_timestamp_oldest_active_read */ stats->txn_sync = 0; stats->txn_commit = 0; stats->txn_rollback = 0; @@ -2167,8 +2171,12 @@ __wt_stat_connection_aggregate( to->txn_pinned_timestamp += WT_STAT_READ(from, txn_pinned_timestamp); to->txn_pinned_timestamp_checkpoint += WT_STAT_READ(from, txn_pinned_timestamp_checkpoint); + to->txn_pinned_timestamp_reader += + WT_STAT_READ(from, txn_pinned_timestamp_reader); to->txn_pinned_timestamp_oldest += WT_STAT_READ(from, txn_pinned_timestamp_oldest); + to->txn_timestamp_oldest_active_read += + WT_STAT_READ(from, txn_timestamp_oldest_active_read); to->txn_sync += WT_STAT_READ(from, txn_sync); to->txn_commit += WT_STAT_READ(from, txn_commit); to->txn_rollback += WT_STAT_READ(from, txn_rollback); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 9967dc3b2b3..c45afbf5730 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -1304,6 +1304,7 @@ __wt_txn_stats_update(WT_SESSION_IMPL *session) WT_TXN_GLOBAL *txn_global; wt_timestamp_t checkpoint_timestamp; wt_timestamp_t commit_timestamp; + wt_timestamp_t oldest_active_read_timestamp; wt_timestamp_t pinned_timestamp; uint64_t checkpoint_pinned, snapshot_pinned; @@ -1329,6 +1330,21 @@ __wt_txn_stats_update(WT_SESSION_IMPL *session) WT_STAT_SET(session, stats, txn_pinned_timestamp_oldest, commit_timestamp - txn_global->oldest_timestamp); + if (__wt_txn_get_pinned_timestamp( + session, &oldest_active_read_timestamp, 0) == 0) { + WT_STAT_SET(session, stats, + txn_timestamp_oldest_active_read, + oldest_active_read_timestamp); + WT_STAT_SET(session, stats, + txn_pinned_timestamp_reader, + commit_timestamp - oldest_active_read_timestamp); + } else { + WT_STAT_SET(session, + stats, txn_timestamp_oldest_active_read, 0); + WT_STAT_SET(session, + stats, txn_pinned_timestamp_reader, 0); + } + WT_STAT_SET(session, stats, txn_pinned_snapshot_range, snapshot_pinned == WT_TXN_NONE ? 0 : txn_global->current - snapshot_pinned); diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index a1c700661ce..ced994cbb9b 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -1272,6 +1272,93 @@ __drop_to(WT_CKPT *ckptbase, const char *name, size_t len) } /* + * __checkpoint_lock_dirty_tree_int -- + * Helper for __checkpoint_lock_dirty_tree. Intended to be called while + * holding the hot backup lock. + */ +static int +__checkpoint_lock_dirty_tree_int( + WT_SESSION_IMPL *session, bool is_checkpoint, + bool force, WT_BTREE *btree, WT_CKPT *ckpt, WT_CKPT *ckptbase) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + + WT_UNUSED(is_checkpoint); + conn = S2C(session); + + /* + * We can't delete checkpoints if a backup cursor is open. WiredTiger + * checkpoints are uniquely named and it's OK to have multiple of them + * in the system: clear the delete flag for them, and otherwise fail. + * Hold the lock until we're done (blocking hot backups from starting), + * we don't want to race with a future hot backup. + */ + if (conn->hot_backup) + WT_CKPT_FOREACH(ckptbase, ckpt) { + if (!F_ISSET(ckpt, WT_CKPT_DELETE)) + continue; + if (WT_PREFIX_MATCH(ckpt->name, WT_CHECKPOINT)) { + F_CLR(ckpt, WT_CKPT_DELETE); + continue; + } + WT_RET_MSG(session, EBUSY, + "checkpoint %s blocked by hot backup: it would" + "delete an existing checkpoint, and checkpoints " + "cannot be deleted during a hot backup", + ckpt->name); + } + /* + * Mark old checkpoints that are being deleted and figure out which + * trees we can skip in this checkpoint. + */ + WT_RET(__checkpoint_mark_skip(session, ckptbase, force)); + if (F_ISSET(btree, WT_BTREE_SKIP_CKPT)) + return (0); + /* + * Lock the checkpoints that will be deleted. + * + * Checkpoints are only locked when tracking is enabled, which covers + * checkpoint and drop operations, but not close. The reasoning is + * there should be no access to a checkpoint during close, because any + * thread accessing a checkpoint will also have the current file handle + * open. + */ + if (WT_META_TRACKING(session)) + WT_CKPT_FOREACH(ckptbase, ckpt) { + if (!F_ISSET(ckpt, WT_CKPT_DELETE)) + continue; + /* + * We can't delete checkpoints referenced by a cursor. + * WiredTiger checkpoints are uniquely named and it's + * OK to have multiple in the system: clear the delete + * flag for them, and otherwise fail. + */ + ret = __wt_session_lock_checkpoint(session, ckpt->name); + if (ret == 0) + continue; + if (ret == EBUSY && + WT_PREFIX_MATCH(ckpt->name, WT_CHECKPOINT)) { + F_CLR(ckpt, WT_CKPT_DELETE); + continue; + } + WT_RET_MSG(session, ret, + "checkpoints cannot be dropped when in-use"); + } + /* + * There are special trees: those being bulk-loaded, salvaged, upgraded + * or verified during the checkpoint. They should never be part of a + * checkpoint: we will fail to lock them because the operations have + * exclusive access to the handles. Named checkpoints will fail in that + * case, ordinary checkpoints skip files that cannot be opened normally. + */ + WT_ASSERT(session, + !is_checkpoint || !F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)); + + return (0); +} + +/* * __checkpoint_lock_dirty_tree -- * Decide whether the tree needs to be included in the checkpoint and if * so, acquire the necessary locks. @@ -1284,18 +1371,14 @@ __checkpoint_lock_dirty_tree(WT_SESSION_IMPL *session, WT_CKPT *ckpt, *ckptbase; WT_CONFIG dropconf; WT_CONFIG_ITEM cval, k, v; - WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; WT_DECL_RET; const char *name; char *name_alloc; - bool hot_backup_locked; btree = S2BT(session); - conn = S2C(session); ckpt = ckptbase = NULL; dhandle = session->dhandle; - hot_backup_locked = false; name_alloc = NULL; /* Only referenced in diagnostic builds. */ @@ -1379,91 +1462,24 @@ __checkpoint_lock_dirty_tree(WT_SESSION_IMPL *session, F_SET(ckpt, WT_CKPT_ADD); /* - * We can't delete checkpoints if a backup cursor is open. WiredTiger - * checkpoints are uniquely named and it's OK to have multiple of them - * in the system: clear the delete flag for them, and otherwise fail. - * Hold the lock until we're done (blocking hot backups from starting), - * we don't want to race with a future hot backup. + * There is some interaction between backups and checkpoints. Perform + * all backup related operations that the checkpoint needs now, while + * holding the hot backup read lock. */ - __wt_readlock(session, &conn->hot_backup_lock); - hot_backup_locked = true; - if (conn->hot_backup) - WT_CKPT_FOREACH(ckptbase, ckpt) { - if (!F_ISSET(ckpt, WT_CKPT_DELETE)) - continue; - if (WT_PREFIX_MATCH(ckpt->name, WT_CHECKPOINT)) { - F_CLR(ckpt, WT_CKPT_DELETE); - continue; - } - WT_ERR_MSG(session, EBUSY, - "checkpoint %s blocked by hot backup: it would " - "delete an existing checkpoint, and checkpoints " - "cannot be deleted during a hot backup", - ckpt->name); - } - - /* - * Mark old checkpoints that are being deleted and figure out which - * trees we can skip in this checkpoint. - */ - WT_ERR(__checkpoint_mark_skip(session, ckptbase, force)); + WT_WITH_HOTBACKUP_READ_LOCK_UNCOND(session, + ret = __checkpoint_lock_dirty_tree_int( + session, is_checkpoint, force, btree, ckpt, ckptbase)); + WT_ERR(ret); if (F_ISSET(btree, WT_BTREE_SKIP_CKPT)) goto err; - /* - * Lock the checkpoints that will be deleted. - * - * Checkpoints are only locked when tracking is enabled, which covers - * checkpoint and drop operations, but not close. The reasoning is - * there should be no access to a checkpoint during close, because any - * thread accessing a checkpoint will also have the current file handle - * open. - */ - if (WT_META_TRACKING(session)) - WT_CKPT_FOREACH(ckptbase, ckpt) { - if (!F_ISSET(ckpt, WT_CKPT_DELETE)) - continue; - - /* - * We can't delete checkpoints referenced by a cursor. - * WiredTiger checkpoints are uniquely named and it's - * OK to have multiple in the system: clear the delete - * flag for them, and otherwise fail. - */ - ret = __wt_session_lock_checkpoint(session, ckpt->name); - if (ret == 0) - continue; - if (ret == EBUSY && - WT_PREFIX_MATCH(ckpt->name, WT_CHECKPOINT)) { - F_CLR(ckpt, WT_CKPT_DELETE); - continue; - } - WT_ERR_MSG(session, ret, - "checkpoints cannot be dropped when in-use"); - } - - /* - * There are special trees: those being bulk-loaded, salvaged, upgraded - * or verified during the checkpoint. They should never be part of a - * checkpoint: we will fail to lock them because the operations have - * exclusive access to the handles. Named checkpoints will fail in that - * case, ordinary checkpoints skip files that cannot be opened normally. - */ - WT_ASSERT(session, - !is_checkpoint || !F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)); - - __wt_readunlock(session, &conn->hot_backup_lock); - WT_ASSERT(session, btree->ckpt == NULL && !F_ISSET(btree, WT_BTREE_SKIP_CKPT)); btree->ckpt = ckptbase; return (0); -err: if (hot_backup_locked) - __wt_readunlock(session, &conn->hot_backup_lock); - - __wt_meta_ckptlist_free(session, &ckptbase); +err: __wt_meta_ckptlist_free(session, &ckptbase); __wt_free(session, name_alloc); return (ret); @@ -1543,16 +1559,13 @@ __checkpoint_mark_skip( void __wt_checkpoint_tree_reconcile_update( WT_SESSION_IMPL *session, wt_timestamp_t oldest_start_ts, - wt_timestamp_t newest_start_ts, wt_timestamp_t newest_stop_ts) + wt_timestamp_t newest_durable_ts, wt_timestamp_t newest_stop_ts) { WT_BTREE *btree; WT_CKPT *ckpt, *ckptbase; btree = S2BT(session); - __wt_timestamp_addr_check(session, - oldest_start_ts, newest_start_ts, newest_stop_ts); - /* * Reconciliation just wrote a checkpoint, everything has been written. * Update the checkpoint with reconciliation information. The reason @@ -1564,7 +1577,7 @@ __wt_checkpoint_tree_reconcile_update( if (F_ISSET(ckpt, WT_CKPT_ADD)) { ckpt->write_gen = btree->write_gen; ckpt->oldest_start_ts = oldest_start_ts; - ckpt->newest_start_ts = newest_start_ts; + ckpt->newest_durable_ts = newest_durable_ts; ckpt->newest_stop_ts = newest_stop_ts; } } diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index 7a502265602..50d24778ffb 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -8,12 +8,6 @@ #include "wt_internal.h" -/* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_TXN_TS_ALREADY_LOCKED 0x1u -#define WT_TXN_TS_INCLUDE_CKPT 0x2u -#define WT_TXN_TS_INCLUDE_OLDEST 0x4u -/* AUTOMATIC FLAG VALUE GENERATION STOP */ - /* * __wt_timestamp_to_string -- * Convert a timestamp to the MongoDB string representation. @@ -164,11 +158,11 @@ __txn_get_read_timestamp( } /* - * __txn_get_pinned_timestamp -- + * __wt_txn_get_pinned_timestamp -- * Calculate the current pinned timestamp. */ -static int -__txn_get_pinned_timestamp( +int +__wt_txn_get_pinned_timestamp( WT_SESSION_IMPL *session, wt_timestamp_t *tsp, uint32_t flags) { WT_CONNECTION_IMPL *conn; @@ -289,10 +283,10 @@ __txn_global_query_timestamp( return (WT_NOTFOUND); ts = txn_global->oldest_timestamp; } else if (WT_STRING_MATCH("oldest_reader", cval.str, cval.len)) - WT_RET(__txn_get_pinned_timestamp( + WT_RET(__wt_txn_get_pinned_timestamp( session, &ts, WT_TXN_TS_INCLUDE_CKPT)); else if (WT_STRING_MATCH("pinned", cval.str, cval.len)) - WT_RET(__txn_get_pinned_timestamp(session, &ts, + WT_RET(__wt_txn_get_pinned_timestamp(session, &ts, WT_TXN_TS_INCLUDE_CKPT | WT_TXN_TS_INCLUDE_OLDEST)); else if (WT_STRING_MATCH("recovery", cval.str, cval.len)) /* Read-only value forever. No lock needed. */ @@ -381,7 +375,7 @@ __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session, bool force) return (0); /* Scan to find the global pinned timestamp. */ - if ((ret = __txn_get_pinned_timestamp( + if ((ret = __wt_txn_get_pinned_timestamp( session, &pinned_timestamp, WT_TXN_TS_INCLUDE_OLDEST)) != 0) return (ret == WT_NOTFOUND ? 0 : ret); @@ -397,7 +391,7 @@ __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session, bool force) * Scan the global pinned timestamp again, it's possible that it got * changed after the previous scan. */ - if ((ret = __txn_get_pinned_timestamp(session, &pinned_timestamp, + if ((ret = __wt_txn_get_pinned_timestamp(session, &pinned_timestamp, WT_TXN_TS_ALREADY_LOCKED | WT_TXN_TS_INCLUDE_OLDEST)) != 0) { __wt_writeunlock(session, &txn_global->rwlock); return (ret == WT_NOTFOUND ? 0 : ret); @@ -636,8 +630,7 @@ __wt_txn_set_commit_timestamp( */ if (has_oldest_ts && commit_ts < oldest_ts) { __wt_timestamp_to_string(commit_ts, ts_string[0]); - __wt_timestamp_to_string( - oldest_ts, ts_string[1]); + __wt_timestamp_to_string(oldest_ts, ts_string[1]); WT_RET_MSG(session, EINVAL, "commit timestamp %s is less than the oldest " "timestamp %s", @@ -646,8 +639,7 @@ __wt_txn_set_commit_timestamp( if (has_stable_ts && commit_ts < stable_ts) { __wt_timestamp_to_string(commit_ts, ts_string[0]); - __wt_timestamp_to_string( - oldest_ts, ts_string[1]); + __wt_timestamp_to_string(stable_ts, ts_string[1]); WT_RET_MSG(session, EINVAL, "commit timestamp %s is less than the stable " "timestamp %s", @@ -746,7 +738,7 @@ __wt_txn_set_durable_timestamp( if (has_stable_ts && durable_ts < stable_ts) { __wt_timestamp_to_string(durable_ts, ts_string[0]); - __wt_timestamp_to_string(oldest_ts, ts_string[1]); + __wt_timestamp_to_string(stable_ts, ts_string[1]); WT_RET_MSG(session, EINVAL, "durable timestamp %s is less than the stable timestamp %s", ts_string[0], ts_string[1]); @@ -878,7 +870,7 @@ __wt_txn_set_read_timestamp( WT_TXN_GLOBAL *txn_global = &S2C(session)->txn_global; wt_timestamp_t ts_oldest; char ts_string[2][WT_TS_INT_STRING_SIZE]; - bool roundup_to_oldest; + bool did_roundup_to_oldest; WT_RET(__wt_txn_context_prepare_check(session)); @@ -896,45 +888,37 @@ __wt_txn_set_read_timestamp( " may only be set once per transaction"); /* - * The read timestamp could be rounded to the oldest timestamp. - */ - roundup_to_oldest = F_ISSET(txn, WT_TXN_TS_ROUND_READ); - - /* * This code is not using the timestamp validate function to * avoid a race between checking and setting transaction * timestamp. */ __wt_readlock(session, &txn_global->rwlock); ts_oldest = txn_global->oldest_timestamp; + did_roundup_to_oldest = false; if (read_ts < ts_oldest) { /* * If given read timestamp is earlier than oldest * timestamp then round the read timestamp to * oldest timestamp. */ - if (roundup_to_oldest) + if (F_ISSET(txn, WT_TXN_TS_ROUND_READ)) { txn->read_timestamp = ts_oldest; - else { - __wt_readunlock(session, &txn_global->rwlock); - __wt_timestamp_to_string(read_ts, ts_string[0]); - __wt_timestamp_to_string(ts_oldest, ts_string[1]); - WT_RET_MSG(session, EINVAL, "read timestamp " - "%s less than the oldest timestamp %s", - ts_string[0], ts_string[1]); + did_roundup_to_oldest = true; + } else { + __wt_readunlock(session, &txn_global->rwlock); + __wt_timestamp_to_string(read_ts, ts_string[0]); + __wt_timestamp_to_string(ts_oldest, ts_string[1]); + WT_RET_MSG(session, EINVAL, "read timestamp " + "%s less than the oldest timestamp %s", + ts_string[0], ts_string[1]); } - } else { + } else txn->read_timestamp = read_ts; - /* - * Reset to avoid a verbose message as read - * timestamp is not rounded to oldest timestamp. - */ - roundup_to_oldest = false; - } __wt_txn_publish_read_timestamp(session); __wt_readunlock(session, &txn_global->rwlock); - if (roundup_to_oldest && WT_VERBOSE_ISSET(session, WT_VERB_TIMESTAMP)) { + if (did_roundup_to_oldest && + WT_VERBOSE_ISSET(session, WT_VERB_TIMESTAMP)) { /* * This message is generated here to reduce the span of * critical section. diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index d08c7f69695..493cdaf5114 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -33,7 +33,7 @@ static int compare_cursors( WT_CURSOR *, const char *, WT_CURSOR *, const char *); static int diagnose_key_error(WT_CURSOR *, int, WT_CURSOR *, int); static int real_checkpointer(void); -static int verify_checkpoint(WT_SESSION *); +static int verify_consistency(WT_SESSION *, bool); /* * start_checkpoints -- @@ -105,6 +105,11 @@ real_checkpointer(void) } while (g.running) { + /* Check for consistency of online data */ + if ((ret = verify_consistency(session, false)) != 0) + return (log_print_err( + "verify_consistency (online)", ret, 1)); + /* Execute a checkpoint */ if ((ret = session->checkpoint( session, checkpoint_config)) != 0) @@ -115,8 +120,9 @@ real_checkpointer(void) goto done; /* Verify the content of the checkpoint. */ - if ((ret = verify_checkpoint(session)) != 0) - return (log_print_err("verify_checkpoint", ret, 1)); + if ((ret = verify_consistency(session, true)) != 0) + return (log_print_err( + "verify_consistency (offline)", ret, 1)); } done: if ((ret = session->close(session, NULL)) != 0) @@ -126,40 +132,48 @@ done: if ((ret = session->close(session, NULL)) != 0) } /* - * verify_checkpoint -- + * verify_consistency -- * Open a cursor on each table at the last checkpoint and walk through * the tables in parallel. The key/values should match across all tables. */ static int -verify_checkpoint(WT_SESSION *session) +verify_consistency(WT_SESSION *session, bool use_checkpoint) { WT_CURSOR **cursors; uint64_t key_count; int i, ret, t_ret; - char ckpt[128], next_uri[128]; - const char *type0, *typei; + const char *ckpt, *type0, *typei; + char ckpt_buf[128], next_uri[128]; ret = t_ret = 0; key_count = 0; - testutil_check(__wt_snprintf( - ckpt, sizeof(ckpt), "checkpoint=%s", g.checkpoint_name)); cursors = calloc((size_t)g.ntables, sizeof(*cursors)); if (cursors == NULL) - return (log_print_err("verify_checkpoint", ENOMEM, 1)); + return (log_print_err("verify_consistency", ENOMEM, 1)); + + if (use_checkpoint) { + testutil_check(__wt_snprintf(ckpt_buf, sizeof(ckpt_buf), + "checkpoint=%s", g.checkpoint_name)); + ckpt = ckpt_buf; + } else { + ckpt = NULL; + testutil_check(session->begin_transaction( + session, "isolation=snapshot")); + } for (i = 0; i < g.ntables; i++) { /* * TODO: LSM doesn't currently support reading from * checkpoints. */ - if (g.cookies[i].type == LSM) + if (use_checkpoint && g.cookies[i].type == LSM) continue; testutil_check(__wt_snprintf( next_uri, sizeof(next_uri), "table:__wt%04d", i)); if ((ret = session->open_cursor( session, next_uri, NULL, ckpt, &cursors[i])) != 0) { (void)log_print_err( - "verify_checkpoint:session.open_cursor", ret, 1); + "verify_consistency:session.open_cursor", ret, 1); goto err; } } @@ -199,7 +213,7 @@ verify_checkpoint(WT_SESSION *session) continue; else if (ret == WT_NOTFOUND || t_ret == WT_NOTFOUND) { (void)log_print_err( - "verify_checkpoint tables with different" + "verify_consistency tables with different" " amount of data", EFAULT, 1); goto err; } @@ -211,21 +225,24 @@ verify_checkpoint(WT_SESSION *session) (void)diagnose_key_error( cursors[0], 0, cursors[i], i); (void)log_print_err( - "verify_checkpoint - mismatching data", + "verify_consistency - mismatching data", EFAULT, 1); goto err; } } } - printf("Finished verifying a checkpoint with %d tables and %" PRIu64 - " keys\n", g.ntables, key_count); + printf("Finished verifying a %s with %d tables and %" PRIu64 + " keys\n", use_checkpoint ? "checkpoint" : "snapshot", + g.ntables, key_count); err: for (i = 0; i < g.ntables; i++) { if (cursors[i] != NULL && (ret = cursors[i]->close(cursors[i])) != 0) (void)log_print_err( - "verify_checkpoint:cursor close", ret, 1); + "verify_consistency:cursor close", ret, 1); } + if (!use_checkpoint) + testutil_check(session->commit_transaction(session, NULL)); free(cursors); return (ret); } diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index ca1bca04528..33836c67110 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -126,13 +126,22 @@ worker_op(WT_CURSOR *cursor, uint64_t keyno, u_int new_val) char valuebuf[64]; cursor->set_key(cursor, keyno); - testutil_check(__wt_snprintf( - valuebuf, sizeof(valuebuf), "%037u", new_val)); - cursor->set_value(cursor, valuebuf); - if ((ret = cursor->insert(cursor)) != 0) { - if (ret == WT_ROLLBACK) - return (WT_ROLLBACK); - return (log_print_err("cursor.insert", ret, 1)); + /* Roughly 5% removes. */ + if (new_val % 19 == 0) { + if ((ret = cursor->remove(cursor)) != 0) { + if (ret == WT_ROLLBACK) + return (WT_ROLLBACK); + return (log_print_err("cursor.remove", ret, 1)); + } + } else { + testutil_check(__wt_snprintf( + valuebuf, sizeof(valuebuf), "%037u", new_val)); + cursor->set_value(cursor, valuebuf); + if ((ret = cursor->insert(cursor)) != 0) { + if (ret == WT_ROLLBACK) + return (WT_ROLLBACK); + return (log_print_err("cursor.insert", ret, 1)); + } } return (0); } diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am index 362d0775a88..53175472c1e 100644 --- a/src/third_party/wiredtiger/test/csuite/Makefile.am +++ b/src/third_party/wiredtiger/test/csuite/Makefile.am @@ -127,6 +127,10 @@ test_wt4333_handle_locks_SOURCES = wt4333_handle_locks/main.c noinst_PROGRAMS += test_wt4333_handle_locks all_TESTS += test_wt4333_handle_locks +test_wt4699_json_SOURCES = wt4699_json/main.c +noinst_PROGRAMS += test_wt4699_json +all_TESTS += test_wt4699_json + # Run this during a "make check" smoke test. TESTS = $(all_TESTS) LOG_COMPILER = $(TEST_WRAPPER) diff --git a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c index fd734b1a4a2..32b9f8f42a8 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c @@ -30,16 +30,11 @@ #include <sys/wait.h> #include <signal.h> -#define CKPT_DISTANCE 1 #define CORRUPT "file:zzz-corrupt.SS" #define KEY "key" #define VALUE "value,value,value" -#define DB0 "CKPT0" -#define DB1 "CKPT1" -#define DB2 "CKPT2" #define SAVE "SAVE" -#define TEST "TEST" /* * NOTE: This assumes the default page size of 4096. If that changes these @@ -47,7 +42,7 @@ */ #define APP_MD_SIZE 4096 #define APP_BUF_SIZE (3 * 1024) -#define APP_STR "long app metadata. " +#define APP_STR "Long app metadata intended to force a page per entry. " static uint64_t data_val; static const char *home; @@ -363,59 +358,6 @@ copy_database(const char *sfx) } /* - * move_data_ahead -- - * Update the tables with new data and take a checkpoint twice. - * WiredTiger keeps the previous checkpoint so we do it twice so that - * the old checkpoint address no longer exists. - */ -static void -move_data_ahead(TABLE_INFO *table_data) -{ - TABLE_INFO *t; - uint64_t i; - - i = 0; - while (i < CKPT_DISTANCE) { - ++data_val; - for (t = table_data; t->name != NULL; t++) - cursor_insert(t->name, data_val); - ++i; - fprintf(stderr, "MOVE DATA: inserted %" PRIu64 ". CKPT.\n", - data_val); - testutil_check(wt_session->checkpoint(wt_session, NULL)); - } -} - -/* - * make_database_copies -- - * Make copies of the database so that we can test various mix and match - * of turtle files and metadata files. We take some checkpoints and - * update the data too. - */ -static void -make_database_copies(TABLE_INFO *table_data) -{ - /* - * If we're running an out-of-sync test, then we want to make copies - * of the turtle and metadata file, then checkpoint and again save a - * copy of the turtle file and the metadata file. Then we add more data - * and checkpoint again at least twice. Using the original and current - * files we can test various out of sync scenarios. - */ - /* - * Take a checkpoint and make a copy. - */ - testutil_check(wt_session->checkpoint(wt_session, NULL)); - copy_database(DB0); - - move_data_ahead(table_data); - copy_database(DB1); - - move_data_ahead(table_data); - copy_database(DB2); -} - -/* * wt_open_corrupt -- * Call wiredtiger_open and expect a corruption error. */ @@ -534,125 +476,6 @@ run_all_verification(const char *sfx, TABLE_INFO *t) open_normal(sfx, t); } -static void -setup_database(const char *src, const char *turtle_dir, const char *meta_dir) -{ - WT_DECL_RET; - char buf[1024]; - - /* - * Remove the test home directory and copy the source to it. - * Then copy the saved turtle and/or metadata file from the - * given args. - */ - testutil_check(__wt_snprintf(buf, sizeof(buf), - "rm -rf ./%s.%s; mkdir ./%s.%s; " - "cp -p %s.%s/* ./%s.%s", - home, TEST, home, TEST, home, src, home, TEST)); - printf("copy: %s\n", buf); - if ((ret = system(buf)) < 0) - testutil_die(ret, "system: %s", buf); - - /* Copy turtle if given. */ - if (turtle_dir != NULL) { - testutil_check(__wt_snprintf(buf, sizeof(buf), - "cp -p %s.%s/%s.%s %s.%s/%s", - home, turtle_dir, WT_METADATA_TURTLE, SAVE, - home, TEST, WT_METADATA_TURTLE)); - printf("copy: %s\n", buf); - if ((ret = system(buf)) < 0) - testutil_die(ret, "system: %s", buf); - } - /* Copy metadata if given. */ - if (meta_dir != NULL) { - testutil_check(__wt_snprintf(buf, sizeof(buf), - "cp -p %s.%s/%s.%s %s.%s/%s", - home, meta_dir, WT_METAFILE, SAVE, - home, TEST, WT_METAFILE)); - printf("copy: %s\n", buf); - if ((ret = system(buf)) < 0) - testutil_die(ret, "system: %s", buf); - } -} - -static void -out_of_sync(TABLE_INFO *table_data) -{ - /* - * We have five directories: - * - The main database directory that we just corrupted/salvaged. - * - A .SAVE copy of the main directory that is coherent prior to - * corrupting. Essentially a copy of the second checkpoint dir. - * - A copy of the main directory before the first checkpoint. DB0 - * - A copy of the main directory after the first checkpoint. DB1 - * - A copy of the main directory after the second checkpoint. DB2 - * - * We want to make a copy of a source directory and then copy a - * turtle or metadata file from another directory. Then detect the - * error, run with salvage and confirm. - */ - /* - * Run in DB0, bring in future metadata from DB1. - */ - test_out_of_sync = true; - printf( - "#\n# OUT OF SYNC: %s with future metadata from %s\n#\n", DB0, DB1); - setup_database(DB0, NULL, DB1); - run_all_verification(TEST, table_data); - - /* - * Run in DB0, bring in future turtle file from DB1. - */ - printf( - "#\n# OUT OF SYNC: %s with future turtle from %s\n#\n", DB0, DB1); - setup_database(DB0, DB1, NULL); - run_all_verification(TEST, table_data); - - /* - * Run in DB1, bring in old metadata file from DB0. - */ - printf("#\n# OUT OF SYNC: %s with old metadata from %s\n#\n", DB1, DB0); - setup_database(DB1, NULL, DB0); - run_all_verification(TEST, table_data); - - /* - * Run in DB1, bring in old turtle file from DB0. - */ - printf("#\n# OUT OF SYNC: %s with old turtle from %s\n#\n", DB1, DB0); - setup_database(DB1, DB0, NULL); - run_all_verification(TEST, table_data); - - /* - * Run in DB1, bring in future metadata file from DB2. - */ - printf( - "#\n# OUT OF SYNC: %s with future metadata from %s\n#\n", DB1, DB2); - setup_database(DB1, NULL, DB2); - run_all_verification(TEST, table_data); - - /* - * Run in DB1, bring in future turtle file from DB2. - */ - printf( - "#\n# OUT OF SYNC: %s with future turtle from %s\n#\n", DB1, DB2); - setup_database(DB1, DB2, NULL); - run_all_verification(TEST, table_data); - - /* - * Run in DB2, bring in old metadata file from DB1. - */ - printf("#\n# OUT OF SYNC: %s with old metadata from %s\n#\n", DB2, DB1); - setup_database(DB2, NULL, DB1); - run_all_verification(TEST, table_data); - - /* - * Run in DB2, bring in old turtle file from DB1. - */ - printf("#\n# OUT OF SYNC: %s with old turtle from %s\n#\n", DB2, DB1); - setup_database(DB2, DB1, NULL); - run_all_verification(TEST, table_data); -} - int main(int argc, char *argv[]) { @@ -700,10 +523,6 @@ main(int argc, char *argv[]) for (t = table_data; t->name != NULL; t++) create_data(t); - /* - * Take some checkpoints and add more data for out of sync testing. - */ - make_database_copies(table_data); testutil_check(opts->conn->close(opts->conn, NULL)); opts->conn = NULL; @@ -737,8 +556,6 @@ main(int argc, char *argv[]) testutil_die(ret, "system: %s", buf); run_all_verification(NULL, &table_data[0]); - out_of_sync(&table_data[0]); - /* * We need to set up the string before we clean up * the structure. Then after the clean up we will diff --git a/src/third_party/wiredtiger/test/csuite/wt4699_json/main.c b/src/third_party/wiredtiger/test/csuite/wt4699_json/main.c new file mode 100644 index 00000000000..636798c0ffd --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt4699_json/main.c @@ -0,0 +1,97 @@ +/*- + * Public Domain 2014-2019 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: WT-4699 + * Test case description: Use a JSON dump cursor on a projection, + * and overwrite the projection string. + * Failure mode: On the first retrieval of a JSON key/value, a configure + * parse error is returned. + */ + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + WT_CURSOR *c; + WT_SESSION *session; + char *jsonkey, *jsonvalue; + char projection[1000]; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, NULL, + "create", &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + /* Create a single record in a table with two fields in its value. */ + testutil_check(session->create(session, opts->uri, + "key_format=i,value_format=ii,columns=(k,v0,v1)")); + testutil_check( + session->open_cursor(session, opts->uri, NULL, NULL, &c)); + c->set_key(c, 1); + c->set_value(c, 1, 1); + testutil_check(c->insert(c)); + testutil_check(c->close(c)); + + /* + * Open a dump JSON cursor on a projection of the table. + * The fields will be listed in a different order. + */ + strcpy(projection, opts->uri); + strcat(projection, "(v1,v0,k)"); + testutil_check( + session->open_cursor(session, projection, NULL, "dump=json", &c)); + testutil_check(c->next(c)); + /* Overwrite the projection, with not enough columns */ + strcpy(projection, opts->uri); + strcat(projection, "(aaa,bbb)"); + testutil_check(c->get_key(c, &jsonkey)); + + /* + * Here's where we would get the parse error. + * When a JSON dump is performed on a projection, we need to format + * all the field names and values in the order listed. + * The implementation uses the projection string from the + * open_cursor call to determine the field names. + */ + testutil_check(c->get_value(c, &jsonvalue)); + testutil_assert(strstr(jsonvalue, "aaa") == NULL); + printf("KEY: %s\n", jsonkey); + printf("VALUE: %s\n", jsonvalue); + testutil_assert(c->next(c) == WT_NOTFOUND); + testutil_check(c->close(c)); + testutil_check(session->close(session, NULL)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 00869b45294..56010762762 100644 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -658,6 +658,20 @@ tasks: ${test_env_vars|} $(pwd)/test_wt4156_metadata_salvage 2>&1 + - name: csuite-wt4699-json-test + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/csuite" + script: | + set -o errexit + set -o verbose + + ${test_env_vars|} $(pwd)/test_wt4699_json 2>&1 + - name: csuite-rwlock-test depends_on: - name: compile @@ -1171,6 +1185,7 @@ buildvariants: - name: csuite-wt4105-large-doc-small-upd-test - name: csuite-wt4117-checksum-test - name: csuite-wt4156-metadata-salvage-test + - name: csuite-wt4699-json-test - name: csuite-rwlock-test - name: csuite-wt2246-col-append-test - name: csuite-wt2323-join-visibility-test diff --git a/src/third_party/wiredtiger/test/suite/helper.py b/src/third_party/wiredtiger/test/suite/helper.py index 0b5db24f0d9..72801cc18a4 100644..100755 --- a/src/third_party/wiredtiger/test/suite/helper.py +++ b/src/third_party/wiredtiger/test/suite/helper.py @@ -39,9 +39,9 @@ def compare_files(self, filename1, filename2): self.pr('compare_files: ' + filename1 + ', ' + filename2) bufsize = 4096 if os.path.getsize(filename1) != os.path.getsize(filename2): - print 'file comparison failed: ' + filename1 + ' size ' +\ + print('file comparison failed: ' + filename1 + ' size ' +\ str(os.path.getsize(filename1)) + ' != ' + filename2 +\ - ' size ' + str(os.path.getsize(filename2)) + ' size ' + str(os.path.getsize(filename2))) return False with open(filename1, "rb") as fp1: with open(filename2, "rb") as fp2: diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index b82a77fe97c..5895325a3d7 100755 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -30,8 +30,14 @@ # Command line test runner # +from __future__ import print_function import glob, json, os, re, sys +try: + xrange +except NameError: #python3 + xrange = range + # Set paths suitedir = sys.path[0] wt_disttop = os.path.dirname(os.path.dirname(suitedir)) @@ -56,14 +62,14 @@ elif os.path.isfile(os.path.join(wt_disttop, 'build_posix', 'wt')): elif os.path.isfile(os.path.join(wt_disttop, 'wt.exe')): wt_builddir = wt_disttop else: - print 'Unable to find useable WiredTiger build' + print('Unable to find useable WiredTiger build') sys.exit(1) # Cannot import wiredtiger and supporting utils until we set up paths # We want our local tree in front of any installed versions of WiredTiger. # Don't change sys.path[0], it's the dir containing the invoked python script. + sys.path.insert(1, os.path.join(wt_builddir, 'lang', 'python')) -sys.path.insert(1, os.path.join(wt_disttop, 'lang', 'python')) # Append to a colon separated path in the environment def append_env_path(name, value): @@ -95,7 +101,7 @@ unittest = wttest.unittest from testscenarios.scenarios import generate_scenarios def usage(): - print 'Usage:\n\ + print('Usage:\n\ $ cd build_posix\n\ $ python ../test/suite/run.py [ options ] [ tests ]\n\ \n\ @@ -122,7 +128,7 @@ Tests:\n\ \n\ When -C or -c are present, there may not be any tests named.\n\ When -s is present, there must be a test named.\n\ -' +') # capture the category (AKA 'subsuite') part of a test name, # e.g. test_util03 -> util @@ -342,7 +348,7 @@ if __name__ == '__main__': configfile = args.pop(0) configwrite = True continue - print 'unknown arg: ' + arg + print('unknown arg: ' + arg) usage() sys.exit(2) testargs.append(arg) @@ -379,7 +385,7 @@ if __name__ == '__main__': for test in tests: dryOutput.add(test.shortDesc()) for line in dryOutput: - print line + print(line) else: result = wttest.runsuite(tests, parallel) sys.exit(0 if result.wasSuccessful() else 1) diff --git a/src/third_party/wiredtiger/test/suite/suite_subprocess.py b/src/third_party/wiredtiger/test/suite/suite_subprocess.py index 853a5cde091..d04a281807a 100755 --- a/src/third_party/wiredtiger/test/suite/suite_subprocess.py +++ b/src/third_party/wiredtiger/test/suite/suite_subprocess.py @@ -26,6 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. +from __future__ import print_function import os, re, subprocess, sys from run import wt_builddir from wttest import WiredTigerTestCase @@ -69,14 +70,14 @@ class suite_subprocess: lines.pop(0) hasPrevious = True if hasError: - print '**************** ' + match + ' in output file: ' + filename + ' ****************' + print('**************** ' + match + ' in output file: ' + filename + ' ****************') if hasPrevious: - print '...' + print('...') for line in lines: - print line, + print(line, end=' ') if hasNext: - print '...' - print '********************************' + print('...') + print('********************************') self.fail('ERROR found in output file: ' + filename) # If the string is of the form '/.../', then return just the embedded @@ -133,8 +134,8 @@ class suite_subprocess: if filesize > 0: with open(filename, 'r') as f: contents = f.read(1000) - print 'ERROR: ' + filename + ' expected to be empty, but contains:\n' - print contents + '...\n' + print('ERROR: ' + filename + ' expected to be empty, but contains:\n') + print(contents + '...\n') self.assertEqual(filesize, 0, filename + ': expected to be empty') def check_non_empty_file(self, filename): @@ -143,7 +144,7 @@ class suite_subprocess: """ filesize = os.path.getsize(filename) if filesize == 0: - print 'ERROR: ' + filename + ' should not be empty (this command expected error output)' + print('ERROR: ' + filename + ' should not be empty (this command expected error output)') self.assertNotEqual(filesize, 0, filename + ': expected to not be empty') def verbose_env(self, envvar): @@ -202,23 +203,23 @@ class suite_subprocess: infilepart = "" if infilename != None: infilepart = "<" + infilename + " " - print str(procargs) - print "*********************************************" - print "**** Run 'wt' via: run " + \ + print(str(procargs)) + print("*********************************************") + print("**** Run 'wt' via: run " + \ " ".join(procargs[3:]) + infilepart + \ - ">" + wtoutname + " 2>" + wterrname - print "*********************************************" + ">" + wtoutname + " 2>" + wterrname) + print("*********************************************") returncode = subprocess.call(procargs) elif self._lldbSubprocess: infilepart = "" if infilename != None: infilepart = "<" + infilename + " " - print str(procargs) - print "*********************************************" - print "**** Run 'wt' via: run " + \ + print(str(procargs)) + print("*********************************************") + print("**** Run 'wt' via: run " + \ " ".join(procargs[3:]) + infilepart + \ - ">" + wtoutname + " 2>" + wterrname - print "*********************************************" + ">" + wtoutname + " 2>" + wterrname) + print("*********************************************") returncode = subprocess.call(procargs) elif infilename: with open(infilename, "r") as wtin: diff --git a/src/third_party/wiredtiger/test/suite/test_alter02.py b/src/third_party/wiredtiger/test/suite/test_alter02.py index 449247b6988..7a135d99b74 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_alter02.py +++ b/src/third_party/wiredtiger/test/suite/test_alter02.py @@ -78,7 +78,7 @@ class test_alter02(wttest.WiredTigerTestCase): try: self.conn = wiredtiger.wiredtiger_open(self.home, conn_params) except wiredtiger.WiredTigerError as e: - print "Failed conn at '%s' with config '%s'" % (dir, conn_params) + print("Failed conn at '%s' with config '%s'" % (dir, conn_params)) self.session = self.conn.open_session() # Verify the metadata string for this URI and that its setting in the @@ -116,12 +116,12 @@ class test_alter02(wttest.WiredTigerTestCase): keys = c.get_key() # txnid, rectype, optype, fileid, logrec_key, logrec_value values = c.get_value() - try: - if self.value in str(values[5]): # logrec_value + # We are only looking for log records that that have a key/value + # pair. + if values[4] != b'': + if self.value.encode() in values[5]: # logrec_value count += 1 - self.assertFalse(value2 in str(values[5])) - except: - pass + self.assertFalse(self.value2.encode() in values[5]) c.close() self.assertEqual(count, expected_keys) diff --git a/src/third_party/wiredtiger/test/suite/test_async01.py b/src/third_party/wiredtiger/test/suite/test_async01.py index 5b13bf5ba5c..fc50327aab1 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_async01.py +++ b/src/third_party/wiredtiger/test/suite/test_async01.py @@ -117,7 +117,7 @@ class test_async01(wttest.WiredTigerTestCase, suite_subprocess): """ table_name1 = 'test_async01' nentries = 100 - async_ops = nentries / 2 + async_ops = nentries // 2 async_threads = 3 current = {} @@ -140,7 +140,7 @@ class test_async01(wttest.WiredTigerTestCase, suite_subprocess): if self.tablekind == 'row': return 'key' + str(i) else: - return long(i+1) + return self.recno(i+1) def genvalue(self, i): if self.tablekind == 'fix': diff --git a/src/third_party/wiredtiger/test/suite/test_async02.py b/src/third_party/wiredtiger/test/suite/test_async02.py index f5319648543..739b27f69bd 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_async02.py +++ b/src/third_party/wiredtiger/test/suite/test_async02.py @@ -114,7 +114,7 @@ class test_async02(wttest.WiredTigerTestCase, suite_subprocess): """ table_name1 = 'test_async02' nentries = 100 - async_ops = nentries / 2 + async_ops = nentries // 2 async_threads = 3 current = {} @@ -137,7 +137,7 @@ class test_async02(wttest.WiredTigerTestCase, suite_subprocess): if self.tablekind == 'row': return 'key' + str(i) else: - return long(i+1) + return self.recno(i+1) def genvalue(self, i): if self.tablekind == 'fix': @@ -187,8 +187,8 @@ class test_async02(wttest.WiredTigerTestCase, suite_subprocess): self.conn.async_flush() self.pr('flushed') - k = self.genkey(self.nentries / 2) - v = self.genvalue(self.nentries / 2) + k = self.genkey(self.nentries // 2) + v = self.genvalue(self.nentries // 2) k1 = self.genkey(self.nentries + 1) v1 = self.genvalue(self.nentries + 1) self.current[k] = wiredtiger.WT_DUPLICATE_KEY diff --git a/src/third_party/wiredtiger/test/suite/test_autoclose.py b/src/third_party/wiredtiger/test/suite/test_autoclose.py index e1ddc4c3359..efe29b3ca83 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_autoclose.py +++ b/src/third_party/wiredtiger/test/suite/test_autoclose.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import wiredtiger, wttest, exceptions +import wiredtiger, wttest # test_autoclose class test_autoclose(wttest.WiredTigerTestCase): @@ -57,7 +57,7 @@ class test_autoclose(wttest.WiredTigerTestCase): inscursor = self.open_cursor() inscursor['key1'] = 'value1' inscursor.close() - self.assertRaisesHavingMessage(exceptions.RuntimeError, + self.assertRaisesHavingMessage(RuntimeError, lambda: inscursor.next(), '/wt_cursor.* is None/') self.drop_table() @@ -72,7 +72,7 @@ class test_autoclose(wttest.WiredTigerTestCase): inscursor = self.open_cursor() inscursor['key1'] = 'value1' self.session.close() - self.assertRaisesHavingMessage(exceptions.RuntimeError, + self.assertRaisesHavingMessage(RuntimeError, lambda: inscursor.next(), '/wt_cursor.* is None/') self.close_conn() @@ -86,7 +86,7 @@ class test_autoclose(wttest.WiredTigerTestCase): inscursor = self.open_cursor() inscursor['key1'] = 'value1' self.close_conn() - self.assertRaisesHavingMessage(exceptions.RuntimeError, + self.assertRaisesHavingMessage(RuntimeError, lambda: inscursor.next(), '/wt_cursor.* is None/') @@ -120,11 +120,11 @@ class test_autoclose(wttest.WiredTigerTestCase): # Note: SWIG generates a TypeError in this case. # A RuntimeError to match the other cases would be better. inscursor2.close() - self.assertRaises(exceptions.TypeError, + self.assertRaises(TypeError, lambda: inscursor.compare(inscursor2)) inscursor2 = None - self.assertRaisesHavingMessage(exceptions.RuntimeError, + self.assertRaisesHavingMessage(RuntimeError, lambda: inscursor.compare(inscursor2), '/wt_cursor.* is None/') @@ -133,7 +133,7 @@ class test_autoclose(wttest.WiredTigerTestCase): Use a session handle after it is explicitly closed. """ self.session.close() - self.assertRaisesHavingMessage(exceptions.RuntimeError, + self.assertRaisesHavingMessage(RuntimeError, lambda: self.create_table(), '/wt_session.* is None/') self.close_conn() @@ -143,7 +143,7 @@ class test_autoclose(wttest.WiredTigerTestCase): Use a session handle after the connection is closed. """ self.close_conn() - self.assertRaisesHavingMessage(exceptions.RuntimeError, + self.assertRaisesHavingMessage(RuntimeError, lambda: self.create_table(), '/wt_session.* is None/') @@ -153,7 +153,7 @@ class test_autoclose(wttest.WiredTigerTestCase): """ conn = self.conn self.close_conn() - self.assertRaisesHavingMessage(exceptions.RuntimeError, + self.assertRaisesHavingMessage(RuntimeError, lambda: conn.open_session(None), '/wt_connection.* is None/') diff --git a/src/third_party/wiredtiger/test/suite/test_backup02.py b/src/third_party/wiredtiger/test/suite/test_backup02.py index 749ca3ff5f6..3f468a2b7bf 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_backup02.py +++ b/src/third_party/wiredtiger/test/suite/test_backup02.py @@ -26,7 +26,10 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import Queue +try: + import Queue as queue # python2 +except ImportError: + import queue import threading, time, wiredtiger, wttest from wtthread import backup_thread, checkpoint_thread, op_thread @@ -57,14 +60,14 @@ class test_backup02(wttest.WiredTigerTestCase): bkp = backup_thread(self.conn, 'backup.dir', done) bkp.start() - queue = Queue.Queue() + work_queue = queue.Queue() my_data = 'a' * self.dsize - for i in xrange(self.nops): - queue.put_nowait(('gi', i, my_data)) + for i in range(self.nops): + work_queue.put_nowait(('gi', i, my_data)) opthreads = [] - for i in xrange(self.nthreads): - t = op_thread(self.conn, uris, self.fmt, queue, done) + for i in range(self.nthreads): + t = op_thread(self.conn, uris, self.fmt, work_queue, done) opthreads.append(t) t.start() @@ -74,10 +77,10 @@ class test_backup02(wttest.WiredTigerTestCase): time.sleep(0.1) my_data = str(more_time) + 'a' * (self.dsize - len(str(more_time))) more_time = more_time - 0.1 - for i in xrange(self.nops): - queue.put_nowait(('gu', i, my_data)) + for i in range(self.nops): + work_queue.put_nowait(('gu', i, my_data)) - queue.join() + work_queue.join() done.set() # # Wait for checkpoint thread to notice status change. # ckpt.join() diff --git a/src/third_party/wiredtiger/test/suite/test_backup04.py b/src/third_party/wiredtiger/test/suite/test_backup04.py index d63c3f87152..1234652f9be 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_backup04.py +++ b/src/third_party/wiredtiger/test/suite/test_backup04.py @@ -26,7 +26,6 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import Queue import threading, time, wiredtiger, wttest import glob, os, shutil from helper import compare_files @@ -34,6 +33,10 @@ from suite_subprocess import suite_subprocess from wtdataset import SimpleDataSet, simple_key from wtscenario import make_scenarios from wtthread import op_thread +try: + xrange +except NameError: #python3 + xrange = range # test_backup04.py # Utilities: wt backup @@ -67,14 +70,14 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): def populate(self, uri, dsize, rows): self.pr('populate: ' + uri + ' with ' + str(rows) + ' rows') cursor = self.session.open_cursor(uri, None) - for i in range(1, rows + 1): + for i in xrange(1, rows + 1): cursor[simple_key(cursor, i)] = str(i) + ':' + 'a' * dsize cursor.close() def update(self, uri, dsize, upd, rows): self.pr('update: ' + uri + ' with ' + str(rows) + ' rows') cursor = self.session.open_cursor(uri, None) - for i in range(1, rows + 1): + for i in xrange(1, rows + 1): cursor[simple_key(cursor, i)] = str(i) + ':' + upd * dsize cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_backup06.py b/src/third_party/wiredtiger/test/suite/test_backup06.py index cb4d76e14fb..d4efba4c6f0 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup06.py +++ b/src/third_party/wiredtiger/test/suite/test_backup06.py @@ -95,8 +95,8 @@ class test_backup06(wttest.WiredTigerTestCase, suite_subprocess): dh_after = stat_cursor[stat.conn.dh_conn_handle_count][2] stat_cursor.close() if (dh_before != dh_after): - print "Dhandles open before backup open: " + str(dh_before) - print "Dhandles open after backup open: " + str(dh_after) + print("Dhandles open before backup open: " + str(dh_before)) + print("Dhandles open after backup open: " + str(dh_after)) self.assertEqual(dh_after == dh_before, True) cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_backup09.py b/src/third_party/wiredtiger/test/suite/test_backup09.py index e80f7f048bf..2141e4f8768 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_backup09.py +++ b/src/third_party/wiredtiger/test/suite/test_backup09.py @@ -73,11 +73,9 @@ class test_backup09(wttest.WiredTigerTestCase): cursor[doc_id] = doc_id last_doc_in_backup = doc_id - self.assertEqual(1, len(filter(lambda x: - x.startswith('WiredTigerLog.'), os.listdir('.')))) + self.assertEqual(1, len([x for x in os.listdir('.') if x.startswith('WiredTigerLog.')])) backup_cursor = self.session.open_cursor('backup:') - self.assertEqual(2, len(filter(lambda x: - x.startswith('WiredTigerLog.'), os.listdir('.')))) + self.assertEqual(2, len([x for x in os.listdir('.') if x.startswith('WiredTigerLog.')])) for i in range(10): doc_id += 1 @@ -91,7 +89,7 @@ class test_backup09(wttest.WiredTigerTestCase): os.mkdir(self.backup_dir) if self.all_log_files: helper.copy_wiredtiger_home('.', self.backup_dir) - log_files_copied = filter(lambda x: x.startswith('WiredTigerLog.'), os.listdir(self.backup_dir)) + log_files_copied = [x for x in os.listdir(self.backup_dir) if x.startswith('WiredTigerLog.')] self.assertEqual(len(log_files_copied), 2) else: while True: diff --git a/src/third_party/wiredtiger/test/suite/test_base05.py b/src/third_party/wiredtiger/test/suite/test_base05.py index 60f52bf39ac..688075b4392 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_base05.py +++ b/src/third_party/wiredtiger/test/suite/test_base05.py @@ -28,6 +28,10 @@ import wiredtiger, wttest from wtscenario import make_scenarios +try: + xrange +except NameError: #python3 + xrange = range # test_base05.py # Cursor operations @@ -127,7 +131,7 @@ class test_base05(wttest.WiredTigerTestCase): non_english_strings = [ # This notation creates 'string' objects that have embedded unicode. '\u20320\u22909', - '\u1571\u1604\u1587\u1617\u1604\u1575\u1605\u32\u1593\u1604\u1610\u1603\u1605', + '\u1571\u1604\u1587\u1617\u1604\u1575\u1605\u0032\u1593\u1604\u1610\u1603\u1605', '\u1513\u1500\u1493\u1501', '\u20170\u26085\u12399', '\u50504\u45397\u54616\u49464\u50836', @@ -146,7 +150,7 @@ class test_base05(wttest.WiredTigerTestCase): """ nstrings = 2 << (n % 10) result = '' - for i in range(nstrings): + for i in xrange(nstrings): if (n + i) % 20 == 0: reflist = self.non_english_strings else: @@ -164,7 +168,7 @@ class test_base05(wttest.WiredTigerTestCase): self.pr('creating cursor') cursor = self.session.open_cursor('table:' + self.table_name1) numbers = {} - for i in range(0, self.nentries): + for i in xrange(0, self.nentries): numbers[i] = i key = self.mixed_string(i) value = self.mixed_string(i+1) @@ -172,7 +176,7 @@ class test_base05(wttest.WiredTigerTestCase): # quick spot check to make sure searches work for divisor in [3, 5, 7]: - i = self.nentries / divisor + i = self.nentries // divisor key = self.mixed_string(i) value = self.mixed_string(i+1) cursor.set_key(key) @@ -205,16 +209,16 @@ class test_base05(wttest.WiredTigerTestCase): self.pr('creating cursor') cursor = self.session.open_cursor('table:' + self.table_name1) strlist = self.non_english_strings - for i in range(0, len(strlist)): + for i in xrange(0, len(strlist)): if convert: - key = val = unicode(strlist[i]) + key = val = str(strlist[i]) else: key = val = strlist[i] cursor[key] = val - for i in range(0, len(strlist)): + for i in xrange(0, len(strlist)): if convert: - key = val = unicode(strlist[i]) + key = val = str(strlist[i]) else: key = val = strlist[i] cursor.set_key(key) diff --git a/src/third_party/wiredtiger/test/suite/test_bug021.py b/src/third_party/wiredtiger/test/suite/test_bug021.py index 3c359e46eea..17e63b6a06d 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug021.py +++ b/src/third_party/wiredtiger/test/suite/test_bug021.py @@ -62,8 +62,8 @@ class test_bug021(wttest.WiredTigerTestCase): c.close() if actual != expected: - print 'expected: ', expected - print ' actual: ', actual + print('expected: ', expected) + print(' actual: ', actual) self.assertEqual(expected, actual) def test_implicit_record_cursor_insert_next(self): @@ -72,21 +72,21 @@ class test_bug021(wttest.WiredTigerTestCase): # Check cursor next/operation inside trailing implicit keys. cursor.set_key(62) - self.assertEquals(cursor.search(), 0) - self.assertEquals(cursor.next(), 0) - self.assertEquals(cursor.next(), 0) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.next(), 0) cursor.set_value(3) - self.assertEquals(cursor.insert(), 0) + self.assertEqual(cursor.insert(), 0) current[62 + 2] = 3 self.check(current) # Check cursor prev/operation inside trailing implicit keys. cursor.set_key(68) - self.assertEquals(cursor.search(), 0) - self.assertEquals(cursor.prev(), 0) - self.assertEquals(cursor.prev(), 0) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.prev(), 0) + self.assertEqual(cursor.prev(), 0) cursor.set_value(7) - self.assertEquals(cursor.insert(), 0) + self.assertEqual(cursor.insert(), 0) current[68 - 2] = 7 def test_implicit_record_cursor_insert_prev(self): @@ -95,21 +95,21 @@ class test_bug021(wttest.WiredTigerTestCase): # Check cursor next/operation inside leading implicit keys. cursor.set_key(2) - self.assertEquals(cursor.search(), 0) - self.assertEquals(cursor.next(), 0) - self.assertEquals(cursor.next(), 0) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.next(), 0) cursor.set_value(3) - self.assertEquals(cursor.insert(), 0) + self.assertEqual(cursor.insert(), 0) current[2 + 2] = 3 self.check(current) # Check cursor prev/operation inside leading implicit keys. cursor.set_key(18) - self.assertEquals(cursor.search(), 0) - self.assertEquals(cursor.prev(), 0) - self.assertEquals(cursor.prev(), 0) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.prev(), 0) + self.assertEqual(cursor.prev(), 0) cursor.set_value(7) - self.assertEquals(cursor.insert(), 0) + self.assertEqual(cursor.insert(), 0) current[18 - 2] = 7 self.check(current) @@ -119,19 +119,19 @@ class test_bug021(wttest.WiredTigerTestCase): # Check cursor next/operation inside trailing implicit keys. cursor.set_key(62) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): - self.assertEquals(cursor.next(), 0) - self.assertEquals(cursor.remove(), 0) + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.remove(), 0) current[62 + i] = 0 self.check(current) # Check cursor prev/operation inside trailing implicit keys. cursor.set_key(68) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): - self.assertEquals(cursor.prev(), 0) - self.assertEquals(cursor.remove(), 0) + self.assertEqual(cursor.prev(), 0) + self.assertEqual(cursor.remove(), 0) current[68 - i] = 0 self.check(current) @@ -141,20 +141,20 @@ class test_bug021(wttest.WiredTigerTestCase): # Check cursor next/operation inside leading implicit keys. cursor.set_key(2) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): - self.assertEquals(cursor.next(), 0) - self.assertEquals(cursor.remove(), 0) + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.remove(), 0) current[2 + i] = 0 self.check(current) # Check cursor prev/operation inside leading implicit keys. cursor.set_key(18) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): current[18 - i] = 0 - self.assertEquals(cursor.prev(), 0) - self.assertEquals(cursor.remove(), 0) + self.assertEqual(cursor.prev(), 0) + self.assertEqual(cursor.remove(), 0) current[18 - i] = 0 self.check(current) @@ -164,22 +164,22 @@ class test_bug021(wttest.WiredTigerTestCase): # Check cursor next/operation inside trailing implicit keys. cursor.set_key(62) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): - self.assertEquals(cursor.next(), 0) + self.assertEqual(cursor.next(), 0) cursor.set_value(i) self.session.breakpoint() - self.assertEquals(cursor.update(), 0) + self.assertEqual(cursor.update(), 0) current[62 + i] = i self.check(current) # Check cursor prev/operation inside trailing implicit keys. cursor.set_key(68) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): - self.assertEquals(cursor.prev(), 0) + self.assertEqual(cursor.prev(), 0) cursor.set_value(i) - self.assertEquals(cursor.update(), 0) + self.assertEqual(cursor.update(), 0) current[68 - i] = i self.check(current) @@ -189,22 +189,22 @@ class test_bug021(wttest.WiredTigerTestCase): # Check cursor next/operation inside leading implicit keys. cursor.set_key(2) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): - self.assertEquals(cursor.next(), 0) + self.assertEqual(cursor.next(), 0) cursor.set_value(i) - self.assertEquals(cursor.update(), 0) + self.assertEqual(cursor.update(), 0) current[2 + i] = i self.check(current) # Check cursor prev/operation inside leading implicit keys. cursor.set_key(18) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): current[18 - i] = 0 - self.assertEquals(cursor.prev(), 0) + self.assertEqual(cursor.prev(), 0) cursor.set_value(i) - self.assertEquals(cursor.update(), 0) + self.assertEqual(cursor.update(), 0) current[18 - i] = i self.check(current) diff --git a/src/third_party/wiredtiger/test/suite/test_bulk01.py b/src/third_party/wiredtiger/test/suite/test_bulk01.py index cda17a67b06..79459abc796 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_bulk01.py +++ b/src/third_party/wiredtiger/test/suite/test_bulk01.py @@ -33,6 +33,10 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet, simple_key, simple_value from wtscenario import make_scenarios +try: + xrange +except NameError: #python3 + xrange = range # Smoke test bulk-load. class test_bulk_load(wttest.WiredTigerTestCase): @@ -60,7 +64,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") - for i in range(1, 1000): + for i in xrange(1, 1000): cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Test a bulk-load triggers variable-length column-store RLE correctly. @@ -75,8 +79,8 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") - for i in range(1, 1000): - cursor[simple_key(cursor, i)] = simple_value(cursor, i/7) + for i in xrange(1, 1000): + cursor[simple_key(cursor, i)] = simple_value(cursor, i//7) # Test a bulk-load variable-length column-store append ignores any key. def test_bulk_load_var_append(self): @@ -87,11 +91,11 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk,append") - for i in range(1, 1000): + for i in xrange(1, 1000): cursor[simple_key(cursor, 37)] = simple_value(cursor, i) cursor.close() cursor = self.session.open_cursor(uri, None, None) - for i in range(1, 1000): + for i in xrange(1, 1000): cursor.set_key(simple_key(cursor, i)) cursor.search() self.assertEqual(cursor.get_value(), simple_value(cursor, i)) @@ -105,7 +109,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") - for i in range(1, 1000): + for i in xrange(1, 1000): if i % 7 == 0: cursor[simple_key(cursor, i)] = simple_value(cursor, i) @@ -117,7 +121,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, None) # Verify all the records are there, in their proper state. - for i in range(1, 1000): + for i in xrange(1, 1000): cursor.set_key(simple_key(cursor, i)) if i % 7 == 0: cursor.search() diff --git a/src/third_party/wiredtiger/test/suite/test_calc_modify.py b/src/third_party/wiredtiger/test/suite/test_calc_modify.py index e3326717945..71ec7a45054 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_calc_modify.py +++ b/src/third_party/wiredtiger/test/suite/test_calc_modify.py @@ -28,6 +28,7 @@ import random, string import wiredtiger, wttest +from wtscenario import make_scenarios r = random.Random(42) # Make things repeatable @@ -52,17 +53,27 @@ class test_calc_modify(wttest.WiredTigerTestCase): REMOVE = 2 REPLACE = 3 + valuefmt = [ + ('item', dict(valuefmt='u')), + ('string', dict(valuefmt='S')), + ] + scenarios = make_scenarios(valuefmt) + def mkstring(self, size, repeat_size=1): - pattern = ''.join(r.choice(string.ascii_letters + string.digits) for _ in xrange(repeat_size)) - return (pattern * ((size + repeat_size - 1) / repeat_size))[:size] + choices = string.ascii_letters + string.digits + if self.valuefmt == 'S': + pattern = ''.join(r.choice(choices) for _ in range(repeat_size)) + else: + pattern = b''.join(bytes([r.choice(choices.encode())]) for _ in range(repeat_size)) + return (pattern * ((size + repeat_size - 1) // repeat_size))[:size] def one_test(self, c, k, oldsz, repeatsz, nmod, maxdiff): oldv = self.mkstring(oldsz, repeatsz) - offsets = sorted(r.sample(xrange(oldsz), nmod)) - modsizes = sorted(r.sample(xrange(maxdiff), nmod + 1)) - lengths = [modsizes[i+1] - modsizes[i] for i in xrange(nmod)] - modtypes = [r.choice((self.ADD, self.REMOVE, self.REPLACE)) for _ in xrange(nmod)] + offsets = sorted(r.sample(range(oldsz), nmod)) + modsizes = sorted(r.sample(range(maxdiff), nmod + 1)) + lengths = [modsizes[i+1] - modsizes[i] for i in range(nmod)] + modtypes = [r.choice((self.ADD, self.REMOVE, self.REPLACE)) for _ in range(nmod)] self.pr("offsets: %s" % offsets) self.pr("modsizes: %s" % modsizes) @@ -70,8 +81,8 @@ class test_calc_modify(wttest.WiredTigerTestCase): self.pr("modtypes: %s" % modtypes) orig = oldv - newv = '' - for i in xrange(nmod): + newv = '' if self.valuefmt == 'S' else b'' + for i in range(nmod): if i > 0 and offsets[i] - offsets[i - 1] < maxdiff: continue newv += orig[:offsets[i]] @@ -104,13 +115,13 @@ class test_calc_modify(wttest.WiredTigerTestCase): self.assertEqual(c[k], newv) def test_calc_modify(self): - self.session.create(self.uri, 'key_format=i,value_format=u') + self.session.create(self.uri, 'key_format=i,value_format=' + self.valuefmt) c = self.session.open_cursor(self.uri) - for k in xrange(1000): + for k in range(1000): size = r.randint(1000, 10000) repeats = r.randint(1, size) nmods = r.randint(1, 10) - maxdiff = r.randint(64, size / 10) + maxdiff = r.randint(64, size // 10) self.pr("size %s, repeats %s, nmods %s, maxdiff %s" % (size, repeats, nmods, maxdiff)) self.one_test(c, k, size, repeats, nmods, maxdiff) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint01.py b/src/third_party/wiredtiger/test/suite/test_checkpoint01.py index 8c758943f68..02849bc773f 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint01.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint01.py @@ -67,14 +67,14 @@ class test_checkpoint(wttest.WiredTigerTestCase): # checkpoint the object, and verify it (which verifies all underlying # checkpoints individually). def build_file_with_checkpoints(self): - for checkpoint_name, entry in self.checkpoints.iteritems(): + for checkpoint_name, entry in self.checkpoints.items(): self.add_records(checkpoint_name) self.session.checkpoint("name=" + checkpoint_name) # Create a dictionary of sorted records a checkpoint should include. def list_expected(self, name): records = {} - for checkpoint_name, entry in self.checkpoints.iteritems(): + for checkpoint_name, entry in self.checkpoints.items(): start, stop = entry[0] for i in range(start, stop+1): records['%010d KEY------' % i] =\ @@ -98,7 +98,7 @@ class test_checkpoint(wttest.WiredTigerTestCase): # Physically verify the file, including the individual checkpoints. self.session.verify(self.uri, None) - for checkpoint_name, entry in self.checkpoints.iteritems(): + for checkpoint_name, entry in self.checkpoints.items(): if entry[1] == 0: self.assertRaises(wiredtiger.WiredTigerError, lambda: self.session.open_cursor( @@ -131,7 +131,7 @@ class test_checkpoint(wttest.WiredTigerTestCase): # Drop remaining checkpoints, all subsequent checkpoint opens should # fail. self.session.checkpoint("drop=(from=all)") - for checkpoint_name, entry in self.checkpoints.iteritems(): + for checkpoint_name, entry in self.checkpoints.items(): self.checkpoints[checkpoint_name] =\ (self.checkpoints[checkpoint_name][0], 0) self.check() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py index a3f37474cd9..3a738b52103 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py @@ -26,10 +26,17 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import Queue +try: + import Queue as queue # python2 +except ImportError: + import queue import threading, time, wiredtiger, wttest from wtthread import checkpoint_thread, op_thread from wtscenario import make_scenarios +try: + xrange +except NameError: #python3 + xrange = range # test_checkpoint02.py # Run background checkpoints repeatedly while doing inserts and other @@ -49,20 +56,20 @@ class test_checkpoint02(wttest.WiredTigerTestCase): uris = list() uris.append(self.uri) - queue = Queue.Queue() + work_queue = queue.Queue() my_data = 'a' * self.dsize for i in xrange(self.nops): if i % 191 == 0 and i != 0: - queue.put_nowait(('b', i, my_data)) - queue.put_nowait(('i', i, my_data)) + work_queue.put_nowait(('b', i, my_data)) + work_queue.put_nowait(('i', i, my_data)) opthreads = [] - for i in xrange(self.nthreads): - t = op_thread(self.conn, uris, self.fmt, queue, done) + for i in range(self.nthreads): + t = op_thread(self.conn, uris, self.fmt, work_queue, done) opthreads.append(t) t.start() - queue.join() + work_queue.join() done.set() for t in opthreads: t.join() diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py index b4fa534c354..5bfdd2945fa 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact02.py +++ b/src/third_party/wiredtiger/test/suite/test_compact02.py @@ -77,7 +77,7 @@ class test_compact02(wttest.WiredTigerTestCase): bigvalue = "abcdefghi" * 1074 # 9*1074 == 9666 smallvalue = "ihgfedcba" * 303 # 9*303 == 2727 - fullsize = nrecords / 2 * len(bigvalue) + nrecords / 2 * len(smallvalue) + fullsize = nrecords // 2 * len(bigvalue) + nrecords // 2 * len(smallvalue) # Return the size of the file def getSize(self): @@ -105,7 +105,7 @@ class test_compact02(wttest.WiredTigerTestCase): try: self.conn = wiredtiger.wiredtiger_open(self.home, conn_params) except wiredtiger.WiredTigerError as e: - print "Failed conn at '%s' with config '%s'" % (dir, conn_params) + print("Failed conn at '%s' with config '%s'" % (dir, conn_params)) self.session = self.conn.open_session(None) # Create a table, add keys with both big and small values. @@ -128,7 +128,7 @@ class test_compact02(wttest.WiredTigerTestCase): # 2. Checkpoint and get stats on the table to confirm the size. self.session.checkpoint() sz = self.getSize() - self.pr('After populate ' + str(sz / mb) + 'MB') + self.pr('After populate ' + str(sz // mb) + 'MB') self.assertGreater(sz, self.fullsize) # 3. Delete the half of the records with the larger record size. @@ -140,7 +140,7 @@ class test_compact02(wttest.WiredTigerTestCase): c.set_key(i) c.remove() c.close() - self.pr('Removed total ' + str((count * 9666) / mb) + 'MB') + self.pr('Removed total ' + str((count * 9666) // mb) + 'MB') # 4. Checkpoint self.session.checkpoint() @@ -157,10 +157,10 @@ class test_compact02(wttest.WiredTigerTestCase): # 6. Get stats on compacted table. sz = self.getSize() - self.pr('After compact ' + str(sz / mb) + 'MB') + self.pr('After compact ' + str(sz // mb) + 'MB') # After compact, the file size should be less than half the full size. - self.assertLess(sz, self.fullsize / 2) + self.assertLess(sz, self.fullsize // 2) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_compress01.py b/src/third_party/wiredtiger/test/suite/test_compress01.py index 7406e971d19..b344e194ace 100644 --- a/src/third_party/wiredtiger/test/suite/test_compress01.py +++ b/src/third_party/wiredtiger/test/suite/test_compress01.py @@ -66,12 +66,12 @@ class test_compress01(wttest.WiredTigerTestCase): params = 'key_format=S,value_format=S,leaf_page_max=4096' self.session.create(self.uri, params) cursor = self.session.open_cursor(self.uri, None) - for idx in xrange(1,self.nrecords): - cursor.set_key(`idx`) - if idx / 12 == 0: - cursor.set_value(`idx` + self.bigvalue) + for idx in range(1,self.nrecords): + cursor.set_key(repr(idx)) + if idx // 12 == 0: + cursor.set_value(repr(idx) + self.bigvalue) else: - cursor.set_value(`idx` + "abcdefg") + cursor.set_value(repr(idx) + "abcdefg") cursor.insert() cursor.close() @@ -79,13 +79,13 @@ class test_compress01(wttest.WiredTigerTestCase): self.reopen_conn() cursor = self.session.open_cursor(self.uri, None) - for idx in xrange(1,self.nrecords): - cursor.set_key(`idx`) + for idx in range(1,self.nrecords): + cursor.set_key(repr(idx)) self.assertEqual(cursor.search(), 0) - if idx / 12 == 0: - self.assertEquals(cursor.get_value(), `idx` + self.bigvalue) + if idx // 12 == 0: + self.assertEqual(cursor.get_value(), repr(idx) + self.bigvalue) else: - self.assertEquals(cursor.get_value(), `idx` + "abcdefg") + self.assertEqual(cursor.get_value(), repr(idx) + "abcdefg") cursor.close() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_config01.py b/src/third_party/wiredtiger/test/suite/test_config01.py index e77e80a1bce..5a866613c05 100644 --- a/src/third_party/wiredtiger/test/suite/test_config01.py +++ b/src/third_party/wiredtiger/test/suite/test_config01.py @@ -42,7 +42,7 @@ class test_config01(test_base03.test_base03): if hasattr(self, 'cache_size'): wtopen_args += ',cache_size=' + str(self.cache_size) conn = self.wiredtiger_open(dir, wtopen_args) - self.pr(`conn`) + self.pr(repr(conn)) return conn if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_config02.py b/src/third_party/wiredtiger/test/suite/test_config02.py index 82a56a0a245..9fbd12f23ab 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_config02.py +++ b/src/third_party/wiredtiger/test/suite/test_config02.py @@ -152,7 +152,7 @@ class test_config02(wttest.WiredTigerTestCase): self.skipTest('Unix specific test skipped on Windows') dir = 'subdir' os.mkdir(dir) - os.chmod(dir, 0555) + os.chmod(dir, 0o555) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.wiredtiger_open(dir, 'create'), '/Permission denied/') diff --git a/src/third_party/wiredtiger/test/suite/test_config03.py b/src/third_party/wiredtiger/test/suite/test_config03.py index c5f98f85898..a5d9ee199a0 100644 --- a/src/third_party/wiredtiger/test/suite/test_config03.py +++ b/src/third_party/wiredtiger/test/suite/test_config03.py @@ -123,7 +123,7 @@ class test_config03(test_base03.test_base03): self.verbose(3, 'wiredtiger_open with args: ' + args) conn = self.wiredtiger_open(dir, args) - self.pr(`conn`) + self.pr(repr(conn)) return conn if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_config04.py b/src/third_party/wiredtiger/test/suite/test_config04.py index b4045814c13..b4045814c13 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_config04.py +++ b/src/third_party/wiredtiger/test/suite/test_config04.py diff --git a/src/third_party/wiredtiger/test/suite/test_cursor01.py b/src/third_party/wiredtiger/test/suite/test_cursor01.py index 6da208e2f45..770b433cf14 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor01.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor01.py @@ -55,7 +55,7 @@ class test_cursor01(wttest.WiredTigerTestCase): if self.tablekind == 'row': return 'key' + str(i) else: - return long(i+1) + return self.recno(i+1) def genvalue(self, i): if self.tablekind == 'fix': diff --git a/src/third_party/wiredtiger/test/suite/test_cursor04.py b/src/third_party/wiredtiger/test/suite/test_cursor04.py index 0ff539a252f..b57d24da847 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor04.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor04.py @@ -82,7 +82,7 @@ class test_cursor04(wttest.WiredTigerTestCase): if self.tablekind == 'row': return 'key' + str(i).zfill(5) # return key00001, key00002, etc. else: - return long(i+1) + return self.recno(i+1) def genvalue(self, i): if self.tablekind == 'fix': diff --git a/src/third_party/wiredtiger/test/suite/test_cursor07.py b/src/third_party/wiredtiger/test/suite/test_cursor07.py index c26720f13e1..82a606ed1bc 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor07.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor07.py @@ -59,8 +59,8 @@ class test_cursor07(wttest.WiredTigerTestCase, suite_subprocess): def test_log_cursor(self): # print "Creating %s with config '%s'" % (self.uri, self.create_params) - create_params = 'key_format=i,value_format=S' - create_nolog_params = 'key_format=i,value_format=S,log=(enabled=false)' + create_params = 'key_format=i,value_format=u' + create_nolog_params = 'key_format=i,value_format=u,log=(enabled=false)' self.session.create(self.uri1, create_params) c1 = self.session.open_cursor(self.uri1, None) self.session.create(self.uri2, create_nolog_params) @@ -69,8 +69,8 @@ class test_cursor07(wttest.WiredTigerTestCase, suite_subprocess): c3 = self.session.open_cursor(self.uri3, None) # A binary value. - value = u'\u0001\u0002abcd\u0003\u0004' - value_nolog = u'\u0001\u0002dcba\u0003\u0004' + value = b'\x01\x02abcd\x03\x04' + value_nolog = b'\x01\x02dcba\x03\x04' # We want to test both adding data to a table that is not logged # that is part of the same transaction as a table that is logged @@ -100,12 +100,11 @@ class test_cursor07(wttest.WiredTigerTestCase, suite_subprocess): keys = c.get_key() # txnid, rectype, optype, fileid, logrec_key, logrec_value values = c.get_value() - try: - if value in str(values[5]): # logrec_value + # We are only looking for log records that that have a key/value + # pair. + if values[4] != b'': + if value in values[5]: # logrec_value count += 1 - self.assertFalse(value2 in str(values[5])) - except: - pass c.close() self.assertEqual(count, self.nkeys) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor08.py b/src/third_party/wiredtiger/test/suite/test_cursor08.py index f1e52607cf2..a21c259c879 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor08.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor08.py @@ -70,7 +70,7 @@ class test_cursor08(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(self.uri, None) # A binary value. - value = u'\u0001\u0002abcd\u0003\u0004' + value = '\u0001\u0002abcd\u0003\u0004' self.session.begin_transaction() for k in range(self.nkeys): @@ -89,11 +89,11 @@ class test_cursor08(wttest.WiredTigerTestCase, suite_subprocess): keys = c.get_key() # txnid, rectype, optype, fileid, logrec_key, logrec_value values = c.get_value() - try: - if value in str(values[5]): # logrec_value + # We are only looking for log records that that have a key/value + # pair. + if values[4] != b'': + if value.encode() in values[5]: # logrec_value count += 1 - except: - pass c.close() self.assertEqual(count, self.nkeys) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor10.py b/src/third_party/wiredtiger/test/suite/test_cursor10.py index 7cb12248512..f6b100ffd21 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor10.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor10.py @@ -47,7 +47,7 @@ class test_cursor10(wttest.WiredTigerTestCase): if self.key_format == 'S': return 'key' + str(i).zfill(5) # return key00001, key00002, etc. else: - return long(i+1) + return self.recno(i+1) def genvalue(self, i): return [ 'v0:' + str(i), i+1, 'v2' + str(i+2), i+3 ] @@ -56,7 +56,7 @@ class test_cursor10(wttest.WiredTigerTestCase): if self.key_format == 'S': return int(k[3:]) else: - return long(k-1) + return self.recno(k-1) def test_projection(self): """ diff --git a/src/third_party/wiredtiger/test/suite/test_cursor12.py b/src/third_party/wiredtiger/test/suite/test_cursor12.py index 84fdad8f242..121180430f7 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor12.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor12.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import random, string +import random, string, sys import wiredtiger, wttest from helper import copy_wiredtiger_home from wtdataset import SimpleDataSet @@ -180,6 +180,39 @@ class test_cursor12(wttest.WiredTigerTestCase): } ] + def setUp(self): + if sys.version_info[0] >= 3 and self.valuefmt == 'u': + # Python3 distinguishes bytes from strings + self.nullbyte = b'\x00' + self.spacebyte = b' ' + else: + self.nullbyte = '\x00' + self.spacebyte = ' ' + super(test_cursor12, self).setUp() + + # Convert a string to the correct type for the value. + def make_value(self, s): + if self.valuefmt == 'u': + return bytes(s.encode()) + else: + return s + + def fix_mods(self, mods): + if bytes != str and self.valuefmt == 'u': + # In Python3, bytes and strings are independent types, and + # the WiredTiger API needs bytes when the format calls for bytes. + newmods = [] + for mod in mods: + # We need to check because we may converted some of the Modify + # records already. + if type(mod.data) == str: + newmods.append(wiredtiger.Modify( + self.make_value(mod.data), mod.offset, mod.size)) + else: + newmods.append(mod) + mods = newmods + return mods + # Create a set of modified records and verify in-memory reads. def modify_load(self, ds, single): # For each test in the list: @@ -190,7 +223,7 @@ class test_cursor12(wttest.WiredTigerTestCase): c = self.session.open_cursor(self.uri, None) for i in self.list: c.set_key(ds.key(row)) - c.set_value(i['o']) + c.set_value(self.make_value(i['o'])) self.assertEquals(c.update(), 0) c.reset() @@ -200,6 +233,7 @@ class test_cursor12(wttest.WiredTigerTestCase): for j in i['mods']: mod = wiredtiger.Modify(j[0], j[1], j[2]) mods.append(mod) + mods = self.fix_mods(mods) self.assertEquals(c.modify(mods), 0) self.session.commit_transaction() c.reset() @@ -207,7 +241,8 @@ class test_cursor12(wttest.WiredTigerTestCase): c.set_key(ds.key(row)) self.assertEquals(c.search(), 0) v = c.get_value() - self.assertEquals(v.replace("\x00", " "), i['f']) + expect = self.make_value(i['f']) + self.assertEquals(v.replace(self.nullbyte, self.spacebyte), expect) if not single: row = row + 1 @@ -223,7 +258,8 @@ class test_cursor12(wttest.WiredTigerTestCase): c.set_key(ds.key(row)) self.assertEquals(c.search(), 0) v = c.get_value() - self.assertEquals(v.replace("\x00", " "), i['f']) + expect = self.make_value(i['f']) + self.assertEquals(v.replace(self.nullbyte, self.spacebyte), expect) if not single: row = row + 1 @@ -292,15 +328,17 @@ class test_cursor12(wttest.WiredTigerTestCase): c = self.session.open_cursor(self.uri, None) self.session.begin_transaction() c.set_key(ds.key(10)) - orig = 'abcdefghijklmnopqrstuvwxyz' + orig = self.make_value('abcdefghijklmnopqrstuvwxyz') c.set_value(orig) self.assertEquals(c.update(), 0) for i in range(0, 50000): - new = "".join([random.choice(string.digits) for i in xrange(5)]) + new = self.make_value("".join([random.choice(string.digits) \ + for i in range(5)])) orig = orig[:10] + new + orig[15:] mods = [] mod = wiredtiger.Modify(new, 10, 5) mods.append(mod) + mods = self.fix_mods(mods) self.assertEquals(c.modify(mods), 0) self.session.commit_transaction() @@ -322,6 +360,7 @@ class test_cursor12(wttest.WiredTigerTestCase): mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) + mods = self.fix_mods(mods) c.set_key(ds.key(10)) self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND) @@ -348,6 +387,7 @@ class test_cursor12(wttest.WiredTigerTestCase): mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) c.set_key(ds.key(30)) + mods = self.fix_mods(mods) self.assertEqual(c.modify(mods), 0) # Test that another transaction cannot modify our uncommitted record. @@ -359,6 +399,7 @@ class test_cursor12(wttest.WiredTigerTestCase): mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) + mods = self.fix_mods(mods) xc.set_key(ds.key(30)) self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND) xs.rollback_transaction() @@ -371,6 +412,7 @@ class test_cursor12(wttest.WiredTigerTestCase): mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) + mods = self.fix_mods(mods) c.set_key(ds.key(30)) self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor13.py b/src/third_party/wiredtiger/test/suite/test_cursor13.py index 3391f1e03ac..6f9126cce6a 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor13.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor13.py @@ -375,11 +375,11 @@ class test_cursor13_big_base(test_cursor13_base): # create some number (self.deep) of cached cursors. def create_uri_map(self, baseuri): uri_map = {} - for i in xrange(0, self.nuris): + for i in range(0, self.nuris): uri = self.uriname(i) cursors = [] self.session.create(uri, None) - for j in xrange(0, self.deep): + for j in range(0, self.deep): cursors.append(self.session.open_cursor(uri, None)) for c in cursors: c.close() @@ -478,8 +478,8 @@ class test_cursor13_sweep(test_cursor13_big_base): # Close cursors in half of the range, and don't # use them during this round, so they will be # closed by sweep. - half = self.nuris / 2 - potential_dead += self.close_uris(uri_map, xrange(0, half)) + half = self.nuris // 2 + potential_dead += self.close_uris(uri_map, list(range(0, half))) bottom_range = half # Let the dhandle sweep run and find the closed cursors. time.sleep(3.0) @@ -489,7 +489,7 @@ class test_cursor13_sweep(test_cursor13_big_base): # The session cursor sweep runs at most once a second and # traverses a fraction of the cached cursors. We'll run for # ten seconds with pauses to make sure we see sweep activity. - pause_point = self.opens_per_round / 100 + pause_point = self.opens_per_round // 100 if pause_point == 0: pause_point = 1 pause_duration = 0.1 @@ -522,7 +522,7 @@ class test_cursor13_sweep(test_cursor13_big_base): # We'll pass the test if we see at least 20% of the 'potentially # dead' cursors swept. There may be more, since the 1% per second # is a minimum. - min_swept = 2 * potential_dead / 10 + min_swept = 2 * potential_dead // 10 self.assertGreaterEqual(swept, min_swept) # No strict equality test for the reopen stats. When we've swept diff --git a/src/third_party/wiredtiger/test/suite/test_cursor14.py b/src/third_party/wiredtiger/test/suite/test_cursor14.py index f650f922a09..4d37069b221 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor14.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor14.py @@ -53,7 +53,7 @@ class test_cursor14(wttest.WiredTigerTestCase): ds = self.dataset(self, uri, 100, key_format=self.keyfmt) ds.populate() - for i in xrange(66000): + for i in range(66000): cursor = self.session.open_cursor(uri, None, None) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_compare.py b/src/third_party/wiredtiger/test/suite/test_cursor_compare.py index 2597a8592a3..49682e48215 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor_compare.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_compare.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import wiredtiger, wttest, exceptions +import wiredtiger, wttest from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import filter_scenarios, make_scenarios @@ -99,7 +99,7 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): wiredtiger.WiredTigerError, lambda: cX.compare(c1), msg) msg = '/wt_cursor.* is None/' self.assertRaisesHavingMessage( - exceptions.RuntimeError, lambda: cX.compare(None), msg) + RuntimeError, lambda: cX.compare(None), msg) if ix0_0 != None: self.assertEqual(ix0_0.compare(ix0_1), 0) ix0_1.reset() @@ -188,7 +188,7 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): wiredtiger.WiredTigerError, lambda: cX.equals(c1), msg) msg = '/wt_cursor.* is None/' self.assertRaisesHavingMessage( - exceptions.RuntimeError, lambda: cX.equals(None), msg) + RuntimeError, lambda: cX.equals(None), msg) if ix0_0 != None: self.assertTrue(ix0_0.equals(ix0_1)) ix0_1.reset() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_pin.py b/src/third_party/wiredtiger/test/suite/test_cursor_pin.py index 8b9784b10d0..692f6e09c26 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_pin.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_pin.py @@ -111,11 +111,11 @@ class test_cursor_pin(wttest.WiredTigerTestCase): for i in range(self.nentries + 1000, self.nentries + 2001): c[ds.key(i)] = ds.value(i) self.forward(c, ds, self.nentries + 5000, - list(range(self.nentries + 1, self.nentries + 1000) +\ - range(self.nentries + 2001, self.nentries + 3000))) + list(list(range(self.nentries + 1, self.nentries + 1000)) +\ + list(range(self.nentries + 2001, self.nentries + 3000)))) self.backward(c, ds, self.nentries + 5000, - list(range(self.nentries + 1, self.nentries + 1000) +\ - range(self.nentries + 2001, self.nentries + 3000))) + list(list(range(self.nentries + 1, self.nentries + 1000)) +\ + list(range(self.nentries + 2001, self.nentries + 3000)))) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_random02.py b/src/third_party/wiredtiger/test/suite/test_cursor_random02.py index 46faa0ce3fc..6532247e271 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_random02.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_random02.py @@ -87,13 +87,13 @@ class test_cursor_random02(wttest.WiredTigerTestCase): ''' self.tty('differentKeys: ' + str(differentKeys) + ' of ' + \ str(num_entries) + ', ' + \ - str((int)((differentKeys * 100) / num_entries)) + '%') + str((int)((differentKeys * 100) // num_entries)) + '%') ''' # Can't test for non-sequential data when there is 1 item in the table if num_entries > 1: self.assertGreater(num_entries - 1, sequentialKeys, 'cursor is returning sequential data') - self.assertGreater(differentKeys, num_entries / 4, + self.assertGreater(differentKeys, num_entries // 4, 'next_random random distribution not adequate') if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py b/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py index 6c96b85dd1e..967bf81138f 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py @@ -179,7 +179,7 @@ class TestCursorTracker(wttest.WiredTigerTestCase): def stretch_content(self, s, sizes): result = s if sizes != None: - sha224 = hashlib.sha224(s) + sha224 = hashlib.sha224(s.encode()) md5 = sha224.digest() low = sizes[0] - len(s) if low < 0: @@ -188,7 +188,7 @@ class TestCursorTracker(wttest.WiredTigerTestCase): if high < 0: high = 0 diff = high - low - nextra = (ord(md5[4]) % (diff + 1)) + low + nextra = (self.ord_byte(md5[4]) % (diff + 1)) + low extra = sha224.hexdigest() while len(extra) < nextra: extra = extra + extra @@ -237,7 +237,7 @@ class TestCursorTracker(wttest.WiredTigerTestCase): # 64 bit key maj = ((bits >> 32) & 0xffffffff) + 1 min = (bits >> 16) & 0xffff - return long((maj << 16) | min) + return self.recno((maj << 16) | min) def decode_key_col_or_fix(self, bits): maj = ((bits << 16) & 0xffffffff) - 1 @@ -296,7 +296,7 @@ class TestCursorTracker(wttest.WiredTigerTestCase): def bitspos(self, bits): list = self.bitlist - return next(i for i in xrange(len(list)) if list[i] == bits) + return next(i for i in range(len(list)) if list[i] == bits) def cur_insert(self, cursor, major, minor): bits = self.triple_to_bits(major, minor, 0) @@ -333,7 +333,7 @@ class TestCursorTracker(wttest.WiredTigerTestCase): cursor.remove() def cur_recno_search(self, cursor, recno): - wtkey = long(recno) + wtkey = self.recno(recno) self.traceapi('cursor.set_key(' + str(wtkey) + ')') cursor.set_key(wtkey) if recno > 0 and recno <= len(self.bitlist): diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py index e606a223953..9546c8f267c 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py @@ -43,9 +43,9 @@ class test_durable_ts03(wttest.WiredTigerTestCase): uri = 'table:test_durable_ts03' nrows = 3000 self.session.create(uri, 'key_format=i,value_format=u') - valueA = "aaaaa" * 100 - valueB = "bbbbb" * 100 - valueC = "ccccc" * 100 + valueA = b"aaaaa" * 100 + valueB = b"bbbbb" * 100 + valueC = b"ccccc" * 100 # Start with setting a stable and oldest timestamp. self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1) + \ diff --git a/src/third_party/wiredtiger/test/suite/test_empty.py b/src/third_party/wiredtiger/test/suite/test_empty.py index 9b4c53b9774..f7949871a11 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_empty.py +++ b/src/third_party/wiredtiger/test/suite/test_empty.py @@ -46,7 +46,7 @@ class test_empty(wttest.WiredTigerTestCase): # Creating an object and then closing it shouldn't write any blocks. def test_empty_create(self): uri = self.type + self.name - self.session.create(uri, 'key_format=' + self.fmt) + self.session.create(uri, 'key_format=' + self.fmt + ',value_format=S') self.session.close() name = self.name if self.type == "table:": @@ -59,7 +59,7 @@ class test_empty(wttest.WiredTigerTestCase): def empty(self): uri = self.type + self.name self.session = self.conn.open_session() - self.session.create(uri, 'key_format=' + self.fmt) + self.session.create(uri, 'key_format=' + self.fmt + ',value_format=S') # Add a few records to the object and remove them. cursor = self.session.open_cursor(uri, None, None) diff --git a/src/third_party/wiredtiger/test/suite/test_empty_value.py b/src/third_party/wiredtiger/test/suite/test_empty_value.py index 6374f4c2a77..27ff6f2391c 100644 --- a/src/third_party/wiredtiger/test/suite/test_empty_value.py +++ b/src/third_party/wiredtiger/test/suite/test_empty_value.py @@ -47,7 +47,7 @@ class test_row_store_empty_values(wttest.WiredTigerTestCase): # Create the object, open the cursor, insert some records with zero-length values. self.session.create(uri, 'value_format=u,key_format=S') cursor = self.session.open_cursor(uri, None) - for i in xrange(1, nentries + 1): + for i in range(1, nentries + 1): cursor[simple_key(cursor, i)] = "" cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt01.py b/src/third_party/wiredtiger/test/suite/test_encrypt01.py index 323a6c7add2..21a507b7aed 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt01.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt01.py @@ -94,7 +94,7 @@ class test_encrypt01(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri, None) r = random.Random() r.seed(0) - for idx in xrange(1,self.nrecords): + for idx in range(1,self.nrecords): start = r.randint(0,9) key = self.bigvalue[start:r.randint(0,100)] + str(idx) val = self.bigvalue[start:r.randint(0,10000)] + str(idx) @@ -109,7 +109,7 @@ class test_encrypt01(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri, None) r.seed(0) - for idx in xrange(1,self.nrecords): + for idx in range(1,self.nrecords): start = r.randint(0,9) key = self.bigvalue[start:r.randint(0,100)] + str(idx) val = self.bigvalue[start:r.randint(0,10000)] + str(idx) diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt02.py b/src/third_party/wiredtiger/test/suite/test_encrypt02.py index 38ff2c04202..5dba7918105 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt02.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt02.py @@ -70,7 +70,7 @@ class test_encrypt02(wttest.WiredTigerTestCase, suite_subprocess): cursor = self.session.open_cursor(self.uri, None) r = random.Random() r.seed(0) - for idx in xrange(1,self.nrecords): + for idx in range(1,self.nrecords): start = r.randint(0,9) key = self.bigvalue[start:r.randint(0,100)] + str(idx) val = self.bigvalue[start:r.randint(0,10000)] + str(idx) @@ -85,7 +85,7 @@ class test_encrypt02(wttest.WiredTigerTestCase, suite_subprocess): cursor = self.session.open_cursor(self.uri, None) r.seed(0) - for idx in xrange(1,self.nrecords): + for idx in range(1,self.nrecords): start = r.randint(0,9) key = self.bigvalue[start:r.randint(0,100)] + str(idx) val = self.bigvalue[start:r.randint(0,10000)] + str(idx) diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt04.py b/src/third_party/wiredtiger/test/suite/test_encrypt04.py index 46be3d4b77a..e58c97b5bac 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt04.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt04.py @@ -120,11 +120,11 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): if str(-1000) in str(err): self.got_forceerror = True raise - self.pr(`conn`) + self.pr(repr(conn)) return conn def create_records(self, cursor, r, low, high): - for idx in xrange(low, high): + for idx in range(low, high): start = r.randint(0,9) key = self.bigvalue[start:r.randint(0,100)] + str(idx) val = self.bigvalue[start:r.randint(0,10000)] + str(idx) @@ -133,13 +133,13 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): cursor.insert() def check_records(self, cursor, r, low, high): - for idx in xrange(low, high): + for idx in range(low, high): start = r.randint(0,9) key = self.bigvalue[start:r.randint(0,100)] + str(idx) val = self.bigvalue[start:r.randint(0,10000)] + str(idx) cursor.set_key(key) self.assertEqual(cursor.search(), 0) - self.assertEquals(cursor.get_value(), val) + self.assertEqual(cursor.get_value(), val) # Evaluate expression, which either must succeed (if expect_okay) # or must fail (if !expect_okay). diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt06.py b/src/third_party/wiredtiger/test/suite/test_encrypt06.py index 1bec3b04570..7663a8d6d7e 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_encrypt06.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt06.py @@ -110,8 +110,9 @@ class test_encrypt06(wttest.WiredTigerTestCase): return (f.read().find(match) != -1) def match_string_in_rundir(self, match): + byte_match = match.encode() for fname in os.listdir('.'): - if self.match_string_in_file(fname, match): + if self.match_string_in_file(fname, byte_match): return True return False @@ -177,7 +178,7 @@ class test_encrypt06(wttest.WiredTigerTestCase): c0 = s.open_cursor(pfx + name0, None) c1 = s.open_cursor(pfx + name1, None) - for idx in xrange(1,self.nrecords): + for idx in range(1,self.nrecords): c0.set_key(str(idx) + txt0) c1.set_key(str(idx) + txt1) c0.set_value(txt0 * (idx % 97), txt0 * 3, txt0 * 5, txt0 * 7) diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt07.py b/src/third_party/wiredtiger/test/suite/test_encrypt07.py index d7ece82ded8..6577e62fa20 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_encrypt07.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt07.py @@ -61,7 +61,7 @@ class test_encrypt07(test_salvage.test_salvage): # (to find a physical spot to damage) we'll need to search for # the rot13 encrypted string. def damage(self, tablename): - self.damage_inner(tablename, self.rot13(self.unique)) + self.damage_inner(tablename, self.rot13(self.unique).encode()) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_env01.py b/src/third_party/wiredtiger/test/suite/test_env01.py index 79ea7b1a594..b4c8591a024 100644 --- a/src/third_party/wiredtiger/test/suite/test_env01.py +++ b/src/third_party/wiredtiger/test/suite/test_env01.py @@ -126,7 +126,7 @@ class test_priv01(wttest.WiredTigerTestCase): edir = 'envdir' os.mkdir(edir) if os.getuid() != os.geteuid(): - print 'Running ' + str(self) + ' as privileged user' + print('Running ' + str(self) + ' as privileged user') self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.common_test(None, edir, None), '/WIREDTIGER_HOME environment variable set but\ @@ -137,7 +137,7 @@ class test_priv01(wttest.WiredTigerTestCase): os.mkdir(edir) privarg = 'use_environment_priv=true' if os.getuid() != os.geteuid(): - print 'Running ' + str(self) + ' as privileged user' + print('Running ' + str(self) + ' as privileged user') self.common_test(None, edir, privarg) self.checkfiles(edir) self.checknofiles(".") diff --git a/src/third_party/wiredtiger/test/suite/test_index01.py b/src/third_party/wiredtiger/test/suite/test_index01.py index cdd0650bca6..dbf47b22b7e 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_index01.py +++ b/src/third_party/wiredtiger/test/suite/test_index01.py @@ -37,9 +37,10 @@ class test_index01(wttest.WiredTigerTestCase): tablename = 'table:' + basename indexbase = 'index:' + basename NUM_INDICES = 6 - index = ['%s:index%d' % (indexbase, i) for i in xrange(NUM_INDICES)] def create_table(self): + self.index = ['%s:index%d' % (self.indexbase, i) \ + for i in range(self.NUM_INDICES)] self.pr('create table') self.session.create(self.tablename, 'key_format=Si,value_format=SSii,columns=(name,ID,dept,job,salary,year)') self.session.create(self.index[0], 'columns=(dept)') @@ -130,7 +131,7 @@ class test_index01(wttest.WiredTigerTestCase): '''Create a table, look for a nonexistent key''' self.create_table() self.check_exists('jones', 10, wiredtiger.WT_NOTFOUND) - for i in xrange(self.NUM_INDICES): + for i in range(self.NUM_INDICES): self.assertEqual(list(self.index_iter(i)), []) self.drop_table() @@ -140,7 +141,7 @@ class test_index01(wttest.WiredTigerTestCase): self.insert('smith', 1, 'HR', 'manager', 100000, 1970) self.check_exists('smith', 1, 0) result = '' - for i in xrange(self.NUM_INDICES): + for i in range(self.NUM_INDICES): result += '\n'.join(repr(cols) for cols in self.index_iter(i)) result += '\n\n' @@ -162,7 +163,7 @@ class test_index01(wttest.WiredTigerTestCase): self.update_nonexistent('Smith', 1, 'HR', 'janitor', 1000, 1970) self.check_exists('smith', 1, 0) result = '' - for i in xrange(self.NUM_INDICES): + for i in range(self.NUM_INDICES): result += '\n'.join(repr(cols) for cols in self.index_iter(i)) result += '\n\n' @@ -186,7 +187,7 @@ class test_index01(wttest.WiredTigerTestCase): self.check_exists('jones', 2, 0) self.insert_duplicate('smith', 1, 'HR', 'manager', 100000, 1970) result = '' - for i in xrange(self.NUM_INDICES): + for i in range(self.NUM_INDICES): result += '\n'.join(repr(cols) for cols in self.index_iter(i)) result += '\n\n' @@ -213,7 +214,7 @@ class test_index01(wttest.WiredTigerTestCase): self.check_exists('smith', 1, 0) self.remove('smith', 1) self.check_exists('smith', 1, wiredtiger.WT_NOTFOUND) - for i in xrange(self.NUM_INDICES): + for i in range(self.NUM_INDICES): self.assertEqual(list(self.index_iter(i)), []) self.drop_table() diff --git a/src/third_party/wiredtiger/test/suite/test_index02.py b/src/third_party/wiredtiger/test/suite/test_index02.py index d92d4d5d59d..86369d9d083 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_index02.py +++ b/src/third_party/wiredtiger/test/suite/test_index02.py @@ -29,6 +29,14 @@ import wiredtiger, wttest from wtscenario import make_scenarios +def cmp(a, b): + if a > b: + return 1 + elif b > a: + return -1 + else: + return 0 + # test_index02.py # test search_near in indices class test_index02(wttest.WiredTigerTestCase): @@ -61,7 +69,7 @@ class test_index02(wttest.WiredTigerTestCase): cur.close() # Retry after reopening - for runs in xrange(2): + for runs in range(2): # search near should find a match cur = self.session.open_cursor(self.indexname, None, None) if self.ncol == 1: @@ -78,14 +86,14 @@ class test_index02(wttest.WiredTigerTestCase): self.session.create(self.tablename, 'key_format=i,value_format=i,columns=(k,v)') self.session.create(self.indexname, self.indexconfig) cur = self.session.open_cursor(self.tablename) - for k in xrange(3): + for k in range(3): cur[k] = 5 * k + 10 cur.close() search_keys = [ 1, 11, 15, 19, 21 ] # search near should find a match - for runs in xrange(2): + for runs in range(2): cur = self.session.open_cursor(self.indexname, None, None) for k in search_keys: if self.ncol == 1: diff --git a/src/third_party/wiredtiger/test/suite/test_index03.py b/src/third_party/wiredtiger/test/suite/test_index03.py index 71312a3db4f..dd6adb75c7e 100644 --- a/src/third_party/wiredtiger/test/suite/test_index03.py +++ b/src/third_party/wiredtiger/test/suite/test_index03.py @@ -59,7 +59,7 @@ class test_index03(wttest.WiredTigerTestCase): c1 = session.open_cursor(uri, None) # Having cursors open across drops is not currently allowed. # On the drop side, we need to begin using the cursor - for i in xrange(100, 200): + for i in range(100, 200): c1[self.key(i)] = self.value(i) self.assertRaises(wiredtiger.WiredTigerError, diff --git a/src/third_party/wiredtiger/test/suite/test_inmem01.py b/src/third_party/wiredtiger/test/suite/test_inmem01.py index 91b3383efce..b86d124cb69 100644 --- a/src/third_party/wiredtiger/test/suite/test_inmem01.py +++ b/src/third_party/wiredtiger/test/suite/test_inmem01.py @@ -102,7 +102,7 @@ class test_inmem01(wttest.WiredTigerTestCase): # Now that the database contains as much data as will fit into # the configured cache, verify removes succeed. cursor = self.session.open_cursor(self.uri, None) - for i in range(1, last_key / 4, 1): + for i in range(1, last_key // 4, 1): cursor.set_key(ds.key(i)) cursor.remove() @@ -125,7 +125,7 @@ class test_inmem01(wttest.WiredTigerTestCase): # Custom "keep filling" helper def fill(self, cursor, ds, start, end): - for i in xrange(start + 1, end + 1): + for i in range(start + 1, end + 1): cursor[ds.key(i)] = ds.value(i) # Keep adding data to the cache until it becomes really full, make sure @@ -163,7 +163,7 @@ class test_inmem01(wttest.WiredTigerTestCase): # many more records into the cache, so don't do as many passes through # the data. checks = 10 if self.valuefmt.endswith('t') else 100 - for run in xrange(checks): + for run in range(checks): ds.check() self.pr('Finished check ' + str(run)) sleep(1) diff --git a/src/third_party/wiredtiger/test/suite/test_inmem02.py b/src/third_party/wiredtiger/test/suite/test_inmem02.py index f3f69a0e7ac..113fe88299b 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_inmem02.py +++ b/src/third_party/wiredtiger/test/suite/test_inmem02.py @@ -44,7 +44,8 @@ class test_inmem02(wttest.WiredTigerTestCase): # Create a new table that is allowed to exceed the cache size, do this # before filling the cache so that the create succeeds self.session.create( - self.uri + '_over', 'ignore_in_memory_cache_size=true') + self.uri + '_over', + 'key_format=S,value_format=S,ignore_in_memory_cache_size=true') # Populate a table with enough data to fill the cache. msg = '/WT_CACHE_FULL.*/' diff --git a/src/third_party/wiredtiger/test/suite/test_join02.py b/src/third_party/wiredtiger/test/suite/test_join02.py index 7fdb0668a64..026f926f5e8 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_join02.py +++ b/src/third_party/wiredtiger/test/suite/test_join02.py @@ -65,7 +65,7 @@ class test_join02(wttest.WiredTigerTestCase): def gen_values(self, i): s = str(i) - x = 'x' * i + x = b'x' * i rs = s[::-1] f = int(s[0:1]) return [i, s, x, rs, f] @@ -225,11 +225,11 @@ class test_join02(wttest.WiredTigerTestCase): c1b.eqmembers = self.mkmbr(lambda x: str(x) == '733') c1b.name = 'c1b' - c2a.low = [ 'x' * 321 ] + c2a.low = [ b'x' * 321 ] c2a.gtmembers = self.mkmbr(lambda x: x > 321) c2a.eqmembers = self.mkmbr(lambda x: x == 321) c2a.name = 'c2a' - c2b.high = [ 'x' * 765 ] + c2b.high = [ b'x' * 765 ] c2b.ltmembers = self.mkmbr(lambda x: x < 765) c2b.eqmembers = self.mkmbr(lambda x: x == 765) c2b.name = 'c2b' diff --git a/src/third_party/wiredtiger/test/suite/test_jsondump02.py b/src/third_party/wiredtiger/test/suite/test_jsondump02.py index 6eea26df6af..b22212a651f 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_jsondump02.py +++ b/src/third_party/wiredtiger/test/suite/test_jsondump02.py @@ -26,10 +26,21 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os +import os, sys import wiredtiger, wttest from suite_subprocess import suite_subprocess +# In Python2, Unicode and strings are different types, +# and need to be converted. In Python3, there is no separate +# unicode type, unicode characters are just embedded as UTF-8 +# in strings. +_python3 = (sys.version_info >= (3, 0, 0)) +def encode(s): + if _python3: + return s + else: + return s.encode('utf-8') + # test_jsondump.py # Test dump output from json cursors. class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): @@ -117,13 +128,12 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): self.set_kv(self.table_uri1, 'KEY001', '\'\"({[]})\"\'\\, etc. allowed') # \u03c0 is pi in Unicode, converted by Python to UTF-8: 0xcf 0x80. # Here's how UTF-8 might be used. - self.set_kv(self.table_uri1, 'KEY002', u'\u03c0'.encode('utf-8')) - # 0xf5-0xff are illegal in Unicode, but may occur legally in C strings. - self.set_kv(self.table_uri1, 'KEY003', '\xff\xfe') + self.set_kv(self.table_uri1, 'KEY002', encode(u'\u03c0')) + self.set_kv(self.table_uri1, 'KEY003', encode(u'\u0abc')) self.set_kv2(self.table_uri2, 'KEY000', 123, 'str0') self.set_kv2(self.table_uri2, 'KEY001', 234, 'str1') - self.set_kv(self.table_uri3, 1, '\x01\x02\x03') - self.set_kv(self.table_uri3, 2, '\x77\x88\x99\x00\xff\xfe') + self.set_kv(self.table_uri3, 1, b'\x01\x02\x03') + self.set_kv(self.table_uri3, 2, b'\x77\x88\x99\x00\x66\x55') self.populate_squarecube(self.table_uri4) table1_json = ( @@ -131,7 +141,7 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): ('"key0" : "KEY001"', '"value0" : ' + '"\'\\\"({[]})\\\"\'\\\\, etc. allowed"'), ('"key0" : "KEY002"', '"value0" : "\\u00cf\\u0080"'), - ('"key0" : "KEY003"', '"value0" : "\\u00ff\\u00fe"')) + ('"key0" : "KEY003"', '"value0" : "\\u00e0\\u00aa\\u00bc"')) self.check_json(self.table_uri1, table1_json) self.session.truncate(self.table_uri1, None, None, None) @@ -156,7 +166,7 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): # bad tokens self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.load_json(self.table_uri2, - (('"abc\u"', ''),)), + (('"abc\\u"', ''),)), '/invalid Unicode/') # bad tokens @@ -222,7 +232,7 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): table3_json = ( ('"key0" : 1', '"value0" : "\\u0001\\u0002\\u0003"'), ('"key0" : 2', - '"value0" : "\\u0077\\u0088\\u0099\\u0000\\u00ff\\u00fe"')) + '"value0" : "\\u0077\\u0088\\u0099\\u0000\\u0066\\u0055"')) self.check_json(self.table_uri3, table3_json) table4_json = ( ('"ikey" : 1,\n"Skey" : "key1"', @@ -323,14 +333,25 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): # i==0 : v:[0x00, 0x01, 0x02] # i==1 : v:[0x01, 0x02, 0x03] # etc. - # A null byte is disallowed in a string value, it is replaced by 'X' + # A null byte or any byte >= 0x7f is disallowed in a string value, + # it is replaced by 'X' def generate_value(self, i, v, isstring): for j in range(0, 3): - val = (i + j) % 256 - if isstring and val == 0: - val = 88 # 'X' + val = i + j + if val >= 256 or (isstring and (val == 0 or val >= 128)): + val = ord('X') v[j] = val + # In Python3, we cannot simply shove random bytes with values >= 0x80 + # into a string, as strings are unicode aware, so we test only up to 0x80. + # Real Unicode strings are tested elsewhere. + def bytes_to_str(self, barray): + mask = 0x7f + result = '' + for b in barray: + result += chr(b & mask) + return result + def test_json_all_bytes(self): """ Test the generated JSON for all byte values in byte array and @@ -343,12 +364,15 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): c6 = self.session.open_cursor(self.table_uri6, None, None) k = bytearray(b'\x00\x00') v = bytearray(b'\x00\x00\x00') - for i in range(0, 512): + for i in range(0, 256): self.generate_key(i, k) self.generate_value(i, v, False) - c5[str(k)] = str(v) + # A 'u' format requires a bytes type with Python3 + c5[bytes(k)] = bytes(v) self.generate_value(i, v, True) # no embedded nuls - c6[str(k)] = str(v) + kstr = self.bytes_to_str(k) + vstr = self.bytes_to_str(v) + c6[kstr] = vstr c5.close() c6.close() @@ -381,10 +405,10 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): table5_json = [] table6_json = [] - for i in range(0, 512): + for i in range(0, 256): self.generate_key(i, k) self.generate_value(i, v, False) - j = i if (i > 0 and i < 254) or (i > 256 and i < 510) else 88 + j = i if (i > 0 and i < 126) else 88 table5_json.append(('"key0" : "' + bin_unicode[k[0]] + bin_unicode[k[1]] + '"', '"value0" : "' + bin_unicode[v[0]] + diff --git a/src/third_party/wiredtiger/test/suite/test_las01.py b/src/third_party/wiredtiger/test/suite/test_las01.py index 91dbab64450..25f34c01952 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_las01.py +++ b/src/third_party/wiredtiger/test/suite/test_las01.py @@ -49,7 +49,7 @@ class test_las01(wttest.WiredTigerTestCase): session.begin_transaction() cursor.set_key(ds.key(nrows + i)) cursor.set_value(value) - self.assertEquals(cursor.update(), 0) + self.assertEqual(cursor.update(), 0) if timestamp == True: session.commit_transaction('commit_timestamp=' + timestamp_str(i + 1)) cursor.close() @@ -82,10 +82,10 @@ class test_las01(wttest.WiredTigerTestCase): cursor = session.open_cursor(uri, None) # Skip the initial rows, which were not updated for i in range(0, nrows+1): - self.assertEquals(cursor.next(), 0) + self.assertEqual(cursor.next(), 0) if (check_value != cursor.get_value()): - print "Check value : " + str(check_value) - print "value : " + str(cursor.get_value()) + print("Check value : " + str(check_value)) + print("value : " + str(cursor.get_value())) self.assertTrue(check_value == cursor.get_value()) cursor.close() session.close() @@ -97,20 +97,20 @@ class test_las01(wttest.WiredTigerTestCase): nrows = 100 ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') ds.populate() - bigvalue = "aaaaa" * 100 + bigvalue = b"aaaaa" * 100 # Initially load huge data cursor = self.session.open_cursor(uri) for i in range(1, 10000): cursor.set_key(ds.key(nrows + i)) cursor.set_value(bigvalue) - self.assertEquals(cursor.insert(), 0) + self.assertEqual(cursor.insert(), 0) cursor.close() self.session.checkpoint() # Scenario: 1 # Check to see LAS working with old snapshot - bigvalue1 = "bbbbb" * 100 + bigvalue1 = b"bbbbb" * 100 self.session.snapshot("name=xxx") # Update the values in different session after snapshot self.large_updates(self.session, uri, bigvalue1, ds, nrows) @@ -120,7 +120,7 @@ class test_las01(wttest.WiredTigerTestCase): # Scenario: 2 # Check to see LAS working with old reader - bigvalue2 = "ccccc" * 100 + bigvalue2 = b"ccccc" * 100 session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') self.large_updates(self.session, uri, bigvalue2, ds, nrows) @@ -131,8 +131,8 @@ class test_las01(wttest.WiredTigerTestCase): # Scenario: 3 # Check to see LAS working with modify operations - bigvalue3 = "ccccc" * 100 - bigvalue3 = 'AA' + bigvalue3[2:] + bigvalue3 = b"ccccc" * 100 + bigvalue3 = b'AA' + bigvalue3[2:] session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') # Apply two modify operations - replacing the first two items with 'A' @@ -147,7 +147,7 @@ class test_las01(wttest.WiredTigerTestCase): # Scenario: 4 # Check to see LAS working with old timestamp - bigvalue4 = "ddddd" * 100 + bigvalue4 = b"ddddd" * 100 self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) self.large_updates(self.session, uri, bigvalue4, ds, nrows, timestamp=True) # Check to see data can be see only till the stable_timestamp diff --git a/src/third_party/wiredtiger/test/suite/test_las02.py b/src/third_party/wiredtiger/test/suite/test_las02.py index d7d54659d51..2ab4d4ec918 100644 --- a/src/third_party/wiredtiger/test/suite/test_las02.py +++ b/src/third_party/wiredtiger/test/suite/test_las02.py @@ -79,17 +79,17 @@ class test_las02(wttest.WiredTigerTestCase): ',stable_timestamp=' + timestamp_str(1)) bigvalue = "aaaaa" * 100 - self.large_updates(uri, bigvalue, ds, nrows / 3, 1) + self.large_updates(uri, bigvalue, ds, nrows // 3, 1) # Check that all updates are seen - self.check(bigvalue, uri, nrows / 3, 1) + self.check(bigvalue, uri, nrows // 3, 1) # Check to see lookaside working with old timestamp bigvalue2 = "ddddd" * 100 self.large_updates(uri, bigvalue2, ds, nrows, 100) # Check that the new updates are only seen after the update timestamp - self.check(bigvalue, uri, nrows / 3, 1) + self.check(bigvalue, uri, nrows // 3, 1) self.check(bigvalue2, uri, nrows, 100) # Force out most of the pages by updating a different tree @@ -98,16 +98,16 @@ class test_las02(wttest.WiredTigerTestCase): # Now truncate half of the records self.session.begin_transaction() end = self.session.open_cursor(uri) - end.set_key(ds.key(nrows / 2)) + end.set_key(ds.key(nrows // 2)) self.session.truncate(None, None, end) end.close() self.session.commit_transaction('commit_timestamp=' + timestamp_str(200)) # Check that the truncate is visible after commit - self.check(bigvalue2, uri, nrows / 2, 200) + self.check(bigvalue2, uri, nrows // 2, 200) # Repeat earlier checks - self.check(bigvalue, uri, nrows / 3, 1) + self.check(bigvalue, uri, nrows // 3, 1) self.check(bigvalue2, uri, nrows, 100) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_las03.py b/src/third_party/wiredtiger/test/suite/test_las03.py index 3c87c3503b8..60ad25cf6d7 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_las03.py +++ b/src/third_party/wiredtiger/test/suite/test_las03.py @@ -63,7 +63,7 @@ class test_las03(wttest.WiredTigerTestCase): nrows = 100 ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') ds.populate() - bigvalue = "aaaaa" * 100 + bigvalue = b"aaaaa" * 100 # Initially load huge data cursor = self.session.open_cursor(uri) @@ -73,7 +73,7 @@ class test_las03(wttest.WiredTigerTestCase): self.session.checkpoint() # Check to see LAS working with old timestamp - bigvalue2 = "ddddd" * 100 + bigvalue2 = b"ddddd" * 100 self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) las_writes_start = self.get_stat(stat.conn.cache_write_lookaside) self.large_updates(self.session, uri, bigvalue2, ds, nrows, 10000) diff --git a/src/third_party/wiredtiger/test/suite/test_lsm02.py b/src/third_party/wiredtiger/test/suite/test_lsm02.py index 344a9c9229e..278f9fa82f0 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_lsm02.py +++ b/src/third_party/wiredtiger/test/suite/test_lsm02.py @@ -43,19 +43,19 @@ class test_lsm02(wttest.WiredTigerTestCase): cursor.set_key(key) cursor.search() if value != cursor.get_value(): - print 'Unexpected value from LSM tree' + print('Unexpected value from LSM tree') cursor.close() # Put some special values that start with the LSM tombstone def test_lsm_tombstone(self): self.session.create(self.uri, 'key_format=S,value_format=u') - v = '\x14\x14' + v = b'\x14\x14' self.add_key(self.uri, 'k1', v) self.verify_key_exists(self.uri, 'k1', v) - v = '\x14\x14\0\0\0\0\0\0' + v = b'\x14\x14\0\0\0\0\0\0' self.add_key(self.uri, 'k2', v) self.verify_key_exists(self.uri, 'k2', v) - v += 'a' * 1000 + v += b'a' * 1000 self.add_key(self.uri, 'k3', v) self.verify_key_exists(self.uri, 'k3', v) diff --git a/src/third_party/wiredtiger/test/suite/test_metadata_cursor01.py b/src/third_party/wiredtiger/test/suite/test_metadata_cursor01.py index e5a6efe3a0b..40e38d884dc 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_metadata_cursor01.py +++ b/src/third_party/wiredtiger/test/suite/test_metadata_cursor01.py @@ -48,7 +48,7 @@ class test_metadata_cursor01(wttest.WiredTigerTestCase): if self.tablekind == 'row': return 'key' + str(i) else: - return long(i+1) + return self.recno(i+1) def genvalue(self, i): if self.tablekind == 'fix': diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap01.py b/src/third_party/wiredtiger/test/suite/test_nsnap01.py index 50f7b8c745b..995b1d2d874 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap01.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap01.py @@ -63,9 +63,9 @@ class test_nsnap01(wttest.WiredTigerTestCase, suite_subprocess): # if there are more than 10 snapshots active, drop the first half snapshots = [] c = self.session.open_cursor(self.uri) - for n in xrange(self.nrows / self.nrows_per_snap): + for n in range(self.nrows // self.nrows_per_snap): if len(snapshots) > self.nsnapshots: - middle = len(snapshots) / 2 + middle = len(snapshots) // 2 dropcfg = ",drop=(to=%d)" % snapshots[middle][0] snapshots = snapshots[middle + 1:] else: @@ -73,10 +73,10 @@ class test_nsnap01(wttest.WiredTigerTestCase, suite_subprocess): self.session.snapshot("name=%d%s" % (n, dropcfg)) snapshots.append((n, end - start)) - for i in xrange(2 * self.nrows_per_snap): + for i in range(2 * self.nrows_per_snap): c[end + i] = "some value" end += 2 * self.nrows_per_snap - for i in xrange(self.nrows_per_snap): + for i in range(self.nrows_per_snap): del c[start + i] start += self.nrows_per_snap diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap02.py b/src/third_party/wiredtiger/test/suite/test_nsnap02.py index e57c26633ca..be979bd9206 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap02.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap02.py @@ -70,13 +70,13 @@ class test_nsnap02(wttest.WiredTigerTestCase, suite_subprocess): # Each snapshot removes a (smaller) bunch of old data snapshots = [] c = self.session.open_cursor(self.uri) - for n in xrange(self.nsnapshots): + for n in range(self.nsnapshots): self.session.snapshot("name=%d" % (n)) snapshots.append((n, end - start, 0)) - for i in xrange(2 * self.nrows_per_snap): + for i in range(2 * self.nrows_per_snap): c[end + i] = "some value" end += 2 * self.nrows_per_snap - for i in xrange(self.nrows_per_snap): + for i in range(self.nrows_per_snap): del c[start + i] start += self.nrows_per_snap return snapshots diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap03.py b/src/third_party/wiredtiger/test/suite/test_nsnap03.py index e69a58f1925..44eac820138 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap03.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap03.py @@ -65,9 +65,9 @@ class test_nsnap03(wttest.WiredTigerTestCase, suite_subprocess): # if there are more than 10 snapshots active, drop the first half snapshots = [] c = self.session.open_cursor(self.uri) - for n in xrange(self.nrows / self.nrows_per_snap): + for n in range(self.nrows // self.nrows_per_snap): if len(snapshots) > self.nsnapshots: - middle = len(snapshots) / 2 + middle = len(snapshots) // 2 dropcfg = ",drop=(to=%d)" % snapshots[middle][0] snapshots = snapshots[middle + 1:] else: @@ -81,10 +81,10 @@ class test_nsnap03(wttest.WiredTigerTestCase, suite_subprocess): self.session.snapshot("name=%d%s" % (n, dropcfg)) snapshots.append((n, end - start)) - for i in xrange(2 * self.nrows_per_snap): + for i in range(2 * self.nrows_per_snap): c[end + i] = "some value" end += 2 * self.nrows_per_snap - for i in xrange(self.nrows_per_snap): + for i in range(self.nrows_per_snap): del c[start + i] start += self.nrows_per_snap diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap04.py b/src/third_party/wiredtiger/test/suite/test_nsnap04.py index eecf0540106..df56c4cffec 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap04.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap04.py @@ -60,7 +60,7 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): snapshots = [] c = self.session.open_cursor(self.uri) - for i in xrange(self.nrows_per_itr): + for i in range(self.nrows_per_itr): c[i] = "some value" # Start a new transaction in a different session @@ -75,7 +75,7 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): self.check_named_snapshot(0, self.nrows_per_itr) # Insert some more content using the original session. - for i in xrange(self.nrows_per_itr): + for i in range(self.nrows_per_itr): c[2 * self.nrows_per_itr + i] = "some value" self.check_named_snapshot(0, self.nrows_per_itr) @@ -91,7 +91,7 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): snapshots = [] c = self.session.open_cursor(self.uri) - for i in xrange(self.nrows_per_itr): + for i in range(self.nrows_per_itr): c[i] = "some value" self.session.begin_transaction("isolation=snapshot") @@ -103,7 +103,7 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): self.check_named_snapshot(0, self.nrows_per_itr) # Insert some more content using the active session. - for i in xrange(self.nrows_per_itr): + for i in range(self.nrows_per_itr): c[self.nrows_per_itr + i] = "some value" self.check_named_snapshot(0, 2 * self.nrows_per_itr) diff --git a/src/third_party/wiredtiger/test/suite/test_pack.py b/src/third_party/wiredtiger/test/suite/test_pack.py index fd0231c4a74..83893ca97af 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_pack.py +++ b/src/third_party/wiredtiger/test/suite/test_pack.py @@ -50,7 +50,7 @@ class test_pack(wttest.WiredTigerTestCase): uri = 'table:' + test_pack.name + '-' + fmtname idx_uri = 'index:' + test_pack.name + '-' + fmtname + ':inverse' nargs = len(v) - colnames = ",".join("v" + str(x) for x in xrange(nargs)) + colnames = ",".join("v" + str(x) for x in range(nargs)) self.session.create(uri, "columns=(k," + colnames + ")," + "key_format=i,value_format=" + fmt) self.session.create(idx_uri, "columns=(" + colnames + ")") @@ -92,16 +92,16 @@ class test_pack(wttest.WiredTigerTestCase): self.check('10SS', 'aaaaa\x00\x00\x00\x00\x00', 'something') self.check('S10S', 'something', 'aaaaa\x00\x00\x00\x00\x00') - self.check('u', r"\x42" * 20) - self.check('uu', r"\x42" * 10, r"\x42" * 10) - self.check('3u', r"\x4") - self.check('3uu', r"\x4", r"\x42" * 10) - self.check('u3u', r"\x42" * 10, r"\x4") - self.check('u', '\x00') - self.check('u', '') - self.check('uu', '', '\x00') - self.check('uu', '\x00', '') - self.check('uu', '', '') + self.check('u', b"\x42" * 20) + self.check('uu', b"\x42" * 10, b"\x42" * 10) + self.check('3u', b"\x04\x03\x02") + self.check('3uu', b"\x04\x03\x02", b"\x42" * 10) + self.check('u3u', b"\x42" * 10, b"\x04\x03\x02") + self.check('u', b'\x00') + self.check('u', b'') + self.check('uu', b'', b'\x00') + self.check('uu', b'\x00', b'') + self.check('uu', b'', b'') self.check('s', "4") self.check("1s", "4") diff --git a/src/third_party/wiredtiger/test/suite/test_prepare01.py b/src/third_party/wiredtiger/test/suite/test_prepare01.py index c23d0a869b2..596ceb6fdf8 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare01.py @@ -112,8 +112,8 @@ class test_prepare01(wttest.WiredTigerTestCase): self.check(cursor, 0, 0) self.session.begin_transaction("ignore_prepare=false") - for i in xrange(self.nentries): - if i > 0 and i % (self.nentries / 37) == 0: + for i in range(self.nentries): + if i > 0 and i % (self.nentries // 37) == 0: self.check(cursor, committed, i) self.session.prepare_transaction("prepare_timestamp=2a") self.session.timestamp_transaction("commit_timestamp=3a") diff --git a/src/third_party/wiredtiger/test/suite/test_prepare03.py b/src/third_party/wiredtiger/test/suite/test_prepare03.py index 512efe0c008..9abfa7b2957 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_prepare03.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare03.py @@ -56,7 +56,7 @@ class test_prepare03(wttest.WiredTigerTestCase): if self.tablekind == 'row': return 'key' + str(i) else: - return long(i+1) + return self.recno(i+1) def genvalue(self, i): if self.tablekind == 'fix': @@ -161,7 +161,7 @@ class test_prepare03(wttest.WiredTigerTestCase): # Search for a specific key. # Verify we get the expected error and then later we can update and # remove it. - cursor.set_key(self.genkey(self.nentries/2)) + cursor.set_key(self.genkey(self.nentries//2)) self.session.begin_transaction() self.session.prepare_transaction("prepare_timestamp=2a") self.assertRaisesWithMessage(wiredtiger.WiredTigerError, @@ -178,7 +178,7 @@ class test_prepare03(wttest.WiredTigerTestCase): self.session.timestamp_transaction("durable_timestamp=2b") self.session.commit_transaction() cursor.search() - cursor.set_value(self.genvalue(self.nentries + self.nentries/2)) + cursor.set_value(self.genvalue(self.nentries + self.nentries//2)) cursor.update() cursor.remove() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare04.py b/src/third_party/wiredtiger/test/suite/test_prepare04.py index 4193c3299b7..9d3d78804c0 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare04.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare04.py @@ -72,7 +72,7 @@ class test_prepare04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(self.uri) # Insert keys 1..100 each with timestamp=key, in some order - orig_keys = range(1, 101) + orig_keys = list(range(1, 101)) keys = orig_keys[:] random.shuffle(keys) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_lookaside01.py b/src/third_party/wiredtiger/test/suite/test_prepare_lookaside01.py index 9b17aa8ac9b..acfbefb7312 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_prepare_lookaside01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_lookaside01.py @@ -58,7 +58,7 @@ class test_prepare_lookaside01(wttest.WiredTigerTestCase): self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) # Commit some updates to get eviction and lookaside fired up - bigvalue1 = "bbbbb" * 100 + bigvalue1 = b"bbbbb" * 100 cursor = self.session.open_cursor(uri) for i in range(1, nsessions * nkeys): self.session.begin_transaction() @@ -71,7 +71,7 @@ class test_prepare_lookaside01(wttest.WiredTigerTestCase): # prepared updates to the lookaside sessions = [0] * nsessions cursors = [0] * nsessions - bigvalue2 = "ccccc" * 100 + bigvalue2 = b"ccccc" * 100 for j in range (0, nsessions): sessions[j] = self.conn.open_session() sessions[j].begin_transaction("isolation=snapshot") @@ -88,7 +88,7 @@ class test_prepare_lookaside01(wttest.WiredTigerTestCase): # Commit more regular updates. To do this, the pages that were just # evicted need to be read back. This ensures reading prepared updates # from the lookaside - bigvalue3 = "ddddd" * 100 + bigvalue3 = b"ddddd" * 100 cursor = self.session.open_cursor(uri) for i in range(1, nsessions * nkeys): self.session.begin_transaction() @@ -110,7 +110,7 @@ class test_prepare_lookaside01(wttest.WiredTigerTestCase): nrows = 100 ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') ds.populate() - bigvalue = "aaaaa" * 100 + bigvalue = b"aaaaa" * 100 # Initially load huge data cursor = self.session.open_cursor(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_lookaside02.py b/src/third_party/wiredtiger/test/suite/test_prepare_lookaside02.py index 01c59885874..daeeff28fef 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_lookaside02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_lookaside02.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. # # test_prepare_lookaside02.py -# Prepare updates can be resolved for both commit / rollback operations. +# Prepare updates can be resolved for both commit // rollback operations. # from helper import copy_wiredtiger_home @@ -63,7 +63,7 @@ class test_prepare_lookaside02(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(self.uri) # Insert keys 1..100 each with timestamp=key, in some order - orig_keys = range(1, 101) + orig_keys = list(range(1, 101)) keys = orig_keys[:] random.shuffle(keys) diff --git a/src/third_party/wiredtiger/test/suite/test_readonly01.py b/src/third_party/wiredtiger/test/suite/test_readonly01.py index 21e4c735374..062b0698048 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_readonly01.py +++ b/src/third_party/wiredtiger/test/suite/test_readonly01.py @@ -100,8 +100,8 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): if self.dirchmod and os.name == 'posix': for f in os.listdir(self.home): if os.path.isfile(f): - os.chmod(f, 0444) - os.chmod(self.home, 0555) + os.chmod(f, 0o444) + os.chmod(self.home, 0o555) self.conn = self.setUpConnectionOpen(self.home) self.session = self.setUpSessionOpen(self.conn) diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig02.py b/src/third_party/wiredtiger/test/suite/test_reconfig02.py index 7d5a88c0d41..68e9704b879 100644 --- a/src/third_party/wiredtiger/test/suite/test_reconfig02.py +++ b/src/third_party/wiredtiger/test/suite/test_reconfig02.py @@ -81,7 +81,7 @@ class test_reconfig02(wttest.WiredTigerTestCase): # # Potentially loop a few times in case it is a very slow system. self.conn.reconfigure("log=(prealloc=true)") - for x in xrange(0, 100): + for x in range(0, 100): time.sleep(1) prep_logs = fnmatch.filter(os.listdir('.'), "*Prep*") if len(prep_logs) != 0: diff --git a/src/third_party/wiredtiger/test/suite/test_salvage.py b/src/third_party/wiredtiger/test/suite/test_salvage.py index ee9a3cde687..0ccff73373b 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_salvage.py +++ b/src/third_party/wiredtiger/test/suite/test_salvage.py @@ -46,7 +46,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): key = '' for i in range(0, self.nentries): key += str(i) - if i == self.nentries / 2: + if i == self.nentries // 2: val = self.unique + '0' else: val = key + key @@ -62,7 +62,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): i = 0 for gotkey, gotval in cursor: wantkey += str(i) - if i == self.nentries / 2: + if i == self.nentries // 2: wantval = self.unique + '0' else: wantval = wantkey + wantkey @@ -87,7 +87,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): wantkey += str(i) if gotkey != wantkey: continue - if i == self.nentries / 2: + if i == self.nentries // 2: wantval = self.unique + '0' else: wantval = wantkey + wantkey @@ -106,7 +106,16 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): cursor.close() def damage(self, tablename): - self.damage_inner(tablename, self.unique) + self.damage_inner(tablename, self.unique.encode()) + + def read_byte(self, fp): + """ + Return a single byte from a file opened in binary mode. + """ + c = fp.read(1) + if self.is_python3(): + c = c[0] # In python3, the read returns bytes (an array). + return c def damage_inner(self, tablename, unique): """ @@ -114,6 +123,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): and modify it. """ self.close_conn() + self.assertTrue(type(unique) == bytes) # we close the connection to guarantee everything is # flushed and closed from the WT point of view. filename = tablename + ".wt" @@ -123,19 +133,19 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): match = unique matchlen = len(match) flen = os.fstat(fp.fileno()).st_size - c = fp.read(1) + c = self.read_byte(fp) while fp.tell() != flen: if match[matchpos] == c: matchpos += 1 if matchpos == matchlen: # We're already positioned, so alter it fp.seek(-1, 1) - fp.write('G') + fp.write(b'G') matchpos = 0 found = 1 else: matchpos = 0 - c = fp.read(1) + c = self.read_byte(fp) # Make sure we found the embedded string self.assertTrue(found) fp.close() diff --git a/src/third_party/wiredtiger/test/suite/test_schema02.py b/src/third_party/wiredtiger/test/suite/test_schema02.py index 85091913a6e..87f64336fe0 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_schema02.py +++ b/src/third_party/wiredtiger/test/suite/test_schema02.py @@ -182,7 +182,7 @@ class test_schema02(wttest.WiredTigerTestCase): cursor = self.session.open_cursor('table:main', None, None) # spot check via search n = self.nentries - for i in (n / 5, 0, n - 1, n - 2, 1): + for i in (n // 5, 0, n - 1, n - 2, 1): cursor.set_key(i, 'key' + str(i)) square = i * i cube = square * i diff --git a/src/third_party/wiredtiger/test/suite/test_schema03.py b/src/third_party/wiredtiger/test/suite/test_schema03.py index 82b106ea5ae..76d709f2eae 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_schema03.py +++ b/src/third_party/wiredtiger/test/suite/test_schema03.py @@ -79,7 +79,7 @@ class tabconfig: elif format == 'i': keys.append(rev) elif format == 'r': - keys.append(long(i+1)) + keys.append(self.recno(i+1)) return keys def gen_values(self, i): @@ -315,7 +315,7 @@ class test_schema03(wttest.WiredTigerTestCase): if self.SHOW_PYTHON: if self.SHOW_PYTHON_ONLY_TABLE == None or self.current_table in self.SHOW_PYTHON_ONLY_TABLE: if self.SHOW_PYTHON_ONLY_SCEN == None or self.scenario_number in self.SHOW_PYTHON_ONLY_SCEN: - print ' ' + s + print(' ' + s) def join_names(self, sep, prefix, list): return sep.join([prefix + str(val) for val in list]) @@ -330,14 +330,14 @@ class test_schema03(wttest.WiredTigerTestCase): def finished_step(self, name): if self.s_restart == name: - print " # Reopening connection at step: " + name + print(" # Reopening connection at step: " + name) self.reopen_conn() def test_schema(self): rand = suite_random.suite_random() if self.SHOW_PYTHON: - print ' ################################################' - print ' # Running scenario ' + str(self.scenario_number) + print(' ################################################') + print(' # Running scenario ' + str(self.scenario_number)) ntables = self.s_ntable @@ -431,8 +431,8 @@ class test_schema03(wttest.WiredTigerTestCase): self.show_python("self.session.create('table:" + tc.tablename + "', '" + config + "')") self.session.create("table:" + tc.tablename, config) - tc.columns_for_groups(range(tc.nkeys, tc.nkeys + tc.nvalues)) - tc.columns_for_indices(range(0, tc.nkeys + tc.nvalues)) + tc.columns_for_groups(list(range(tc.nkeys, tc.nkeys + tc.nvalues))) + tc.columns_for_indices(list(range(0, tc.nkeys + tc.nvalues))) self.finished_step('table') @@ -461,7 +461,7 @@ class test_schema03(wttest.WiredTigerTestCase): for tc in tabconfigs: self.current_table = tc.tableidx max = rand.rand_range(0, self.nentries) - self.populate(tc, xrange(0, max)) + self.populate(tc, list(range(0, max))) self.finished_step('populate0') @@ -476,7 +476,7 @@ class test_schema03(wttest.WiredTigerTestCase): # populate second batch for tc in tabconfigs: self.current_table = tc.tableidx - self.populate(tc, xrange(tc.nentries, self.nentries)) + self.populate(tc, list(range(tc.nentries, self.nentries))) self.finished_step('populate1') diff --git a/src/third_party/wiredtiger/test/suite/test_schema04.py b/src/third_party/wiredtiger/test/suite/test_schema04.py index a69846f7558..157c89802e1 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema04.py +++ b/src/third_party/wiredtiger/test/suite/test_schema04.py @@ -66,9 +66,9 @@ class test_schema04(wttest.WiredTigerTestCase): cursor = self.session.open_cursor('table:schema04', None, None) if phase == 0: range_from = 0 - range_to = self.nentries / 2 + range_to = self.nentries // 2 else: - range_from = self.nentries / 2 + range_from = self.nentries // 2 range_to = self.nentries for i in range(range_from, range_to): diff --git a/src/third_party/wiredtiger/test/suite/test_schema05.py b/src/third_party/wiredtiger/test/suite/test_schema05.py index 5a9be65e315..9a5e5d49c31 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema05.py +++ b/src/third_party/wiredtiger/test/suite/test_schema05.py @@ -84,9 +84,9 @@ class test_schema05(wttest.WiredTigerTestCase): cursor = self.session.open_cursor('table:schema05', None, None) if phase == 0: range_from = 0 - range_to = self.nentries / 2 + range_to = self.nentries // 2 elif phase == 1: - range_from = self.nentries / 2 + range_from = self.nentries // 2 range_to = self.nentries - 5 else: range_from = self.nentries - 5 diff --git a/src/third_party/wiredtiger/test/suite/test_schema06.py b/src/third_party/wiredtiger/test/suite/test_schema06.py index 818525fafd4..f516175bffc 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema06.py +++ b/src/third_party/wiredtiger/test/suite/test_schema06.py @@ -91,7 +91,7 @@ class test_schema06(wttest.WiredTigerTestCase): cursor = self.session.open_cursor('table:main', None, None) # spot check via search n = self.nentries - for i in (n / 5, 0, n - 1, n - 2, 1): + for i in (n // 5, 0, n - 1, n - 2, 1): cursor.set_key(i, 'key' + str(i)) square = i * i cube = square * i @@ -124,7 +124,7 @@ class test_schema06(wttest.WiredTigerTestCase): cursor = self.session.open_cursor('index:main:S1i4', None, None) count = 0 for s1key, i4key, s1, i2, s3, i4 in cursor: - i = int(i4key ** (1 / 3.0) + 0.0001) # cuberoot + i = int(i4key ** (1 // 3.0) + 0.0001) # cuberoot self.assertEqual(s1key, s1) self.assertEqual(i4key, i4) ikey = i @@ -145,7 +145,7 @@ class test_schema06(wttest.WiredTigerTestCase): cursor = self.session.open_cursor('index:main:i2S1i4', None, None) count = 0 for i2key, s1key, i4key, s1, i2, s3, i4 in cursor: - i = int(i4key ** (1 / 3.0) + 0.0001) # cuberoot + i = int(i4key ** (1 // 3.0) + 0.0001) # cuberoot self.assertEqual(i2key, i2) self.assertEqual(s1key, s1) self.assertEqual(i4key, i4) diff --git a/src/third_party/wiredtiger/test/suite/test_schema07.py b/src/third_party/wiredtiger/test/suite/test_schema07.py index 1c1755439f3..b35cca1611a 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema07.py +++ b/src/third_party/wiredtiger/test/suite/test_schema07.py @@ -40,7 +40,7 @@ class test_schema07(wttest.WiredTigerTestCase): s = self.session # We have a 10MB cache, metadata is (well) over 512B per table, # if we can create 20K tables, something must be cleaning up. - for i in xrange(20000): + for i in range(20000): uri = '%s-%06d' % (self.tablename, i) s.create(uri) c = s.open_cursor(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_split.py b/src/third_party/wiredtiger/test/suite/test_split.py index cde840fb055..233ae2133bc 100644 --- a/src/third_party/wiredtiger/test/suite/test_split.py +++ b/src/third_party/wiredtiger/test/suite/test_split.py @@ -49,7 +49,7 @@ class test_split(wttest.WiredTigerTestCase): # IF IT FAILS, IT MAY BE RECONCILIATION ISN'T CREATING THE SAME SIZE # PAGES AS BEFORE. - # Create a 4KB page (more than 3KB): 40 records w / 10 byte keys + # Create a 4KB page (more than 3KB): 40 records w // 10 byte keys # and 81 byte values. for i in range(35): cursor['%09d' % i] = 8 * ('%010d' % i) diff --git a/src/third_party/wiredtiger/test/suite/test_stat01.py b/src/third_party/wiredtiger/test/suite/test_stat01.py index c815b99d6bd..ea4e0b04f48 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_stat01.py +++ b/src/third_party/wiredtiger/test/suite/test_stat01.py @@ -107,7 +107,8 @@ class test_stat01(wttest.WiredTigerTestCase): # Test simple object statistics. def test_basic_data_source_stats(self): # Build an object. - config = self.config + ',key_format=' + self.keyfmt + config = self.config + ',key_format=' + self.keyfmt + \ + ',value_format=S' self.session.create(self.uri, config) cursor = self.session.open_cursor(self.uri, None, None) value = "" diff --git a/src/third_party/wiredtiger/test/suite/test_stat04.py b/src/third_party/wiredtiger/test/suite/test_stat04.py index 153ed983308..1717f70ecc7 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat04.py +++ b/src/third_party/wiredtiger/test/suite/test_stat04.py @@ -93,7 +93,7 @@ class test_stat04(wttest.WiredTigerTestCase, suite_subprocess): count += 1 # Remove a number of entries, at each step checking that stats match. - for i in range(0, self.nentries / 37): + for i in range(0, self.nentries // 37): cursor.set_key(self.genkey(i*11 % self.nentries)) if cursor.remove() == 0: count -= 1 diff --git a/src/third_party/wiredtiger/test/suite/test_stat09.py b/src/third_party/wiredtiger/test/suite/test_stat09.py new file mode 100644 index 00000000000..8198ebd780d --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_stat09.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import random +import wiredtiger, wttest + +# test_stat09.py +# Check oldest active read timestamp statistic + +def timestamp_str(t): + return '%x' % t + +class test_stat09(wttest.WiredTigerTestCase): + tablename = 'test_stat09' + uri = 'table:' + tablename + conn_config = 'statistics=(all)' + + # Check the oldest active read statistic to be at the expected values + def check_stat_oldest_read(self, statcursor, expected_oldest, all_committed): + self.check_stats(statcursor, expected_oldest, + 'transaction: transaction read timestamp of the oldest active reader') + + # If the active oldest timestamp is 0, it implies there are no active readers, + # the pinned range because of them is expected to be 0 in that case + if expected_oldest == 0: + expected_pinned = 0 + else: + expected_pinned = all_committed - expected_oldest + self.check_stats(statcursor, expected_pinned, + 'transaction: transaction range of timestamps pinned by the oldest ' + 'active read timestamp') + + # Do a quick check of the entries in the stats cursor, the "lookfor" + # string should appear with the exact val of "expected_val". + def check_stats(self, statcursor, expected_val, lookfor): + # Reset the cursor, we're called multiple times. + statcursor.reset() + + found = False + foundval = 0 + for id, desc, valstr, val in statcursor: + if desc == lookfor: + found = True + foundval = val + self.printVerbose(2, ' stat: \'' + desc + '\', \'' + + valstr + '\', ' + str(val)) + break + + self.assertTrue(found, 'in stats, did not see: ' + lookfor) + self.assertTrue(foundval == expected_val) + + def test_oldest_active_read(self): + self.session.create(self.uri, 'key_format=i,value_format=i') + c = self.session.open_cursor(self.uri) + + # Insert some data: keys 1..100 each with timestamp=key, in some order + commit_range = 100 + orig_keys = list(range(1, commit_range + 1)) + keys = orig_keys[:] + random.shuffle(keys) + + for k in keys: + self.session.begin_transaction() + c[k] = 1 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + + # Create a cursor on statistics that we can use repeatedly + allstat_cursor = self.session.open_cursor('statistics:', None, None) + + # There being no active reader, the corresponding statistic should be 0 + self.check_stat_oldest_read(allstat_cursor, 0, commit_range) + + # Introduce multiple transactions with varying read_timestamp + s1 = self.conn.open_session() + s1.begin_transaction('read_timestamp=' + timestamp_str(10)) + s2 = self.conn.open_session() + s2.begin_transaction('read_timestamp=' + timestamp_str(20)) + s3 = self.conn.open_session() + s3.begin_transaction('read_timestamp=' + timestamp_str(30)) + s4 = self.conn.open_session() + s4.begin_transaction('read_timestamp=' + timestamp_str(40)) + s5 = self.conn.open_session() + s5.begin_transaction('read_timestamp=' + timestamp_str(50)) + + # Check oldest reader + self.check_stat_oldest_read(allstat_cursor, 10, commit_range) + + # Close the oldest reader and check again + s1.commit_transaction() + self.check_stat_oldest_read(allstat_cursor, 20, commit_range) + + # Set and advance the oldest timestamp, it should be ignored for + # determining the oldest active read. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(5)) + self.check_stat_oldest_read(allstat_cursor, 20, commit_range) + + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(30)) + self.check_stat_oldest_read(allstat_cursor, 20, commit_range) + + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(150)) + self.check_stat_oldest_read(allstat_cursor, 20, commit_range) + + # Move the commit timestamp and check again + commit_range = 200 + s2.commit_transaction('commit_timestamp=' + timestamp_str(commit_range)) + self.check_stat_oldest_read(allstat_cursor, 30, commit_range) + + # Close all the readers and check the oldest reader, it should be back to 0 + s3.commit_transaction() + s4.commit_transaction() + s5.commit_transaction() + self.check_stat_oldest_read(allstat_cursor, 0, commit_range) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_sweep01.py b/src/third_party/wiredtiger/test/suite/test_sweep01.py index 976a671081d..026c89b4dc9 100644 --- a/src/third_party/wiredtiger/test/suite/test_sweep01.py +++ b/src/third_party/wiredtiger/test/suite/test_sweep01.py @@ -153,55 +153,55 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): # in the presence of recent checkpoints. # if (close1 >= close2): - print "XX: close1: " + str(close1) + " close2: " + str(close2) - print "remove1: " + str(remove1) + " remove2: " + str(remove2) - print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2) - print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2) - print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2) - print "tod1: " + str(tod1) + " tod2: " + str(tod2) - print "ref1: " + str(ref1) + " ref2: " + str(ref2) - print "nfile1: " + str(nfile1) + " nfile2: " + str(nfile2) + print("XX: close1: " + str(close1) + " close2: " + str(close2)) + print("remove1: " + str(remove1) + " remove2: " + str(remove2)) + print("sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)) + print("sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)) + print("ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)) + print("tod1: " + str(tod1) + " tod2: " + str(tod2)) + print("ref1: " + str(ref1) + " ref2: " + str(ref2)) + print("nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)) self.assertEqual(close1 < close2, True) if (remove1 >= remove2): - print "close1: " + str(close1) + " close2: " + str(close2) - print "XX: remove1: " + str(remove1) + " remove2: " + str(remove2) - print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2) - print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2) - print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2) - print "tod1: " + str(tod1) + " tod2: " + str(tod2) - print "ref1: " + str(ref1) + " ref2: " + str(ref2) - print "nfile1: " + str(nfile1) + " nfile2: " + str(nfile2) + print("close1: " + str(close1) + " close2: " + str(close2)) + print("XX: remove1: " + str(remove1) + " remove2: " + str(remove2)) + print("sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)) + print("sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)) + print("ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)) + print("tod1: " + str(tod1) + " tod2: " + str(tod2)) + print("ref1: " + str(ref1) + " ref2: " + str(ref2)) + print("nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)) self.assertEqual(remove1 < remove2, True) if (sweep1 >= sweep2): - print "close1: " + str(close1) + " close2: " + str(close2) - print "remove1: " + str(remove1) + " remove2: " + str(remove2) - print "XX: sweep1: " + str(sweep1) + " sweep2: " + str(sweep2) - print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2) - print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2) - print "tod1: " + str(tod1) + " tod2: " + str(tod2) - print "ref1: " + str(ref1) + " ref2: " + str(ref2) + print("close1: " + str(close1) + " close2: " + str(close2)) + print("remove1: " + str(remove1) + " remove2: " + str(remove2)) + print("XX: sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)) + print("sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)) + print("ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)) + print("tod1: " + str(tod1) + " tod2: " + str(tod2)) + print("ref1: " + str(ref1) + " ref2: " + str(ref2)) self.assertEqual(sweep1 < sweep2, True) if (nfile2 >= nfile1): - print "close1: " + str(close1) + " close2: " + str(close2) - print "remove1: " + str(remove1) + " remove2: " + str(remove2) - print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2) - print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2) - print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2) - print "tod1: " + str(tod1) + " tod2: " + str(tod2) - print "ref1: " + str(ref1) + " ref2: " + str(ref2) - print "XX: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2) + print("close1: " + str(close1) + " close2: " + str(close2)) + print("remove1: " + str(remove1) + " remove2: " + str(remove2)) + print("sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)) + print("sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)) + print("ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)) + print("tod1: " + str(tod1) + " tod2: " + str(tod2)) + print("ref1: " + str(ref1) + " ref2: " + str(ref2)) + print("XX: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)) self.assertEqual(nfile2 < nfile1, True) # The only files that should be left are the metadata, the lookaside # file, the lock file, and the active file. if (nfile2 != final_nfile): - print "close1: " + str(close1) + " close2: " + str(close2) - print "remove1: " + str(remove1) + " remove2: " + str(remove2) - print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2) - print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2) - print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2) - print "tod1: " + str(tod1) + " tod2: " + str(tod2) - print "ref1: " + str(ref1) + " ref2: " + str(ref2) - print "XX2: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2) + print("close1: " + str(close1) + " close2: " + str(close2)) + print("remove1: " + str(remove1) + " remove2: " + str(remove2)) + print("sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)) + print("sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)) + print("ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)) + print("tod1: " + str(tod1) + " tod2: " + str(tod2)) + print("ref1: " + str(ref1) + " ref2: " + str(ref2)) + print("XX2: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)) self.assertEqual(nfile2 == final_nfile, True) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp02.py b/src/third_party/wiredtiger/test/suite/test_timestamp02.py index 3918f8cdeeb..c8a60b6f11f 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp02.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp02.py @@ -59,7 +59,7 @@ class test_timestamp02(wttest.WiredTigerTestCase, suite_subprocess): actual = dict((k, v) for k, v in c if v != 0) self.assertTrue(actual == expected) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(c[k], v, "for key " + str(k)) c.close() if txn_config: @@ -71,7 +71,7 @@ class test_timestamp02(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(self.uri) # Insert keys 1..100 each with timestamp=key, in some order - orig_keys = range(1, 101) + orig_keys = list(range(1, 101)) keys = orig_keys[:] random.shuffle(keys) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp03.py b/src/third_party/wiredtiger/test/suite/test_timestamp03.py index 029e3292aac..4dc1117bb2e 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp03.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp03.py @@ -83,7 +83,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): actual = dict((k, v) for k, v in cur if v != 0) self.assertTrue(actual == expected) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(cur[k], v, "for key " + str(k)) cur.close() if txn_config: @@ -176,7 +176,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # Insert keys 1..100 each with timestamp=key, in some order nkeys = 100 - orig_keys = range(1, nkeys+1) + orig_keys = list(range(1, nkeys+1)) keys = orig_keys[:] random.shuffle(keys) @@ -267,20 +267,21 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 4a # This scenario is same as earlier one with read_timestamp earlier than - # oldest_timestamp and using the option of round_to_oldest + # oldest_timestamp and using the option of rounding read_timestamp to + # the oldest_timestamp earlier_ts = timestamp_str(90) self.check(self.session, - 'read_timestamp=' + earlier_ts +',round_to_oldest=true', + 'read_timestamp=' + earlier_ts +',roundup_timestamps=(read=true)', self.table_ts_log, dict((k, self.value) for k in orig_keys)) self.check(self.session, - 'read_timestamp=' + earlier_ts +',round_to_oldest=true', + 'read_timestamp=' + earlier_ts +',roundup_timestamps=(read=true)', self.table_ts_nolog, dict((k, self.value) for k in orig_keys)) # Tables not using the timestamps should see updated values (i.e. value2). self.check(self.session, - 'read_timestamp=' + earlier_ts +',round_to_oldest=true', + 'read_timestamp=' + earlier_ts +',roundup_timestamps=(read=true)', self.table_nots_log, dict((k, self.value2) for k in orig_keys)) self.check(self.session, - 'read_timestamp=' + earlier_ts +',round_to_oldest=true', + 'read_timestamp=' + earlier_ts +',roundup_timestamps=(read=true)', self.table_nots_nolog, dict((k, self.value2) for k in orig_keys)) # Scenario: 5 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp04.py b/src/third_party/wiredtiger/test/suite/test_timestamp04.py index 9251bfab06e..cf692d43edd 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp04.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp04.py @@ -70,12 +70,12 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): if missing == False: actual = dict((k, v) for k, v in cur if v != 0) if actual != expected: - print "missing: ", sorted(set(expected.items()) - set(actual.items())) - print "extras: ", sorted(set(actual.items()) - set(expected.items())) + print("missing: ", sorted(set(expected.items()) - set(actual.items()))) + print("extras: ", sorted(set(actual.items()) - set(expected.items()))) self.assertTrue(actual == expected) # Search for the expected items as well as iterating. - for k, v in expected.iteritems(): + for k, v in expected.items(): if missing == False: self.assertEqual(cur[k], v, "for key " + str(k)) else: @@ -104,7 +104,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): try: self.conn = wiredtiger.wiredtiger_open(self.home, conn_params) except wiredtiger.WiredTigerError as e: - print "Failed conn at '%s' with config '%s'" % (dir, conn_params) + print("Failed conn at '%s' with config '%s'" % (dir, conn_params)) self.session = self.conn.open_session(None) def test_rollback_to_stable(self): @@ -131,7 +131,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Insert keys each with timestamp=key, in some order. key_range = 10000 - keys = range(1, key_range + 1) + keys = list(range(1, key_range + 1)) # Set keys 1-key_range to value 1. for k in keys: @@ -159,7 +159,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 2 # Roll back half timestamps. - stable_ts = timestamp_str(key_range / 2) + stable_ts = timestamp_str(key_range // 2) self.conn.set_timestamp('stable_timestamp=' + stable_ts) self.conn.rollback_to_stable() stat_cursor = self.session.open_cursor('statistics:', None, None) @@ -182,9 +182,9 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Check that we see the inserted value (i.e. 1) for the keys in a # timestamped table until the stable_timestamp only. self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_nolog, dict((k, 1) for k in keys[:(key_range / 2)])) + self.table_ts_nolog, dict((k, 1) for k in keys[:(key_range // 2)])) self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_nolog, dict((k, 1) for k in keys[(key_range / 2 + 1):]), missing=True) + self.table_ts_nolog, dict((k, 1) for k in keys[(key_range // 2 + 1):]), missing=True) # For logged tables, the behavior of rollback_to_stable changes based on # whether connection level logging is enabled or not. @@ -198,9 +198,9 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Check that we see the insertions are rolled back in timestamped tables # until the stable_timestamp. self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_log, dict((k, 1) for k in keys[:(key_range / 2)])) + self.table_ts_log, dict((k, 1) for k in keys[:(key_range // 2)])) self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_log, dict((k, 1) for k in keys[(key_range / 2 + 1):]), missing=True) + self.table_ts_log, dict((k, 1) for k in keys[(key_range // 2 + 1):]), missing=True) # Bump the oldest timestamp, we're not going back. self.conn.set_timestamp('oldest_timestamp=' + stable_ts) @@ -229,7 +229,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 4 # Advance the stable_timestamp by a quarter range and rollback. # Three-fourths of the later timestamps will be rolled back. - rolled_range = key_range + key_range / 4 + rolled_range = key_range + key_range // 4 stable_ts = timestamp_str(rolled_range) self.conn.set_timestamp('stable_timestamp=' + stable_ts) self.conn.rollback_to_stable() @@ -257,10 +257,10 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # the updated value (i.e. 2) for the first quarter keys and old values # (i.e. 1) for the second quarter keys. self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_nolog, dict((k, 2 if k <= key_range / 4 else 1) - for k in keys[:(key_range / 2)])) + self.table_ts_nolog, dict((k, 2 if k <= key_range // 4 else 1) + for k in keys[:(key_range // 2)])) self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_nolog, dict((k, 1) for k in keys[(1 + key_range / 2):]), missing=True) + self.table_ts_nolog, dict((k, 1) for k in keys[(1 + key_range // 2):]), missing=True) # For logged tables behavior changes for rollback_to_stable based on # whether connection level logging is enabled or not. @@ -275,10 +275,10 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # the updated value (i.e. 2) for the first quarter keys and old values # (i.e. 1) for the second quarter keys. self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_log, dict((k, (2 if k <= key_range / 4 else 1)) - for k in keys[:(key_range / 2)])) + self.table_ts_log, dict((k, (2 if k <= key_range // 4 else 1)) + for k in keys[:(key_range // 2)])) self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_log, dict((k, 1) for k in keys[(1 + key_range / 2):]), missing=True) + self.table_ts_log, dict((k, 1) for k in keys[(1 + key_range // 2):]), missing=True) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp05.py b/src/third_party/wiredtiger/test/suite/test_timestamp05.py index 5b63b54597b..6ef6a92851e 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp05.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp05.py @@ -69,9 +69,7 @@ class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): # Insert keys 1..100 each with timestamp=key, in some order nkeys = 100 - keys = range(1, nkeys+1) - - for k in keys: + for k in range(1, nkeys+1): c[k] = 'some value' # Start timestamps at 50 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp06.py b/src/third_party/wiredtiger/test/suite/test_timestamp06.py index 95e3dbce427..a085f0f0d27 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp06.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp06.py @@ -72,11 +72,11 @@ class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): cur = session.open_cursor(tablename, None) actual = dict((k, v) for k, v in cur if v != 0) if actual != expected: - print "missing: ", sorted(set(expected.items()) - set(actual.items())) - print "extras: ", sorted(set(actual.items()) - set(expected.items())) + print("missing: ", sorted(set(expected.items()) - set(actual.items()))) + print("extras: ", sorted(set(actual.items()) - set(expected.items()))) self.assertTrue(actual == expected) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(cur[k], v, "for key " + str(k)) cur.close() if txn_config: @@ -131,7 +131,7 @@ class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): # Insert keys 1..100 nkeys = 100 - orig_keys = range(1, nkeys+1) + orig_keys = list(range(1, nkeys+1)) keys = orig_keys[:] random.shuffle(keys) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp07.py b/src/third_party/wiredtiger/test/suite/test_timestamp07.py index ba29fa24bd7..ab76ff7ea10 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp07.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp07.py @@ -191,7 +191,7 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): # print "tables created" # Insert keys 1..nkeys each with timestamp=key, in some order. - orig_keys = range(1, self.nkeys+1) + orig_keys = list(range(1, self.nkeys+1)) keys = orig_keys[:] random.shuffle(keys) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp13.py b/src/third_party/wiredtiger/test/suite/test_timestamp13.py index 65e0cb7df6f..1585a4cb194 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp13.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp13.py @@ -107,11 +107,11 @@ class test_timestamp13(wttest.WiredTigerTestCase, suite_subprocess): 'key_format=i,value_format=i' + self.extra_config) self.conn.set_timestamp('oldest_timestamp=10') - self.session.begin_transaction('isolation=snapshot') # Rounding to the oldest timestamp will allow the stale read_timestamp # to succeed. The follow-up call to get the read timestamp returns the # chosen read timestamp. - self.session.timestamp_transaction('read_timestamp=5,round_to_oldest=true') + self.session.begin_transaction('isolation=snapshot,roundup_timestamps=(read=true)') + self.session.timestamp_transaction('read_timestamp=5') self.assertTimestampsEqual( self.session.query_timestamp('get=read'), '10') diff --git a/src/third_party/wiredtiger/test/suite/test_truncate01.py b/src/third_party/wiredtiger/test/suite/test_truncate01.py index a136b8220f6..6d941fb8043 100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate01.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate01.py @@ -159,8 +159,8 @@ class test_truncate_cursor_end(wttest.WiredTigerTestCase): c2 = self.session.open_cursor(uri, None) c2.set_key(ds.key(2000)) self.session.truncate(None, c1, c2, None) - self.assertEquals(c1.close(), 0) - self.assertEquals(c2.close(), 0) + self.assertEqual(c1.close(), 0) + self.assertEqual(c2.close(), 0) self.session.drop(uri) if self.type == "table:": @@ -171,8 +171,8 @@ class test_truncate_cursor_end(wttest.WiredTigerTestCase): c2 = self.session.open_cursor(uri, None) c2.set_key(ds.key(2000)) self.session.truncate(None, c1, c2, None) - self.assertEquals(c1.close(), 0) - self.assertEquals(c2.close(), 0) + self.assertEqual(c1.close(), 0) + self.assertEqual(c2.close(), 0) self.session.drop(uri) # Test truncation of empty objects. @@ -199,14 +199,14 @@ class test_truncate_empty(wttest.WiredTigerTestCase): c1.set_key(simple_key(c1, 1000)) c2 = self.session.open_cursor(uri, None) c2.set_key(simple_key(c2, 2000)) - self.assertEquals(self.session.truncate(None, c1, c2, None), 0) + self.assertEqual(self.session.truncate(None, c1, c2, None), 0) # Test truncation of empty objects using a URI def test_truncate_empty_uri(self): uri = self.type + self.name self.session.create(uri, ',key_format=' + self.keyfmt + ',value_format=S') - self.assertEquals(self.session.truncate(uri, None, None, None), 0) + self.assertEqual(self.session.truncate(uri, None, None, None), 0) # Test session.truncate. class test_truncate_cursor(wttest.WiredTigerTestCase): @@ -272,7 +272,7 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None) for i in range(begin, end + 1): expected[ds.key(i)] = [0] - for k, v in expected.iteritems(): + for k, v in expected.items(): cursor.set_key(k) if v == [0] and \ cursor.key_format == 'r' and cursor.value_format == '8t': diff --git a/src/third_party/wiredtiger/test/suite/test_txn01.py b/src/third_party/wiredtiger/test/suite/test_txn01.py index 2de136d9231..daf9b5193c6 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn01.py +++ b/src/third_party/wiredtiger/test/suite/test_txn01.py @@ -110,8 +110,8 @@ class test_txn01(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri, None) self.check(cursor, 0, 0) self.session.begin_transaction() - for i in xrange(self.nentries): - if i > 0 and i % (self.nentries / 37) == 0: + for i in range(self.nentries): + if i > 0 and i % (self.nentries // 37) == 0: self.check(cursor, committed, i) self.session.commit_transaction() committed = i diff --git a/src/third_party/wiredtiger/test/suite/test_txn02.py b/src/third_party/wiredtiger/test/suite/test_txn02.py index b91cf56f84c..9a090aaab04 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn02.py +++ b/src/third_party/wiredtiger/test/suite/test_txn02.py @@ -91,7 +91,7 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): # Each check_log() call takes a second, so we don't call it for # every scenario, we'll limit it to the value of checklog_calls. checklog_calls = 100 if wttest.islongtest() else 2 - checklog_mod = (len(scenarios) / checklog_calls + 1) + checklog_mod = (len(scenarios) // checklog_calls + 1) _debug = False def debug(self, msg): @@ -124,7 +124,7 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): c = session.open_cursor(self.uri, None) actual = dict((k, v) for k, v in c if v != 0) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(c[k], v) c.close() if txn_config: diff --git a/src/third_party/wiredtiger/test/suite/test_txn04.py b/src/third_party/wiredtiger/test/suite/test_txn04.py index 62f9630139a..42ed7baab41 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn04.py +++ b/src/third_party/wiredtiger/test/suite/test_txn04.py @@ -81,7 +81,7 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): c = session.open_cursor(self.uri, None) actual = dict((k, v) for k, v in c if v != 0) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(c[k], v) c.close() if txn_config: diff --git a/src/third_party/wiredtiger/test/suite/test_txn05.py b/src/third_party/wiredtiger/test/suite/test_txn05.py index 4ce67a9df61..6fd72827618 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn05.py +++ b/src/third_party/wiredtiger/test/suite/test_txn05.py @@ -82,7 +82,7 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): c = session.open_cursor(self.uri, None) actual = dict((k, v) for k, v in c if v != 0) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(c[k], v) c.close() if txn_config: @@ -237,7 +237,7 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): # Check the log state after the entire op completes # and run recovery. - if self.scenario_number % (len(test_txn05.scenarios) / 100 + 1) == 0: + if self.scenario_number % (len(test_txn05.scenarios) // 100 + 1) == 0: self.check_log(committed) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_txn06.py b/src/third_party/wiredtiger/test/suite/test_txn06.py index bad62f55292..c0bbe332c79 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_txn06.py +++ b/src/third_party/wiredtiger/test/suite/test_txn06.py @@ -45,7 +45,7 @@ class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): SimpleDataSet(self, self.source_uri, self.nrows).populate() # Now scan the table and copy the rows into a new table - c_src = self.session.create(self.uri, "key_format=S") + c_src = self.session.create(self.uri, "key_format=S,value_format=S") c_src = self.session.open_cursor(self.source_uri) c = self.session.open_cursor(self.uri) for k, v in c_src: diff --git a/src/third_party/wiredtiger/test/suite/test_txn07.py b/src/third_party/wiredtiger/test/suite/test_txn07.py index ceb86344cdc..3bc708fb9d1 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn07.py +++ b/src/third_party/wiredtiger/test/suite/test_txn07.py @@ -93,7 +93,7 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): c = session.open_cursor(self.uri, None) actual = dict((k, v) for k, v in c if v != 0) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(c[k], v) c.close() if txn_config: diff --git a/src/third_party/wiredtiger/test/suite/test_txn09.py b/src/third_party/wiredtiger/test/suite/test_txn09.py index c6a695a50ad..f216cbc04ac 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn09.py +++ b/src/third_party/wiredtiger/test/suite/test_txn09.py @@ -92,7 +92,7 @@ class test_txn09(wttest.WiredTigerTestCase, suite_subprocess): c = session.open_cursor(self.uri, None) actual = dict((k, v) for k, v in c if v != 0) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(c[k], v) c.close() if txn_config: diff --git a/src/third_party/wiredtiger/test/suite/test_txn19.py b/src/third_party/wiredtiger/test/suite/test_txn19.py index 98053a5c7a8..604d8bed8bb 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn19.py +++ b/src/third_party/wiredtiger/test/suite/test_txn19.py @@ -124,7 +124,7 @@ class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): # (having no records initially). The last log file is this # (nrecords/2 + 1), given that we start with log 1. def record_to_logfile(self, recordnum): - return recordnum / 2 + 1 + return recordnum // 2 + 1 # Returns the first record number in a log file. def logfile_to_record(self, logfile): diff --git a/src/third_party/wiredtiger/test/suite/test_unicode01.py b/src/third_party/wiredtiger/test/suite/test_unicode01.py index c46c8529224..baa85e1daf8 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_unicode01.py +++ b/src/third_party/wiredtiger/test/suite/test_unicode01.py @@ -26,21 +26,27 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import wiredtiger, wtscenario, wttest +import sys, wiredtiger, wtscenario, wttest +_python3 = (sys.version_info >= (3, 0, 0)) # test_unicode01.py -# Make sure UTF8 config can be passed to WT_SESSION::create -# -# There are a couple of tricks here: -# 1) we don't want to treat the whole file as UTF-8, because this would -# be the only one in our tree and that causes problems for some -# scripts; and -# 2) we can't pass in a Unicode object directly because the -# SWIG-generated code expects a simple Python string. +# Make sure UTF8 config can be passed to WT_SESSION::create. +# Python turns Unicode strings into UTF-8. class test_unicode01(wttest.WiredTigerTestCase): def test_unicode(self): - self.session.create('table:t', - u'app_metadata={"name" : "Employ\xe9s"}'.encode('utf-8')) + # We use valid Unicode characters that are examples in + # the Unicode standard. + metadata_unicode = u'app_metadata={"name" : "Employ\u222b\u67d2\ud4db"}' + + # In Python2, Unicode and strings are different types, + # and need to be converted. In Python3, there is no separate + # unicode type, unicode characters are just embedded as UTF-8 + # in strings. + if _python3: + metadata_string = metadata_unicode + else: + metadata_string = metadata_unicode.encode('utf-8') + self.session.create('table:t', metadata_string) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_util01.py b/src/third_party/wiredtiger/test/suite/test_util01.py index 9287983ce91..d4c74ec3a01 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_util01.py +++ b/src/third_party/wiredtiger/test/suite/test_util01.py @@ -26,10 +26,12 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import string, os +import string, os, sys from suite_subprocess import suite_subprocess import wiredtiger, wttest +_python3 = (sys.version_info >= (3, 0, 0)) + # test_util01.py # Utilities: wt dump, as well as the dump cursor class test_util01(wttest.WiredTigerTestCase, suite_subprocess): @@ -57,9 +59,10 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): def compare_files(self, filename1, filename2): inheader = isconfig = False - for l1, l2 in zip(open(filename1, "rb"), open(filename2, "rb")): + for l1, l2 in zip(open(filename1, "r"), open(filename2, "r")): if isconfig: if not self.compare_config(l1, l2): + self.tty('Failed comparing: ' + l1 + '<<<>>>' + l2) return False elif l1 != l2: return False @@ -71,31 +74,41 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): inheader = isconfig = False return True - def get_string(self, i, len): + def get_bytes(self, i, len): """ Return a pseudo-random, but predictable string that uses all characters. As a special case, key 0 returns all characters 1-255 repeated """ - ret = '' + ret = b'' if i == 0: for j in range (0, len): - # we ensure that there are no internal nulls, that would - # truncate the string when we're using the 'S' encoding - # The last char in a string is null anyway, so that's tested. - ret += chr(j%255 + 1) + ret += bytes([j%255 + 1]) else: - for j in range(0, len / 3): + for j in range(0, len // 3): k = i + j - # no internal nulls... - ret += chr(k%255 + 1) + chr((k*3)%255 + 1) + chr((k*7)%255 + 1) - return ret + ret += bytes([k%255 + 1, (k*3)%255 + 1, (k*7)%255 + 1]) + return ret + bytes([0]) # Add a final null byte def get_key(self, i): - return ("%0.6d" % i) + ':' + self.get_string(i, 20) + return (b"%0.6d" % i) + b':' + self.get_bytes(i, 20) def get_value(self, i): - return self.get_string(i, 1000) + return self.get_bytes(i, 1000) + + if _python3: + def _ord(self, byte): + return byte + + def _byte_to_str(self, byte): + return chr(byte) + + else: + def _ord(self, byte): + return ord(byte) + + def _byte_to_str(self, byte): + return byte def dumpstr(self, s, hexoutput): """ @@ -105,6 +118,7 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): """ result = '' for c in s: + c = self._byte_to_str(c) if hexoutput: result += "%0.2x" % ord(c) elif c == '\\': @@ -114,13 +128,18 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): else: result += '\\' + "%0.2x" % ord(c) if hexoutput: - result += '00\n' + result += '\n' else: - result += '\\00\n' + result += '\n' return result def table_config(self): - return 'key_format=S,value_format=S' + # Using u configuration lets us store and print all the byte values. + return 'key_format=u,value_format=u' + + def dump_kv_to_line(self, b): + # The output from dump is a 'u' format. + return b.strip(b'\x00').decode() + '\n' def dump(self, usingapi, hexoutput): params = self.table_config() @@ -160,7 +179,8 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): dumpcurs = self.session.open_cursor('table:' + self.tablename, None, dumpopt) for key, val in dumpcurs: - dumpout.write(str(key) + "\n" + str(val) + "\n") + dumpout.write(self.dump_kv_to_line(key) + \ + self.dump_kv_to_line(val)) dumpcurs.close() else: dumpargs = ["dump"] diff --git a/src/third_party/wiredtiger/test/suite/test_util02.py b/src/third_party/wiredtiger/test/suite/test_util02.py index 277b799cfd7..4af6f5747f7 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_util02.py +++ b/src/third_party/wiredtiger/test/suite/test_util02.py @@ -65,7 +65,7 @@ class test_util02(wttest.WiredTigerTestCase, suite_subprocess): # The last char in a string is null anyway, so that's tested. ret += chr(j%255 + 1) else: - for j in range(0, len / 3): + for j in range(0, len // 3): k = i + j # no internal nulls... ret += chr(k%255 + 1) + chr((k*3)%255 + 1) + chr((k*7)%255 + 1) @@ -75,7 +75,7 @@ class test_util02(wttest.WiredTigerTestCase, suite_subprocess): if self.key_format == 'S': return ("%0.6d" % i) + ':' + self.get_string(i, 20) elif self.key_format == 'r': - return long(i + 1) + return self.recno(i + 1) else: return i + 1 diff --git a/src/third_party/wiredtiger/test/suite/test_util09.py b/src/third_party/wiredtiger/test/suite/test_util09.py index 37a2ad8f165..98a96d0c7a9 100644 --- a/src/third_party/wiredtiger/test/suite/test_util09.py +++ b/src/third_party/wiredtiger/test/suite/test_util09.py @@ -39,7 +39,7 @@ class test_util09(wttest.WiredTigerTestCase, suite_subprocess): def populate_file(self, filename, low, high): """ - Insert some simple key / value lines into the file + Insert some simple key // value lines into the file """ keys = {} with open("loadtext.in", "w") as f: diff --git a/src/third_party/wiredtiger/test/suite/test_util13.py b/src/third_party/wiredtiger/test/suite/test_util13.py index ea13093a230..5a92fb6a8ad 100644 --- a/src/third_party/wiredtiger/test/suite/test_util13.py +++ b/src/third_party/wiredtiger/test/suite/test_util13.py @@ -100,11 +100,11 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): # the actual configuration and they match. match = all(item in da.items() for item in dx.items()) if match == False: - print "MISMATCH:" - print "Original dict: " - print da - print "Expected config: " - print dx + print("MISMATCH:") + print("Original dict: ") + print(da) + print("Expected config: ") + print(dx) return match def compare_files(self, expect_subset, dump_out): diff --git a/src/third_party/wiredtiger/test/suite/test_verify.py b/src/third_party/wiredtiger/test/suite/test_verify.py index 39fb5efa672..dda68f43cd2 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_verify.py +++ b/src/third_party/wiredtiger/test/suite/test_verify.py @@ -29,6 +29,10 @@ import os, struct from suite_subprocess import suite_subprocess import wiredtiger, wttest +try: + xrange +except NameError: #python3 + xrange = range # test_verify.py # Utilities: wt verify @@ -42,7 +46,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): """ cursor = self.session.open_cursor('table:' + tablename, None, None) key = '' - for i in range(0, self.nentries): + for i in xrange(0, self.nentries): key += str(i) cursor[key] = key + key cursor.close() @@ -77,7 +81,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): filename = tablename + ".wt" filesize = os.path.getsize(filename) - position = (filesize * pct) / 100 + position = (filesize * pct) // 100 self.pr('damaging file at: ' + str(position)) fp = open(filename, "r+b") @@ -130,7 +134,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): self.session.create('table:' + self.tablename, params) self.populate(self.tablename) with self.open_and_position(self.tablename, 75) as f: - for i in range(0, 4096): + for i in xrange(0, 4096): f.write(struct.pack('B', 0)) # open_and_position closed the session/connection, reopen them now. @@ -149,7 +153,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): self.session.create('table:' + self.tablename, params) self.populate(self.tablename) with self.open_and_position(self.tablename, 75) as f: - for i in range(0, 4096): + for i in xrange(0, 4096): f.write(struct.pack('B', 0)) self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out", failure=True) @@ -164,8 +168,8 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): self.session.create('table:' + self.tablename, params) self.populate(self.tablename) with self.open_and_position(self.tablename, 25) as f: - for i in range(0, 100): - f.write('\x01\xff\x80') + for i in xrange(0, 100): + f.write(b'\x01\xff\x80') self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out", failure=True) self.check_non_empty_file("verifyerr.out") diff --git a/src/third_party/wiredtiger/test/suite/wtdataset.py b/src/third_party/wiredtiger/test/suite/wtdataset.py index b0c9d370517..9620c2127b0 100644..100755 --- a/src/third_party/wiredtiger/test/suite/wtdataset.py +++ b/src/third_party/wiredtiger/test/suite/wtdataset.py @@ -26,6 +26,10 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # +import sys +_python3 = (sys.version_info >= (3, 0, 0)) +if _python3: + xrange = range class BaseDataSet(object): """ @@ -73,6 +77,8 @@ class BaseDataSet(object): def key_by_format(i, key_format): if key_format == 'i' or key_format == 'r': return i + elif _python3 and key_format == 'u': + return bytes(('%015d' % i).encode()) elif key_format == 'S' or key_format == 'u': return str('%015d' % i) else: @@ -84,6 +90,8 @@ class BaseDataSet(object): def value_by_format(i, value_format): if value_format == 'i' or value_format == 'r': return i + elif _python3 and value_format == 'u': + return bytes((str(i) + ': abcdefghijklmnopqrstuvwxyz').encode()) elif value_format == 'S' or value_format == 'u': return str(i) + ': abcdefghijklmnopqrstuvwxyz' elif value_format == '8t': diff --git a/src/third_party/wiredtiger/test/suite/wtscenario.py b/src/third_party/wiredtiger/test/suite/wtscenario.py index 8b18ee91ca9..610b0de1723 100644 --- a/src/third_party/wiredtiger/test/suite/wtscenario.py +++ b/src/third_party/wiredtiger/test/suite/wtscenario.py @@ -33,8 +33,8 @@ import suite_random # Support scenarios based testing def powerrange(start, stop, mult): """ - Like xrange, generates a range from start to stop. - Unlike xrange, the range is inclusive of stop, + Like range, generates a range from start to stop. + Unlike range, the range is inclusive of stop, each step is multiplicative, and as a special case, the stop value is returned as the last item. """ @@ -269,9 +269,9 @@ class wtscenario: for lmax in powerrange(lmin, 512*megabyte, 1024): for cache in [megabyte, 32*megabyte, 1000*megabyte]: scen = wtscenario() - scen.ioverflow = max(imin / 40, 40) + scen.ioverflow = max(imin // 40, 40) scen.imax = imax - scen.loverflow = max(lmin / 40, 40) + scen.loverflow = max(lmin // 40, 40) scen.lmax = lmax scen.cache_size = cache s.append((scen.shortName(), dict(session_create_scenario=scen))) diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index ca4a8295373..d3d75d24c42 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -29,6 +29,7 @@ # WiredTigerTestCase # parent class for all test cases # +from __future__ import print_function # If unittest2 is available, use it in preference to (the old) unittest try: @@ -45,6 +46,8 @@ def shortenWithEllipsis(s, maxlen): s = s[0:maxlen-3] + '...' return s +_python3 = (sys.version_info >= (3, 0, 0)) + class CapturedFd(object): """ CapturedFd encapsulates a file descriptor (e.g. 1 or 2) that is diverted @@ -192,7 +195,7 @@ class WiredTigerTestCase(unittest.TestCase): WiredTigerTestCase._parentTestdir = d WiredTigerTestCase._builddir = builddir WiredTigerTestCase._origcwd = os.getcwd() - WiredTigerTestCase._resultfile = open(os.path.join(d, 'results.txt'), "w", 0) # unbuffered + WiredTigerTestCase._resultfile = open(os.path.join(d, 'results.txt'), "w", 1) # line buffered WiredTigerTestCase._gdbSubprocess = gdbSub WiredTigerTestCase._lldbSubprocess = lldbSub WiredTigerTestCase._longtest = longtest @@ -293,7 +296,7 @@ class WiredTigerTestCase(unittest.TestCase): else: extfiles[ext] = complete if len(extfiles) != 0: - result = ',extensions=[' + ','.join(extfiles.values()) + ']' + result = ',extensions=[' + ','.join(list(extfiles.values())) + ']' return result # Can be overridden, but first consider setting self.conn_config @@ -311,10 +314,10 @@ class WiredTigerTestCase(unittest.TestCase): try: conn = self.wiredtiger_open(home, conn_param) except wiredtiger.WiredTigerError as e: - print "Failed wiredtiger_open: dir '%s', config '%s'" % \ - (home, conn_param) + print("Failed wiredtiger_open: dir '%s', config '%s'" % \ + (home, conn_param)) raise e - self.pr(`conn`) + self.pr(repr(conn)) return conn # Replacement for wiredtiger.wiredtiger_open that returns @@ -392,13 +395,25 @@ class WiredTigerTestCase(unittest.TestCase): self.tearDown() raise + # Used as part of tearDown determining if there is an error. + def list2reason(self, result, fieldname): + exc_list = getattr(result, fieldname, None) + if exc_list and exc_list[-1][0] is self: + return exc_list[-1][1] + def tearDown(self): - excinfo = sys.exc_info() - passed = (excinfo == (None, None, None)) - if passed: - skipped = False - else: - skipped = (excinfo[0] == unittest.SkipTest) + # This approach works for all our support Python versions and + # is suggested by one of the answers in: + # https://stackoverflow.com/questions/4414234/getting-pythons-unittest-results-in-a-teardown-method + if hasattr(self, '_outcome'): # Python 3.4+ + result = self.defaultTestResult() # these 2 methods have no side effects + self._feedErrorsToResult(result, self._outcome.errors) + else: # Python 3.2 - 3.3 or 3.0 - 3.1 and 2.7 + result = getattr(self, '_outcomeForDoCleanups', self._resultForDoCleanups) + error = self.list2reason(result, 'errors') + failure = self.list2reason(result, 'failures') + passed = not error and not failure + self.pr('finishing') # Close all connections that weren't explicitly closed. @@ -425,26 +440,26 @@ class WiredTigerTestCase(unittest.TestCase): os.chdir(self.origcwd) # Make sure no read-only files or directories were left behind - os.chmod(self.testdir, 0777) + os.chmod(self.testdir, 0o777) for root, dirs, files in os.walk(self.testdir): for d in dirs: - os.chmod(os.path.join(root, d), 0777) + os.chmod(os.path.join(root, d), 0o777) for f in files: - os.chmod(os.path.join(root, f), 0666) + os.chmod(os.path.join(root, f), 0o666) + self.pr('passed=' + str(passed)) # Clean up unless there's a failure - if (passed or skipped) and not WiredTigerTestCase._preserveFiles: + if passed and not WiredTigerTestCase._preserveFiles: shutil.rmtree(self.testdir, ignore_errors=True) else: self.pr('preserving directory ' + self.testdir) elapsed = time.time() - self.starttime if elapsed > 0.001 and WiredTigerTestCase._verbose >= 2: - print "%s: %.2f seconds" % (str(self), elapsed) - if not passed and not skipped: - print "ERROR in " + str(self) + print("%s: %.2f seconds" % (str(self), elapsed)) + if not passed: + print("ERROR in " + str(self)) self.pr('FAIL') - self.prexception(excinfo) self.pr('preserving directory ' + self.testdir) if WiredTigerTestCase._verbose > 2: self.prhead('TEST COMPLETED') @@ -515,7 +530,7 @@ class WiredTigerTestCase(unittest.TestCase): raised = False try: expr() - except BaseException, err: + except BaseException as err: if not isinstance(err, exceptionType): self.fail('Exception of incorrect type raised, got type: ' + \ str(type(err))) @@ -550,7 +565,7 @@ class WiredTigerTestCase(unittest.TestCase): """ try: expr() - except BaseException, err: + except BaseException as err: sys.stderr.write('Exception: ' + str(err)) raise @@ -607,7 +622,7 @@ class WiredTigerTestCase(unittest.TestCase): @staticmethod def prout(s): - os.write(WiredTigerTestCase._dupout, s + '\n') + os.write(WiredTigerTestCase._dupout, str.encode(s + '\n')) def pr(self, s): """ @@ -632,6 +647,30 @@ class WiredTigerTestCase(unittest.TestCase): traceback.print_exception(excinfo[0], excinfo[1], excinfo[2], None, WiredTigerTestCase._resultfile) WiredTigerTestCase._resultfile.write('\n') + def recno(self, i): + """ + return a recno key + """ + if _python3: + return i + else: + return long(i) + + def ord_byte(self, b): + """ + return the 'ord' of a single byte. + In Python2 a set of bytes is represented as a string, and a single + byte is a string of length one. In Python3, bytes are an array of + ints, so no explicit ord() call is needed. + """ + if _python3: + return b + else: + return ord(b) + + def is_python3(self): + return _python3 + # print directly to tty, useful for debugging def tty(self, message): WiredTigerTestCase.tty(message) diff --git a/src/third_party/wiredtiger/test/suite/wtthread.py b/src/third_party/wiredtiger/test/suite/wtthread.py index 5eab6527fe8..482bdd73c4b 100644..100755 --- a/src/third_party/wiredtiger/test/suite/wtthread.py +++ b/src/third_party/wiredtiger/test/suite/wtthread.py @@ -26,7 +26,10 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import Queue +try: + import Queue as queue # python2 +except ImportError: + import queue import os, shutil, sys, threading, time, wiredtiger, wttest from helper import compare_tables @@ -86,13 +89,13 @@ class backup_thread(threading.Thread): # wttest to do that.. if not compare_tables( self, sess, uris, "checkpoint=WiredTigerCheckpoint"): - print "Error: checkpoint tables differ." + print("Error: checkpoint tables differ.") else: wttest.WiredTigerTestCase.printVerbose( 3, "Checkpoint tables match") if not compare_tables(self, bkp_session, uris): - print "Error: backup tables differ." + print("Error: backup tables differ.") else: wttest.WiredTigerTestCase.printVerbose( 3, "Backup tables match") @@ -109,11 +112,11 @@ class backup_thread(threading.Thread): # 'd' for drop a table # 't' for create a table and insert a single item into it class op_thread(threading.Thread): - def __init__(self, conn, uris, key_fmt, queue, done): + def __init__(self, conn, uris, key_fmt, work_queue, done): self.conn = conn self.uris = uris self.key_fmt = key_fmt - self.queue = queue + self.work_queue = work_queue self.done = done threading.Thread.__init__(self) @@ -127,7 +130,7 @@ class op_thread(threading.Thread): cursors.append(sess.open_cursor(next_uri, None, None)) while not self.done.isSet(): try: - op, key, value = self.queue.get_nowait() + op, key, value = self.work_queue.get_nowait() if op == 'gi': # Group insert a number of tables. sess.begin_transaction() for next_cur in cursors: @@ -166,8 +169,8 @@ class op_thread(threading.Thread): # These operations can fail, if the drop in another # thread happened pass - self.queue.task_done() - except Queue.Empty: + self.work_queue.task_done() + except queue.Empty: # Wait on the queue until done is flagged time.sleep(0.01) if (len(self.uris) == 1): |