diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2021-08-10 15:16:08 +1000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-08-10 05:40:43 +0000 |
| commit | 13397fb0bf9aa9aca4e8fe6b736cc458bba1d494 (patch) | |
| tree | bd6cfb32acfd590821f7c7d38739ca454f0e6687 | |
| parent | 2cc4cf27e7fa3fb7a158f05f8ae1b9beab742515 (diff) | |
| download | mongo-13397fb0bf9aa9aca4e8fe6b736cc458bba1d494.tar.gz | |
Import wiredtiger: ffebd365d6c3c18422437bcae51d8a26a17772a0 from branch mongodb-5.0
ref: 8b35a8d923..ffebd365d6
for: 5.0.3
WT-6755 Documentation: populate developer glossary
WT-6905 Write row-store and column-store subpage for Architecture Guide
WT-6910 Write "history store" subpage for Architecture Guide
WT-7007 Backup architecture guide page
WT-7198 Fix test_backup15 failure with backup mismatch
WT-7352 Fix test_hs01 conflict between concurrent operations in cursor modify
WT-7521 Remove excess ckplist invalidations
WT-7592 Remove log_flush("sync=background") support
WT-7599 Update the CONFIG file based on the release that is about to run for compatibility tests
WT-7673 Investigate and fix manydbs test failure on Windows
WT-7830 Migrate the python setup scripts to use cmake
WT-7838 Ability for ordered timestamp assertion to do more than a log message
WT-7842 Remove explicit ulimit -n call in many-collection-test
WT-7860 Improve code coverage reporting
WT-7876 Update rollback to stable test to use correct boolean values and update statistic checking logic
WT-7893 Remove ignored message from wiredtiger_open in test_encrypt08
WT-7895 Fix arch-data-file.dox documentation build failure
WT-7897 Enable verbose logging for test_backup15 to aid debugging
WT-7898 Upload many-coll-test artifacts
WT-7900 Fix insertion of new records in test format for column-store
WT-7901 test suite cleanup
WT-7908 Make variable-length column store work again with the static tests
WT-7935 Add arm64 implementation of rdtsc equivalent instruction
WT-7937 Fix s_docs to use sh, not bash syntax
WT-7938 Fix rollback-to-stable memory leak on error
219 files changed, 4664 insertions, 3174 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 43dadd9ff40..9663188789b 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -1385,12 +1385,10 @@ methods = { 'WT_SESSION.log_flush' : Method([ Config('sync', 'on', r''' forcibly flush the log and wait for it to achieve the synchronization - level specified. The \c background setting initiates a background - synchronization intended to be used with a later call to - WT_SESSION::transaction_sync. The \c off setting forces any + level specified. The \c off setting forces any buffered log records to be written to the file system. The \c on setting forces log records to be written to the storage device''', - choices=['background', 'off', 'on']), + choices=['off', 'on']), ]), 'WT_SESSION.log_printf' : Method([]), @@ -1581,14 +1579,6 @@ methods = { ]), 'WT_SESSION.strerror' : Method([]), -'WT_SESSION.transaction_sync' : Method([ - Config('timeout_ms', '1200000', # !!! Must match WT_SESSION_BG_SYNC_MSEC - r''' - maximum amount of time to wait for background sync to complete in - milliseconds. A value of zero disables the timeout and returns - immediately''', - type='int'), -]), 'WT_SESSION.truncate' : Method([]), 'WT_SESSION.upgrade' : Method([]), @@ -1706,13 +1696,10 @@ methods = { min=1), Config('sync', '', r''' override whether to sync log records when the transaction commits, - inherited from ::wiredtiger_open \c transaction_sync. - The \c background setting initiates a background - synchronization intended to be used with a later call to - WT_SESSION::transaction_sync. The \c off setting does not + inherited from ::wiredtiger_open \c transaction_sync. The \c off setting does not wait for record to be written or synchronized. The \c on setting forces log records to be written to the storage device''', - choices=['background', 'off', 'on']), + choices=['off', 'on']), ]), 'WT_SESSION.prepare_transaction' : Method([ diff --git a/src/third_party/wiredtiger/dist/docs_data.py b/src/third_party/wiredtiger/dist/docs_data.py index 0aa93464bef..831cdc61f30 100644 --- a/src/third_party/wiredtiger/dist/docs_data.py +++ b/src/third_party/wiredtiger/dist/docs_data.py @@ -11,6 +11,9 @@ class ArchDocPage: # List of all architecture subsections ########################################## arch_doc_pages = [ + ArchDocPage('arch-backup', + ['WT_CURSOR_BACKUP'], + ['src/cursor/cur_backup.c', 'src/cursor/cur_backup_incr.c']), ArchDocPage('arch-block', ['WT_BLOCK', 'WT_BLOCK_CKPT', 'WT_BLOCK_DESC', 'WT_BLOCK_HEADER', 'WT_BM', 'WT_EXTLIST'], @@ -24,9 +27,6 @@ arch_doc_pages = [ ['src/block/block_ckpt.c', 'src/block/block_ckpt_scan.c', 'src/conn/conn_ckpt.c', 'src/meta/meta_ckpt.c', 'src/txn/txn_ckpt.c']), - ArchDocPage('arch-column', - ['WT_BTREE'], - ['src/include/btree.h']), ArchDocPage('arch-cursor', ['WT_CURSOR', 'WT_CURSOR_BACKUP', 'WT_CURSOR_BTREE', 'WT_CURSOR_BULK', 'WT_CURSOR_DATA_SOURCE', 'WT_CURSOR_DUMP', 'WT_CURSOR_INDEX', @@ -67,7 +67,7 @@ arch_doc_pages = [ ArchDocPage('arch-python', [], ['lang/python/']), - ArchDocPage('arch-row', + ArchDocPage('arch-row-column', ['WT_BTREE'], ['src/include/btree.h']), ArchDocPage('arch-rts', diff --git a/src/third_party/wiredtiger/dist/s_docs b/src/third_party/wiredtiger/dist/s_docs index 1077ce5f332..83b31aa1a09 100755 --- a/src/third_party/wiredtiger/dist/s_docs +++ b/src/third_party/wiredtiger/dist/s_docs @@ -1,7 +1,8 @@ #! /bin/bash t=__wt.$$ -trap 'rm -f $t' 0 1 2 3 13 15 +t2=__wt.$$.2 +trap 'rm -f $t $t2' 0 1 2 3 13 15 # Skip this when building release packages: docs are built separately test -n "$WT_RELEASE_BUILD" && exit 0 @@ -53,6 +54,18 @@ wtperf_config() echo 'q') | ed ../src/docs/wtperf.dox 1>/dev/null 2>/dev/null } +glossarychk() +{ + (cd ../src/docs && grep '^<tr><td>' arch-glossary.dox | cut -d'>' -f3 | sed -e 's/<.*//') > $t + sort --ignore-case $t > $t2 + if ! diff $t $t2; then + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo "arch-glossary.dox entries out of order" + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + e=1 + fi +} + structurechk() { # @page names should match the source file name @@ -116,7 +129,7 @@ spellchk() (cd ../src/docs && # Separate quoted newlines "line\nline" so "nline" is not reported. - sed -e 's/\("[^"]*\)\\n\([^"]*"\)/\1 \2/' *.dox | \ + sed -e 's/\("[^"]*\)\\n\([^"]*"\)/\1 \2/' -e 's/[@\\]sic *\([^ ]*\) //g' *.dox | \ aspell --encoding=iso-8859-1 --lang=en_US --personal=./spell.ok list) | sort -u > $t test -s $t && { @@ -177,6 +190,9 @@ EOF e=1 } + # Copy any needed javascript + cp ../src/docs/js/* ../docs/ + # Fix up bad links doxygen generates in navtree.js (cd ../docs && sed -i~ -e 's,/\.html,/,' -e 's,\.html\.html,.html,' navtree.js && @@ -243,6 +259,7 @@ wtperf_config asciichk spellchk structurechk +glossarychk # Check the docs data input file. check_docs_data diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 99e582bdfc5..d0b2ab55adf 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -52,7 +52,9 @@ Btree Buf Bxxx Bzip +CACHEABLE CAS +CBT CCCCCCCCCCCCCCCCCC CELL's CELLs @@ -573,6 +575,7 @@ bzDecompressInit bzalloc bzfree bzip +cacheable calc call's calloc @@ -619,6 +622,7 @@ clsm cmd cmp cnt +cntvct colcheck colgroup colgroups @@ -1070,6 +1074,7 @@ mmap mmrand mnt movemask +mrs msecs msg msvc @@ -1417,6 +1422,7 @@ uint uintmax unbare unbuffered +uncacheable uncompressed uncompressing uncustomized diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 82c73e9f780..32d3d237858 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -573,7 +573,6 @@ conn_stats = [ TxnStat('txn_set_ts_oldest_upd', 'set timestamp oldest updates'), TxnStat('txn_set_ts_stable', 'set timestamp stable calls'), TxnStat('txn_set_ts_stable_upd', 'set timestamp stable updates'), - TxnStat('txn_sync', 'transaction sync calls'), TxnStat('txn_timestamp_oldest_active_read', 'transaction read timestamp of the oldest active reader', 'no_clear,no_scale'), TxnStat('txn_walk_sessions', 'transaction walk of concurrent sessions'), @@ -585,7 +584,6 @@ conn_stats = [ YieldStat('child_modify_blocked_page', 'page reconciliation yielded due to child modification'), YieldStat('conn_close_blocked_lsm', 'connection close yielded for lsm manager shutdown'), YieldStat('dhandle_lock_blocked', 'data handle lock yielded'), - YieldStat('log_server_sync_blocked', 'log server sync yielded for log write'), YieldStat('page_busy_blocked', 'page acquire busy blocked'), YieldStat('page_del_rollback_blocked', 'page delete rollback time sleeping for state change (usecs)'), YieldStat('page_forcible_evict_blocked', 'page acquire eviction blocked'), diff --git a/src/third_party/wiredtiger/examples/c/Makefile.am b/src/third_party/wiredtiger/examples/c/Makefile.am index 6c20b31a146..e83bac63f2c 100644 --- a/src/third_party/wiredtiger/examples/c/Makefile.am +++ b/src/third_party/wiredtiger/examples/c/Makefile.am @@ -25,7 +25,6 @@ noinst_PROGRAMS = \ ex_schema \ ex_smoke \ ex_stat \ - ex_sync \ ex_thread ex_encrypt_LDFLAGS = -rdynamic diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index 9742a4f59a0..c769ffff4b4 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -700,10 +700,6 @@ session_ops(WT_SESSION *session) error_check(session->truncate(session, "table:mytable", NULL, NULL, NULL)); /*! [Truncate a table] */ - /*! [Transaction sync] */ - error_check(session->transaction_sync(session, NULL)); - /*! [Transaction sync] */ - /*! [Reset the session] */ error_check(session->reset(session)); /*! [Reset the session] */ diff --git a/src/third_party/wiredtiger/examples/c/ex_sync.c b/src/third_party/wiredtiger/examples/c/ex_sync.c deleted file mode 100644 index 0ee99e6bc8d..00000000000 --- a/src/third_party/wiredtiger/examples/c/ex_sync.c +++ /dev/null @@ -1,130 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * ex_sync.c - * demonstrates how to use the transaction sync configuration. - */ -#include <test_util.h> - -static const char *home; -static const char *const uri = "table:test"; - -#define CONN_CONFIG "create,cache_size=100MB,log=(archive=false,enabled=true)" -#define MAX_KEYS 100 - -int -main(int argc, char *argv[]) -{ - WT_CONNECTION *wt_conn; - WT_CURSOR *cursor; - WT_SESSION *session; - int i, record_count, ret; - char k[32], v[32]; - const char *conf; - - home = example_setup(argc, argv); - error_check(wiredtiger_open(home, NULL, CONN_CONFIG, &wt_conn)); - - error_check(wt_conn->open_session(wt_conn, NULL, NULL, &session)); - error_check(session->create(session, uri, "key_format=S,value_format=S")); - - error_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); - /* - * Perform some operations with individual auto-commit transactions. - */ - error_check(session->begin_transaction(session, NULL)); - for (record_count = 0, i = 0; i < MAX_KEYS; i++, record_count++) { - if (i == MAX_KEYS / 2) { - error_check(session->commit_transaction(session, "sync=background")); - ret = session->transaction_sync(session, "timeout_ms=0"); - if (ret == ETIMEDOUT) - printf("Transactions not yet stable\n"); - else if (ret != 0) { - fprintf( - stderr, "session.transaction_sync: error %s\n", session->strerror(session, ret)); - exit(1); - } - error_check(session->begin_transaction(session, NULL)); - } else { - if ((record_count % 3) == 0) - conf = "sync=background"; - else - conf = "sync=off"; - error_check(session->commit_transaction(session, conf)); - error_check(session->begin_transaction(session, NULL)); - } - (void)snprintf(k, sizeof(k), "key%d", i); - (void)snprintf(v, sizeof(v), "value%d", i); - cursor->set_key(cursor, k); - cursor->set_value(cursor, v); - error_check(cursor->insert(cursor)); - } - error_check(session->commit_transaction(session, "sync=background")); - printf("Wait forever until stable\n"); - error_check(session->transaction_sync(session, NULL)); - printf("Transactions now stable\n"); - error_check(session->begin_transaction(session, NULL)); - /* - * Perform some operations within a single transaction. - */ - for (i = MAX_KEYS; i < MAX_KEYS + 5; i++, record_count++) { - (void)snprintf(k, sizeof(k), "key%d", i); - (void)snprintf(v, sizeof(v), "value%d", i); - cursor->set_key(cursor, k); - cursor->set_value(cursor, v); - error_check(cursor->insert(cursor)); - } - error_check(session->commit_transaction(session, "sync=on")); - error_check(session->transaction_sync(session, "timeout_ms=0")); - - /* - * Demonstrate using log_flush to force the log to disk. - */ - for (i = 0; i < MAX_KEYS; i++, record_count++) { - (void)snprintf(k, sizeof(k), "key%d", record_count); - (void)snprintf(v, sizeof(v), "value%d", record_count); - cursor->set_key(cursor, k); - cursor->set_value(cursor, v); - error_check(cursor->insert(cursor)); - } - error_check(session->log_flush(session, "sync=on")); - - for (i = 0; i < MAX_KEYS; i++, record_count++) { - (void)snprintf(k, sizeof(k), "key%d", record_count); - (void)snprintf(v, sizeof(v), "value%d", record_count); - cursor->set_key(cursor, k); - cursor->set_value(cursor, v); - error_check(cursor->insert(cursor)); - } - error_check(cursor->close(cursor)); - error_check(session->log_flush(session, "sync=off")); - error_check(session->log_flush(session, "sync=on")); - - error_check(wt_conn->close(wt_conn, NULL)); - - return (EXIT_SUCCESS); -} diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 258ff799271..0d24bead828 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "8b35a8d923eca1558aee73c45a5031ab65e82032" + "commit": "ffebd365d6c3c18422437bcae51d8a26a17772a0" } diff --git a/src/third_party/wiredtiger/lang/python/setup.py b/src/third_party/wiredtiger/lang/python/setup.py index 22261bc8272..d6cedb3ab3d 100644 --- a/src/third_party/wiredtiger/lang/python/setup.py +++ b/src/third_party/wiredtiger/lang/python/setup.py @@ -27,8 +27,8 @@ # OTHER DEALINGS IN THE SOFTWARE. # -import re, os, sys -from distutils.core import setup, Extension +import re, os +from setuptools import setup, Extension # OS X hack: turn off the Universal binary support that is built into the # Python build machinery, just build for the default CPU architecture. diff --git a/src/third_party/wiredtiger/lang/python/setup_pip.py b/src/third_party/wiredtiger/lang/python/setup_pip.py index 0dac330389e..ede4f472548 100755 --- a/src/third_party/wiredtiger/lang/python/setup_pip.py +++ b/src/third_party/wiredtiger/lang/python/setup_pip.py @@ -34,11 +34,9 @@ # "python setup_pip.py sdist", this creates a tar.gz file under ./dist . from __future__ import print_function import os, os.path, re, shutil, site, sys -from setuptools import setup, Distribution -from distutils.extension import Extension +from setuptools import setup, Distribution, Extension import distutils.sysconfig import distutils.ccompiler -from distutils.errors import CompileError, LinkError import subprocess from subprocess import call import setuptools.command.install @@ -75,12 +73,12 @@ def check_needed_dependencies(builtins, inc_paths, lib_paths): distutils.sysconfig.customize_compiler(compiler) compiler.set_library_dirs(library_dirs) missing = [] - for name, libname, instructions in builtins: + for _, libname, instructions in builtins: found = compiler.find_library_file(library_dirs, libname) if found is None: msg(libname + ": missing") msg(instructions) - msg("after installing it, set LD_LIBRARY_PATH or DYLD_LIBRARY_PATH") + msg("after installing it, set CMAKE_LIBRARY_PATH") missing.append(libname) else: package_top = os.path.dirname(os.path.dirname(found)) @@ -94,41 +92,12 @@ def check_needed_dependencies(builtins, inc_paths, lib_paths): #if len(missing) > 0: # die("install packages for: " + str(missing)) -# find_executable -- -# Locate an executable in the PATH. -def find_executable(exename, path): - p = subprocess.Popen(['which', exename ], stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=True) - out, err = p.communicate('') - out = str(out) # needed for Python3 - if out == '': - if err != '': - err = ': "' + err + '"' - die('"' + exename + '": not found in path' + err) - dirname = os.path.dirname(out) - if not dirname in path: - path.append(dirname) - -# get_build_path -- -# Create a PATH that can be used for installation. Apparently, -# installation commands are run with a restricted PATH, and -# autoreconf/aclocal will not normally be found. -def get_build_path(): - build_paths = [] - find_executable('autoreconf', build_paths) - find_executable('aclocal', build_paths) - build_path = os.environ['PATH'] + ':' + ':'.join(build_paths) - return build_path - # get_compile_flags -- # Get system specific compile flags. Return a triple: C preprocessor # flags, C compilation flags and linker flags. def get_compile_flags(inc_paths, lib_paths): # Suppress warnings building SWIG generated code - if sys.platform == 'win32' and cc == 'msvc': - cflags = ['/arch:SSE2', '/EHsc'] - cppflags = [] - ldflags = [] + if sys.platform == 'win32': # Windows untested and incomplete, don't claim that it works. die('Windows is not supported by this setup script') else: @@ -181,44 +150,12 @@ def get_library_dirs(): dirs.append("/opt/local/lib") dirs.append("/usr/lib") dirs.append("/usr/lib64") - for path in ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'LIBRARY_PATH']: + for path in ['CMAKE_LIBRARY_PATH', 'LIBRARY_PATH']: if path in os.environ: dirs.extend(os.environ[path].split(':')) dirs = list(set(filter(os.path.isdir, dirs))) return dirs -# source_filter -# Make any needed changes to the original sources list and return a manifest, -# a list of source file names relative to the new staging root. Any entry -# that needs to be renamed returned in a dictionary where the entry's key -# is the new name and the value is the old source name. -def source_filter(sources): - result = [] - movers = dict() - py_dir = os.path.join('lang', 'python') - pywt_dir = os.path.join(py_dir, 'wiredtiger') - pywt_build_dir = os.path.join('build_posix', py_dir, 'wiredtiger') - pywt_prefix = pywt_dir + os.path.sep - for f in sources: - if not re.match(source_regex, f): - continue - src = f - dest = f - # move all lang/python files to the top level. - if dest.startswith(pywt_prefix): - dest = os.path.basename(dest) - if dest == 'init.py': - dest = '__init__.py' - if dest != src: - movers[dest] = src - result.append(dest) - # Add SWIG generated files - result.append(os.path.join(py_dir, 'wiredtiger_wrap.c')) - wiredtiger_py = 'swig_wiredtiger.py' - result.append('swig_wiredtiger.py') - movers['swig_wiredtiger.py'] = os.path.join(py_dir, 'wiredtiger.py') - return result, movers - ################################################################ # Do some initial setup and checks. this_abs_script = os.path.abspath(__file__) @@ -243,10 +180,10 @@ if sys.maxsize < 2**32: die('need to be running on a 64 bit system, and have a 64 bit Python') python_rel_dir = os.path.join('lang', 'python') -build_dir = os.path.join(wt_dir, 'build_posix') -makefile = os.path.join(build_dir, 'Makefile') +build_dir = os.path.join(wt_dir, 'cmake_pip_build') +makefile = os.path.join(build_dir, 'build.ninja') built_sentinal = os.path.join(build_dir, 'built.txt') -conf_make_dir = 'build_posix' +conf_make_dir = 'cmake_pip_build' wt_swig_lib_name = os.path.join(python_rel_dir, '_wiredtiger.so') ################################################################ @@ -257,11 +194,6 @@ long_description = 'WiredTiger is a ' + short_description + '.\n\n' + \ open(os.path.join(wt_dir, 'README')).read() wt_ver, wt_full_ver = get_wiredtiger_versions(wt_dir) -build_path = get_build_path() - -# We only need a small set of directories to build a WT library, -# we also include any files at the top level. -source_regex = r'^(?:(?:api|build_posix|ext|lang/python|src|dist)/|[^/]*$)' # The builtins that we include in this distribution. builtins = [ @@ -277,7 +209,7 @@ builtins = [ 'It can be installed via: apt-get install zlib1g' ], [ 'zstd', 'zstd', 'Need to install zstd\n' + \ - 'It can be installed via: apt-get install zstd' ] + 'It can be installed via: apt-get install libzstd-dev' ] ] builtin_names = [b[0] for b in builtins] builtin_libraries = [b[1] for b in builtins] @@ -285,22 +217,19 @@ builtin_libraries = [b[1] for b in builtins] # Here's the configure/make operations we perform before the python extension # is linked. configure_cmds = [ - './makemake --clean-and-make', - './reconf', - # force building a position independent library; it will be linked - # into a single shared library with the SWIG interface code. - 'CFLAGS="${CFLAGS:-} -fPIC -DPIC" ' + \ - '../configure --enable-python --with-builtins=' + ','.join(builtin_names) + 'cmake -B cmake_pip_build -G Ninja -DENABLE_STATIC=1 -DCMAKE_C_FLAGS="${CFLAGS:-}" -DENABLE_PYTHON=1 ' + \ + ' '.join(map(lambda name: '-DHAVE_BUILTIN_EXTENSION_' + name.upper() + '=1', builtin_names)), ] # build all the builtins, at the moment they are all compressors. make_cmds = [] for name in builtin_names: - make_cmds.append('(cd ext/compressors/' + name + '/; make)') -make_cmds.append('make libwiredtiger.la') + make_cmds.append('ninja -C ' + build_dir + ' ext/compressors/' + name + '/all') +make_cmds.append('ninja -C ' + build_dir + ' libwiredtiger.a') +make_cmds.append('ninja -C ' + build_dir + ' lang/python/all') -inc_paths = [ os.path.join(build_dir, 'src', 'include'), build_dir, '.' ] -lib_paths = [ '.' ] # wiredtiger.so is moved into the top level directory +inc_paths = [ os.path.join(build_dir, 'include'), os.path.join(build_dir, 'config'), build_dir, '.' ] +lib_paths = [ '.' ] check_needed_dependencies(builtins, inc_paths, lib_paths) @@ -320,7 +249,7 @@ if pip_command == 'sdist': if python2: die('Python3 should be used to create a source distribution') - sources, movers = source_filter(get_sources_curdir()) + sources = get_sources_curdir() stage_dir = os.path.join(python_rel_dir, 'stage') shutil.rmtree(stage_dir, True) os.makedirs(stage_dir) @@ -329,23 +258,34 @@ if pip_command == 'sdist': d = os.path.join(stage_dir, os.path.dirname(f)) if not os.path.isdir(d): os.makedirs(d) - if f in movers: - src = movers[f] - else: - src = f # Symlinks are not followed in setup, we need to use real files. - shutil.copy2(src, os.path.join(stage_dir, f)) + shutil.copy2(f, os.path.join(stage_dir, f)) + # Copy files in lang/python/wiredtiger to the root folder. + pywt_path = os.path.join(stage_dir, "lang", "python", "wiredtiger") + pywt_files = os.listdir(pywt_path) + for f in pywt_files: + basename = os.path.basename(f) + src = os.path.join(pywt_path, f) + if basename == 'init.py': + shutil.copy2(src, os.path.join(stage_dir, '__init__.py')) + else: + shutil.copy2(src, os.path.join(stage_dir, basename)) os.chdir(stage_dir) sys.argv.append('--dist-dir=' + os.path.join('..', 'dist')) else: - sources = [ os.path.join(python_rel_dir, 'wiredtiger_wrap.c') ] + sources = [ os.path.join(conf_make_dir, python_rel_dir, 'CMakeFiles', '__wiredtiger.dir', 'wiredtigerPYTHON_wrap.c') ] wt_ext = Extension('_wiredtiger', sources = sources, extra_compile_args = cflags + cppflags, extra_link_args = ldflags, libraries = builtin_libraries, - extra_objects = [ os.path.join(build_dir, '.libs', 'libwiredtiger.a') ], + # FIXME-WT-7905: Remove manual linking of static extension libraries. + # Unfortunately CMake's use of the builtin doesn't currently support linking in the extension + # objects into a static libwiredtiger archive. As a workaround, we need to manually link + # the ext/compressor libraries. + extra_objects = [ os.path.join(build_dir, 'libwiredtiger.a') ] + \ + list(map(lambda name: os.path.join(build_dir, 'ext', 'compressors', name) + '/libwiredtiger_' + name + '.a', builtin_names)), include_dirs = inc_paths, library_dirs = lib_paths, ) @@ -353,7 +293,8 @@ extensions = [ wt_ext ] env = { "CFLAGS" : ' '.join(cflags), "CPPFLAGS" : ' '.join(cppflags), "LDFLAGS" : ' '.join(ldflags), - "PATH" : build_path } + "CMAKE_LIBRARY_PATH" : os.getenv("CMAKE_LIBRARY_PATH", default=""), + "PATH" : os.getenv("PATH", default="") } class BinaryDistribution(Distribution): def is_pure(self): @@ -362,6 +303,10 @@ class BinaryDistribution(Distribution): class WTInstall(setuptools.command.install.install): def run(self): self.run_command("build_ext") + # Copy the SWIG generated python module to our build directory. This will + # then subsequently be installed by pip into the package directory. + shutil.move(os.path.join(conf_make_dir, python_rel_dir, 'wiredtiger.py'), + os.path.join(self.build_lib, 'wiredtiger','swig_wiredtiger.py')) return setuptools.command.install.install.run(self) class WTBuildExt(setuptools.command.build_ext.build_ext): @@ -376,12 +321,12 @@ class WTBuildExt(setuptools.command.build_ext.build_ext): except OSError: pass self.execute( - lambda: build_commands(configure_cmds, conf_make_dir, env), [], + lambda: build_commands(configure_cmds, wt_dir, env), [], 'wiredtiger configure') if not os.path.isfile(makefile): die('configure failed, file does not exist: ' + makefile) self.execute( - lambda: build_commands(make_cmds, conf_make_dir, env), [], + lambda: build_commands(make_cmds, wt_dir, env), [], 'wiredtiger make') open(built_sentinal, 'a').close() return setuptools.command.build_ext.build_ext.run(self) @@ -404,7 +349,7 @@ setup( package_dir = { 'wiredtiger' : '.' }, cmdclass = { 'install': WTInstall, 'build_ext': WTBuildExt }, package_data = { - 'wiredtiger' : [ wt_swig_lib_name, '*.py' ] + 'wiredtiger' : [ '*.py'] }, classifiers=[ 'Intended Audience :: Developers', @@ -417,6 +362,10 @@ setup( 'Operating System :: POSIX :: BSD', 'Operating System :: POSIX :: Linux', 'Operating System :: POSIX :: SunOS/Solaris', + ], + install_requires=[ + 'cmake', + 'ninja', ] ) diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 96ad1432c87..32a6172cde6 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -116,7 +116,7 @@ new_page: __wt_upd_value_clear(cbt->upd_value); if (cbt->ins != NULL) restart_read: - WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins->upd, NULL)); + WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins->upd)); if (cbt->upd_value->type == WT_UPDATE_INVALID) { cbt->v = __bit_getv_recno(cbt->ref, cbt->recno, btree->bitcnt); cbt->iface.value.data = &cbt->v; @@ -211,6 +211,7 @@ __cursor_var_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skip return (WT_NOTFOUND); __cursor_set_recno(cbt, cbt->ref->ref_recno); cbt->cip_saved = NULL; + F_CLR(cbt, WT_CBT_CACHEABLE_RLE_CELL); goto new_page; } @@ -245,56 +246,84 @@ restart_read: } /* - * If we're at the same slot as the last reference and there's no matching insert list item, - * re-use the return information (so encoded items with large repeat counts aren't - * repeatedly decoded). Otherwise, unpack the cell and build the return information. + * There's no matching insert list item. If we're on the same slot as the last reference, + * and the cell is cacheable (it might not be if it's not globally visible), reuse the + * previous return data to avoid repeatedly decoding items. */ - if (cbt->cip_saved != cip) { - cell = WT_COL_PTR(page, cip); - __wt_cell_unpack_kv(session, page->dsk, cell, &unpack); - if (unpack.type == WT_CELL_DEL) { - if ((rle = __wt_cell_rle(&unpack)) == 1) { - ++*skippedp; - continue; - } - - /* - * There can be huge gaps in the variable-length column-store name space appearing - * as deleted records. If more than one deleted record, do the work of finding the - * next record to return instead of looping through the records. - * - * First, find the smallest record in the update list that's larger than the current - * record. - */ - ins = __col_insert_search_gt(cbt->ins_head, cbt->recno); - - /* - * Second, for records with RLEs greater than 1, the above call to __col_var_search - * located this record in the page's list of repeating records, and returned the - * starting record. The starting record plus the RLE is the record to which we could - * skip, if there was no smaller record in the update list. - */ - cbt->recno = rle_start + rle; - if (ins != NULL && WT_INSERT_RECNO(ins) < cbt->recno) - cbt->recno = WT_INSERT_RECNO(ins); + if (cbt->cip_saved == cip && F_ISSET(cbt, WT_CBT_CACHEABLE_RLE_CELL)) { + F_CLR(&cbt->iface, WT_CURSTD_VALUE_EXT); + F_SET(&cbt->iface, WT_CURSTD_VALUE_INT); + cbt->iface.value.data = cbt->tmp->data; + cbt->iface.value.size = cbt->tmp->size; + return (0); + } - /* Adjust for the outer loop increment. */ - --cbt->recno; + /* Unpack the cell and build the return information. */ + cell = WT_COL_PTR(page, cip); + __wt_cell_unpack_kv(session, page->dsk, cell, &unpack); + rle = __wt_cell_rle(&unpack); + if (unpack.type == WT_CELL_DEL) { + if (rle == 1) { ++*skippedp; continue; } - WT_RET(__wt_bt_col_var_cursor_walk_txn_read(session, cbt, page, &unpack, cip)); - if (cbt->upd_value->type == WT_UPDATE_INVALID || - cbt->upd_value->type == WT_UPDATE_TOMBSTONE) { - ++*skippedp; - continue; - } + /* + * There can be huge gaps in the variable-length column-store name space appearing as + * deleted records. If more than one deleted record, do the work of finding the next + * record to return instead of looping through the records. + * + * First, find the smallest record in the update list that's larger than the current + * record. + */ + ins = __col_insert_search_gt(cbt->ins_head, cbt->recno); - return (0); + /* + * Second, for records with RLEs greater than 1, the above call to __col_var_search + * located this record in the page's list of repeating records, and returned the + * starting record. The starting record plus the RLE is the record to which we could + * skip, if there was no smaller record in the update list. + */ + cbt->recno = rle_start + rle; + if (ins != NULL && WT_INSERT_RECNO(ins) < cbt->recno) + cbt->recno = WT_INSERT_RECNO(ins); + + /* Adjust for the outer loop increment. */ + --cbt->recno; + ++*skippedp; + continue; + } + + WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins ? cbt->ins->upd : NULL)); + if (cbt->upd_value->type == WT_UPDATE_INVALID || + cbt->upd_value->type == WT_UPDATE_TOMBSTONE) { + ++*skippedp; + continue; } - cbt->iface.value.data = cbt->tmp->data; - cbt->iface.value.size = cbt->tmp->size; + WT_RET(__wt_value_return(cbt, cbt->upd_value)); + + /* + * It is only safe to cache the value for other keys in the same RLE cell if it is globally + * visible. Otherwise, there might be some older timestamp where the value isn't uniform + * across the cell. Always set cip_saved so it's easy to tell when we change cells. + */ + cbt->cip_saved = cip; + if (rle > 1 && + __wt_txn_visible_all(session, unpack.tw.start_txn, unpack.tw.durable_start_ts)) { + /* + * Copy the value into cbt->tmp to cache it. This is perhaps unfortunate, because + * copying isn't free, but it's currently necessary. The memory we're copying might be + * on the disk page (which is safe because the page is pinned as long as the cursor is + * sitting on it) but if not it belongs to cbt->upd_value, and that (though it belongs + * to the cursor and won't disappear arbitrarily) might be invalidated or changed by + * other paths through this function on a subsequent call but before we are done with + * this RLE cell. In principle those paths could clear WT_CBT_CACHEABLE_RLE_CELL, but + * the code is currently structured in a way that makes that difficult. + */ + WT_RET(__wt_buf_set(session, cbt->tmp, cbt->iface.value.data, cbt->iface.value.size)); + F_SET(cbt, WT_CBT_CACHEABLE_RLE_CELL); + } else + F_CLR(cbt, WT_CBT_CACHEABLE_RLE_CELL); return (0); } /* NOTREACHED */ @@ -413,8 +442,8 @@ restart_read_page: WT_STAT_CONN_DATA_INCR(session, cursor_search_near_prefix_fast_paths); return (WT_NOTFOUND); } - WT_RET(__wt_txn_read( - session, cbt, &cbt->iface.key, WT_RECNO_OOB, WT_ROW_UPDATE(page, rip), NULL)); + WT_RET( + __wt_txn_read(session, cbt, &cbt->iface.key, WT_RECNO_OOB, WT_ROW_UPDATE(page, rip))); if (cbt->upd_value->type == WT_UPDATE_INVALID) { ++*skippedp; continue; @@ -590,6 +619,7 @@ __wt_btcur_iterate_setup(WT_CURSOR_BTREE *cbt) /* Clear saved iteration cursor position information. */ cbt->cip_saved = NULL; cbt->rip_saved = NULL; + F_CLR(cbt, WT_CBT_CACHEABLE_RLE_CELL); /* * If we don't have a search page, then we're done, we're starting at the beginning or end of diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index 5eb2dbdbc25..bcf9f7e6d5a 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -256,7 +256,7 @@ new_page: __wt_upd_value_clear(cbt->upd_value); if (cbt->ins != NULL) restart_read: - WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins->upd, NULL)); + WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins->upd)); if (cbt->upd_value->type == WT_UPDATE_INVALID) { cbt->v = __bit_getv_recno(cbt->ref, cbt->recno, btree->bitcnt); cbt->iface.value.data = &cbt->v; @@ -327,7 +327,7 @@ __cursor_var_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skip WT_INSERT *ins; WT_PAGE *page; WT_SESSION_IMPL *session; - uint64_t rle_start; + uint64_t rle, rle_start; session = CUR2S(cbt); page = cbt->ref->page; @@ -350,6 +350,7 @@ __cursor_var_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skip return (WT_NOTFOUND); __cursor_set_recno(cbt, cbt->last_standard_recno); cbt->cip_saved = NULL; + F_CLR(cbt, WT_CBT_CACHEABLE_RLE_CELL); goto new_page; } @@ -385,55 +386,84 @@ restart_read: } /* - * If we're at the same slot as the last reference and there's no matching insert list item, - * re-use the return information (so encoded items with large repeat counts aren't - * repeatedly decoded). Otherwise, unpack the cell and build the return information. + * There's no matching insert list item. If we're on the same slot as the last reference, + * and the cell is cacheable (it might not be if it's not globally visible), reuse the + * previous return data to avoid repeatedly decoding items. */ - if (cbt->cip_saved != cip) { - cell = WT_COL_PTR(page, cip); - __wt_cell_unpack_kv(session, page->dsk, cell, &unpack); - if (unpack.type == WT_CELL_DEL) { - if (__wt_cell_rle(&unpack) == 1) { - ++*skippedp; - continue; - } - - /* - * There can be huge gaps in the variable-length column-store name space appearing - * as deleted records. If more than one deleted record, do the work of finding the - * next record to return instead of looping through the records. - * - * First, find the largest record in the update list that's smaller than the current - * record. - */ - ins = __col_insert_search_lt(cbt->ins_head, cbt->recno); - - /* - * Second, for records with RLEs greater than 1, the above call to __col_var_search - * located this record in the page's list of repeating records, and returned the - * starting record. The starting record - 1 is the record to which we could skip, if - * there was no larger record in the update list. - */ - cbt->recno = rle_start - 1; - if (ins != NULL && WT_INSERT_RECNO(ins) > cbt->recno) - cbt->recno = WT_INSERT_RECNO(ins); + if (cbt->cip_saved == cip && F_ISSET(cbt, WT_CBT_CACHEABLE_RLE_CELL)) { + F_CLR(&cbt->iface, WT_CURSTD_VALUE_EXT); + F_SET(&cbt->iface, WT_CURSTD_VALUE_INT); + cbt->iface.value.data = cbt->tmp->data; + cbt->iface.value.size = cbt->tmp->size; + return (0); + } - /* Adjust for the outer loop decrement. */ - ++cbt->recno; + /* Unpack the cell and build the return information. */ + cell = WT_COL_PTR(page, cip); + __wt_cell_unpack_kv(session, page->dsk, cell, &unpack); + rle = __wt_cell_rle(&unpack); + if (unpack.type == WT_CELL_DEL) { + if (rle == 1) { ++*skippedp; continue; } - WT_RET(__wt_bt_col_var_cursor_walk_txn_read(session, cbt, page, &unpack, cip)); - if (cbt->upd_value->type == WT_UPDATE_INVALID || - cbt->upd_value->type == WT_UPDATE_TOMBSTONE) { - ++*skippedp; - continue; - } - return (0); + /* + * There can be huge gaps in the variable-length column-store name space appearing as + * deleted records. If more than one deleted record, do the work of finding the next + * record to return instead of looping through the records. + * + * First, find the largest record in the update list that's smaller than the current + * record. + */ + ins = __col_insert_search_lt(cbt->ins_head, cbt->recno); + + /* + * Second, for records with RLEs greater than 1, the above call to __col_var_search + * located this record in the page's list of repeating records, and returned the + * starting record. The starting record - 1 is the record to which we could skip, if + * there was no larger record in the update list. + */ + cbt->recno = rle_start - 1; + if (ins != NULL && WT_INSERT_RECNO(ins) > cbt->recno) + cbt->recno = WT_INSERT_RECNO(ins); + + /* Adjust for the outer loop decrement. */ + ++cbt->recno; + ++*skippedp; + continue; + } + + WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins ? cbt->ins->upd : NULL)); + if (cbt->upd_value->type == WT_UPDATE_INVALID || + cbt->upd_value->type == WT_UPDATE_TOMBSTONE) { + ++*skippedp; + continue; } - cbt->iface.value.data = cbt->tmp->data; - cbt->iface.value.size = cbt->tmp->size; + WT_RET(__wt_value_return(cbt, cbt->upd_value)); + + /* + * It is only safe to cache the value for other keys in the same RLE cell if it is globally + * visible. Otherwise, there might be some older timestamp where the value isn't uniform + * across the cell. Always set cip_saved so it's easy to tell when we change cells. + */ + cbt->cip_saved = cip; + if (rle > 1 && + __wt_txn_visible_all(session, unpack.tw.start_txn, unpack.tw.durable_start_ts)) { + /* + * Copy the value into cbt->tmp to cache it. This is perhaps unfortunate, because + * copying isn't free, but it's currently necessary. The memory we're copying might be + * on the disk page (which is safe because the page is pinned as long as the cursor is + * sitting on it) but if not it belongs to cbt->upd_value, and that (though it belongs + * to the cursor and won't disappear arbitrarily) might be invalidated or changed by + * other paths through this function on a subsequent call but before we are done with + * this RLE cell. In principle those paths could clear WT_CBT_CACHEABLE_RLE_CELL, but + * the code is currently structured in a way that makes that difficult. + */ + WT_RET(__wt_buf_set(session, cbt->tmp, cbt->iface.value.data, cbt->iface.value.size)); + F_SET(cbt, WT_CBT_CACHEABLE_RLE_CELL); + } else + F_CLR(cbt, WT_CBT_CACHEABLE_RLE_CELL); return (0); } /* NOTREACHED */ @@ -561,8 +591,8 @@ restart_read_page: WT_STAT_CONN_DATA_INCR(session, cursor_search_near_prefix_fast_paths); return (WT_NOTFOUND); } - WT_RET(__wt_txn_read( - session, cbt, &cbt->iface.key, WT_RECNO_OOB, WT_ROW_UPDATE(page, rip), NULL)); + WT_RET( + __wt_txn_read(session, cbt, &cbt->iface.key, WT_RECNO_OOB, WT_ROW_UPDATE(page, rip))); if (cbt->upd_value->type == WT_UPDATE_INVALID) { ++*skippedp; continue; diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 46da608e096..ef8e42f97e5 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -297,7 +297,7 @@ __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint64_t recno, bool *vali * Check for an update ondisk or in the history store. For column store, an insert object * can have the same key as an on-page or history store object. */ - WT_RET(__wt_txn_read(session, cbt, key, recno, NULL, NULL)); + WT_RET(__wt_txn_read(session, cbt, key, recno, cbt->ins ? cbt->ins->upd : NULL)); if (cbt->upd_value->type != WT_UPDATE_INVALID) { if (cbt->upd_value->type == WT_UPDATE_TOMBSTONE) return (0); @@ -325,8 +325,7 @@ __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint64_t recno, bool *vali WT_RET(__wt_txn_read(session, cbt, key, WT_RECNO_OOB, (page->modify != NULL && page->modify->mod_row_update != NULL) ? page->modify->mod_row_update[cbt->slot] : - NULL, - NULL)); + NULL)); if (cbt->upd_value->type != WT_UPDATE_INVALID) { if (cbt->upd_value->type == WT_UPDATE_TOMBSTONE) return (0); diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index fdbb192774c..d5251268944 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -655,7 +655,7 @@ __wt_btree_tree_open(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_ * the disk image on return, the in-memory object steals it. */ WT_ERR(__wt_page_inmem(session, NULL, dsk.data, - WT_DATA_IN_ITEM(&dsk) ? WT_PAGE_DISK_ALLOC : WT_PAGE_DISK_MAPPED, &page)); + WT_DATA_IN_ITEM(&dsk) ? WT_PAGE_DISK_ALLOC : WT_PAGE_DISK_MAPPED, &page, NULL)); dsk.mem = NULL; /* Finish initializing the root, root reference links. */ diff --git a/src/third_party/wiredtiger/src/btree/bt_page.c b/src/third_party/wiredtiger/src/btree/bt_page.c index 22766b682bf..d37b3cdb204 100644 --- a/src/third_party/wiredtiger/src/btree/bt_page.c +++ b/src/third_party/wiredtiger/src/btree/bt_page.c @@ -10,9 +10,9 @@ static void __inmem_col_fix(WT_SESSION_IMPL *, WT_PAGE *); static void __inmem_col_int(WT_SESSION_IMPL *, WT_PAGE *); -static int __inmem_col_var(WT_SESSION_IMPL *, WT_PAGE *, uint64_t, size_t *); +static int __inmem_col_var(WT_SESSION_IMPL *, WT_PAGE *, uint64_t, bool *, size_t *); static int __inmem_row_int(WT_SESSION_IMPL *, WT_PAGE *, size_t *); -static int __inmem_row_leaf(WT_SESSION_IMPL *, WT_PAGE *); +static int __inmem_row_leaf(WT_SESSION_IMPL *, WT_PAGE *, bool *); static int __inmem_row_leaf_entries(WT_SESSION_IMPL *, const WT_PAGE_HEADER *, uint32_t *); /* @@ -121,12 +121,184 @@ err: } /* + * __page_inmem_prepare_update -- + * Create the actual update for a prepared value. + */ +static int +__page_inmem_prepare_update(WT_SESSION_IMPL *session, WT_ITEM *value, WT_CELL_UNPACK_KV *unpack, + WT_UPDATE **updp, size_t *sizep) +{ + WT_DECL_RET; + WT_UPDATE *upd, *tombstone; + size_t size, total_size; + + *sizep = 0; + + tombstone = upd = NULL; + total_size = 0; + + WT_RET(__wt_upd_alloc(session, value, WT_UPDATE_STANDARD, &upd, &size)); + total_size += size; + upd->durable_ts = unpack->tw.durable_start_ts; + upd->start_ts = unpack->tw.start_ts; + upd->txnid = unpack->tw.start_txn; + F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS); + + /* + * Instantiate both update and tombstone if the prepared update is a tombstone. This is required + * to ensure that written prepared delete operation must be removed from the data store, when + * the prepared transaction gets rollback. + */ + if (WT_TIME_WINDOW_HAS_STOP(&unpack->tw)) { + WT_ERR(__wt_upd_alloc_tombstone(session, &tombstone, &size)); + total_size += size; + tombstone->durable_ts = WT_TS_NONE; + tombstone->start_ts = unpack->tw.stop_ts; + tombstone->txnid = unpack->tw.stop_txn; + tombstone->prepare_state = WT_PREPARE_INPROGRESS; + F_SET(tombstone, WT_UPDATE_PREPARE_RESTORED_FROM_DS); + + /* + * Mark the update also as in-progress if the update and tombstone are from same transaction + * by comparing both the transaction and timestamps as the transaction information gets lost + * after restart. + */ + if (unpack->tw.start_ts == unpack->tw.stop_ts && + unpack->tw.durable_start_ts == unpack->tw.durable_stop_ts && + unpack->tw.start_txn == unpack->tw.stop_txn) { + upd->durable_ts = WT_TS_NONE; + upd->prepare_state = WT_PREPARE_INPROGRESS; + } + + tombstone->next = upd; + *updp = tombstone; + } else { + upd->durable_ts = WT_TS_NONE; + upd->prepare_state = WT_PREPARE_INPROGRESS; + *updp = upd; + } + + *sizep = total_size; + return (0); + +err: + __wt_free(session, upd); + __wt_free(session, tombstone); + + return (ret); +} + +/* + * __wt_page_inmem_prepare -- + * Instantiate prepared updates. + */ +int +__wt_page_inmem_prepare(WT_SESSION_IMPL *session, WT_REF *ref) +{ + WT_CELL *cell; + WT_CELL_UNPACK_KV unpack; + WT_COL *cip; + WT_CURSOR_BTREE cbt; + WT_DECL_ITEM(key); + WT_DECL_ITEM(value); + WT_DECL_RET; + WT_PAGE *page; + WT_ROW *rip; + WT_UPDATE *upd; + size_t size, total_size; + uint64_t recno, rle; + uint32_t i; + + page = ref->page; + upd = NULL; + total_size = 0; + + /* + * We don't handle in-memory prepare resolution here, and prepare only applies to row-store and + * variable-length column store leaf pages. + */ + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_IN_MEMORY)); + WT_ASSERT(session, page->type == WT_PAGE_COL_VAR || page->type == WT_PAGE_ROW_LEAF); + + __wt_btcur_init(session, &cbt); + __wt_btcur_open(&cbt); + + WT_ERR(__wt_scr_alloc(session, 0, &value)); + if (page->type == WT_PAGE_COL_VAR) { + recno = ref->ref_recno; + WT_COL_FOREACH (page, cip, i) { + /* Search for prepare records. */ + cell = WT_COL_PTR(page, cip); + __wt_cell_unpack_kv(session, page->dsk, cell, &unpack); + rle = __wt_cell_rle(&unpack); + if (!unpack.tw.prepare) { + recno += rle; + continue; + } + + /* Get the value. */ + WT_ERR(__wt_page_cell_data_ref(session, page, &unpack, value)); + + /* For each record, create an update to resolve the prepare. */ + for (; rle > 0; --rle, ++recno) { + /* Create an update to resolve the prepare. */ + WT_ERR(__page_inmem_prepare_update(session, value, &unpack, &upd, &size)); + total_size += size; + + /* Search the page and apply the modification. */ + WT_ERR(__wt_col_search(&cbt, recno, ref, true, NULL)); +#ifdef HAVE_DIAGNOSTIC + WT_ERR(__wt_col_modify(&cbt, recno, NULL, upd, WT_UPDATE_INVALID, true, true)); +#else + WT_ERR(__wt_col_modify(&cbt, recno, NULL, upd, WT_UPDATE_INVALID, true)); +#endif + upd = NULL; + } + } + } else { + WT_ERR(__wt_scr_alloc(session, 0, &key)); + WT_ROW_FOREACH (page, rip, i) { + /* Search for prepare records. */ + __wt_row_leaf_value_cell(session, page, rip, &unpack); + if (!unpack.tw.prepare) + continue; + + /* Get the key/value pair and create an update to resolve the prepare. */ + WT_ERR(__wt_row_leaf_key(session, page, rip, key, false)); + WT_ERR(__wt_page_cell_data_ref(session, page, &unpack, value)); + WT_ERR(__page_inmem_prepare_update(session, value, &unpack, &upd, &size)); + total_size += size; + + /* Search the page and apply the modification. */ + WT_ERR(__wt_row_search(&cbt, key, true, ref, true, NULL)); +#ifdef HAVE_DIAGNOSTIC + WT_ERR(__wt_row_modify(&cbt, key, NULL, upd, WT_UPDATE_INVALID, true, true)); +#else + WT_ERR(__wt_row_modify(&cbt, key, NULL, upd, WT_UPDATE_INVALID, true)); +#endif + upd = NULL; + } + } + + __wt_cache_page_inmem_incr(session, page, total_size); + + if (0) { +err: + __wt_free_update_list(session, &upd); + } + WT_TRET(__wt_btcur_close(&cbt, true)); + __wt_scr_free(session, &key); + __wt_scr_free(session, &value); + return (ret); +} + +/* * __wt_page_inmem -- * Build in-memory page information. */ int -__wt_page_inmem( - WT_SESSION_IMPL *session, WT_REF *ref, const void *image, uint32_t flags, WT_PAGE **pagep) +__wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, const void *image, uint32_t flags, + WT_PAGE **pagep, bool *preparedp) { WT_DECL_RET; WT_PAGE *page; @@ -136,6 +308,9 @@ __wt_page_inmem( *pagep = NULL; + if (preparedp != NULL) + *preparedp = false; + dsk = image; alloc_entries = 0; @@ -206,13 +381,13 @@ __wt_page_inmem( __inmem_col_int(session, page); break; case WT_PAGE_COL_VAR: - WT_ERR(__inmem_col_var(session, page, dsk->recno, &size)); + WT_ERR(__inmem_col_var(session, page, dsk->recno, preparedp, &size)); break; case WT_PAGE_ROW_INT: WT_ERR(__inmem_row_int(session, page, &size)); break; case WT_PAGE_ROW_LEAF: - WT_ERR(__inmem_row_leaf(session, page)); + WT_ERR(__inmem_row_leaf(session, page, preparedp)); break; default: WT_ERR(__wt_illegal_value(session, page->type)); @@ -314,7 +489,8 @@ __inmem_col_var_repeats(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t *np) * Build in-memory index for variable-length, data-only leaf pages in column-store trees. */ static int -__inmem_col_var(WT_SESSION_IMPL *session, WT_PAGE *page, uint64_t recno, size_t *sizep) +__inmem_col_var( + WT_SESSION_IMPL *session, WT_PAGE *page, uint64_t recno, bool *preparedp, size_t *sizep) { WT_CELL_UNPACK_KV unpack; WT_COL *cip; @@ -322,10 +498,12 @@ __inmem_col_var(WT_SESSION_IMPL *session, WT_PAGE *page, uint64_t recno, size_t size_t size; uint64_t rle; uint32_t indx, n, repeat_off; + bool prepare; void *p; repeats = NULL; repeat_off = 0; + prepare = false; /* * Walk the page, building references: the page contains unsorted value items. The value items @@ -359,11 +537,19 @@ __inmem_col_var(WT_SESSION_IMPL *session, WT_PAGE *page, uint64_t recno, size_t repeats[repeat_off].recno = recno; repeats[repeat_off++].rle = rle; } + + /* If we find a prepare, we'll have to instantiate it in the update chain later. */ + if (unpack.tw.prepare) + prepare = true; + indx++; recno += rle; } WT_CELL_FOREACH_END; + if (preparedp != NULL && prepare) + *preparedp = true; + return (0); } @@ -529,24 +715,20 @@ __inmem_row_leaf_entries(WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, ui * Build in-memory index for row-store leaf pages. */ static int -__inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) +__inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page, bool *preparedp) { - enum { PREPARE_INSTANTIATE, PREPARE_INITIALIZED, PREPARE_IGNORE } prepare; WT_BTREE *btree; WT_CELL_UNPACK_KV unpack; - WT_DECL_ITEM(value); WT_DECL_RET; WT_ROW *rip; - WT_UPDATE *tombstone, *upd; - size_t size, total_size; uint32_t best_prefix_count, best_prefix_start, best_prefix_stop; uint32_t last_slot, prefix_count, prefix_start, prefix_stop, slot; uint8_t smallest_prefix; + bool prepare; btree = S2BT(session); - tombstone = upd = NULL; last_slot = 0; - size = total_size = 0; + prepare = false; /* The code depends on the prefix count variables, other initialization shouldn't matter. */ best_prefix_count = prefix_count = 0; @@ -554,15 +736,6 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) prefix_start = prefix_stop = 0; /* [-Wconditional-uninitialized] */ best_prefix_start = best_prefix_stop = 0; /* [-Wconditional-uninitialized] */ - /* - * Optionally instantiate prepared updates. In-memory databases restore non-obsolete updates on - * the page as part of the __split_multi_inmem function. - */ - prepare = F_ISSET(session, WT_SESSION_INSTANTIATE_PREPARE) && - !F_ISSET(S2C(session), WT_CONN_IN_MEMORY) ? - PREPARE_INSTANTIATE : - PREPARE_IGNORE; - /* Walk the page, building indices. */ rip = page->pg_row; WT_CELL_FOREACH_KV (session, page->dsk, unpack) { @@ -657,72 +830,9 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) WT_ERR(__wt_illegal_value(session, unpack.type)); } - if (!unpack.tw.prepare || prepare == PREPARE_IGNORE) - continue; - - /* First prepared transaction setup. */ - if (prepare == PREPARE_INSTANTIATE) { - WT_ERR(__wt_page_modify_init(session, page)); - if (!F_ISSET(btree, WT_BTREE_READONLY)) - __wt_page_modify_set(session, page); - - /* Allocate the per-page update array. */ - WT_ERR(__wt_calloc_def(session, page->entries, &page->modify->mod_row_update)); - total_size += page->entries * sizeof(*page->modify->mod_row_update); - - WT_ERR(__wt_scr_alloc(session, 0, &value)); - - prepare = PREPARE_INITIALIZED; - } - - /* Make sure that there is no in-memory update for this key. */ - WT_ASSERT(session, page->modify->mod_row_update[WT_ROW_SLOT(page, rip - 1)] == NULL); - - /* Take the value from the page cell. */ - WT_ERR(__wt_page_cell_data_ref(session, page, &unpack, value)); - - WT_ERR(__wt_upd_alloc(session, value, WT_UPDATE_STANDARD, &upd, &size)); - total_size += size; - upd->durable_ts = unpack.tw.durable_start_ts; - upd->start_ts = unpack.tw.start_ts; - upd->txnid = unpack.tw.start_txn; - F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS); - - /* - * Instantiate both update and tombstone if the prepared update is a tombstone. This is - * required to ensure that written prepared delete operation must be removed from the data - * store, when the prepared transaction gets rollback. - */ - if (WT_TIME_WINDOW_HAS_STOP(&unpack.tw)) { - WT_ERR(__wt_upd_alloc_tombstone(session, &tombstone, &size)); - total_size += size; - tombstone->durable_ts = WT_TS_NONE; - tombstone->start_ts = unpack.tw.stop_ts; - tombstone->txnid = unpack.tw.stop_txn; - tombstone->prepare_state = WT_PREPARE_INPROGRESS; - F_SET(tombstone, WT_UPDATE_PREPARE_RESTORED_FROM_DS); - - /* - * Mark the update also as in-progress if the update and tombstone are from same - * transaction by comparing both the transaction and timestamps as the transaction - * information gets lost after restart. - */ - if (unpack.tw.start_ts == unpack.tw.stop_ts && - unpack.tw.durable_start_ts == unpack.tw.durable_stop_ts && - unpack.tw.start_txn == unpack.tw.stop_txn) { - upd->durable_ts = WT_TS_NONE; - upd->prepare_state = WT_PREPARE_INPROGRESS; - } - - tombstone->next = upd; - } else { - upd->durable_ts = WT_TS_NONE; - upd->prepare_state = WT_PREPARE_INPROGRESS; - tombstone = upd; - } - - page->modify->mod_row_update[WT_ROW_SLOT(page, rip - 1)] = tombstone; - tombstone = upd = NULL; + /* If we find a prepare, we'll have to instantiate it in the update chain later. */ + if (unpack.tw.prepare) + prepare = true; } WT_CELL_FOREACH_END; @@ -743,12 +853,9 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) if (best_prefix_count <= 10) F_SET_ATOMIC(page, WT_PAGE_BUILD_KEYS); - __wt_cache_page_inmem_incr(session, page, total_size); + if (preparedp != NULL && prepare) + *preparedp = true; err: - __wt_free(session, tombstone); - __wt_free(session, upd); - __wt_scr_free(session, &value); - return (ret); } diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index dc070d5c700..687c21ec086 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -94,7 +94,7 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) uint64_t time_diff, time_start, time_stop; uint32_t page_flags; uint8_t previous_state; - bool timer; + bool prepare, timer; time_start = time_stop = 0; @@ -159,11 +159,10 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) page_flags = WT_DATA_IN_ITEM(&tmp) ? WT_PAGE_DISK_ALLOC : WT_PAGE_DISK_MAPPED; if (LF_ISSET(WT_READ_IGNORE_CACHE_SIZE)) FLD_SET(page_flags, WT_PAGE_EVICT_NO_PROGRESS); - F_SET(session, WT_SESSION_INSTANTIATE_PREPARE); - ret = __wt_page_inmem(session, ref, tmp.data, page_flags, ¬used); - F_CLR(session, WT_SESSION_INSTANTIATE_PREPARE); - WT_ERR(ret); + WT_ERR(__wt_page_inmem(session, ref, tmp.data, page_flags, ¬used, &prepare)); tmp.mem = NULL; + if (prepare) + WT_ERR(__wt_page_inmem_prepare(session, ref)); skip_read: switch (previous_state) { diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index 7db2b39d486..8cb00cbde81 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -639,7 +639,7 @@ __slvg_trk_leaf(WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, uint8_t *ad * Page flags are 0 because we aren't releasing the memory used to read the page into memory * and we don't want page discard to free it. */ - WT_ERR(__wt_page_inmem(session, NULL, dsk, 0, &page)); + WT_ERR(__wt_page_inmem(session, NULL, dsk, 0, &page, NULL)); WT_ERR(__wt_row_leaf_key_copy(session, page, &page->pg_row[0], &trk->row_start)); WT_ERR( __wt_row_leaf_key_copy(session, page, &page->pg_row[page->entries - 1], &trk->row_stop)); @@ -1691,7 +1691,7 @@ __slvg_row_trk_update_start(WT_SESSION_IMPL *session, WT_ITEM *stop, uint32_t sl */ WT_RET(__wt_scr_alloc(session, trk->trk_size, &dsk)); WT_ERR(__wt_bt_read(session, dsk, trk->trk_addr, trk->trk_addr_size)); - WT_ERR(__wt_page_inmem(session, NULL, dsk->data, 0, &page)); + WT_ERR(__wt_page_inmem(session, NULL, dsk->data, 0, &page, NULL)); /* * Walk the page, looking for a key sorting greater than the specified stop key -- that's our diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index e6ca5c7118b..f7ca3b9f080 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -1360,6 +1360,7 @@ __split_multi_inmem(WT_SESSION_IMPL *session, WT_PAGE *orig, WT_MULTI *multi, WT WT_UPDATE *prev_onpage, *upd; uint64_t recno; uint32_t i, slot; + bool prepare; /* * In 04/2016, we removed column-store record numbers from the WT_PAGE structure, leading to @@ -1381,13 +1382,17 @@ __split_multi_inmem(WT_SESSION_IMPL *session, WT_PAGE *orig, WT_MULTI *multi, WT * our caller will not discard the disk image when discarding the original page, and our caller * will discard the allocated page on error, when discarding the allocated WT_REF. */ - F_SET(session, WT_SESSION_INSTANTIATE_PREPARE); - ret = __wt_page_inmem(session, ref, multi->disk_image, WT_PAGE_DISK_ALLOC, &page); - F_CLR(session, WT_SESSION_INSTANTIATE_PREPARE); - WT_RET(ret); + WT_RET(__wt_page_inmem(session, ref, multi->disk_image, WT_PAGE_DISK_ALLOC, &page, &prepare)); multi->disk_image = NULL; /* + * In-memory databases restore non-obsolete updates directly in this function, don't call the + * underlying page functions to do it. + */ + if (prepare && !F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) + WT_RET(__wt_page_inmem_prepare(session, ref)); + + /* * Put the re-instantiated page in the same LRU queue location as the original page, unless this * was a forced eviction, in which case we leave the new page with the read generation unset. * Eviction will set the read generation next time it visits this page. diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index ec7b66b1012..632526f83df 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -31,7 +31,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U WT_PAGE *page; WT_PAGE_MODIFY *mod; WT_SESSION_IMPL *session; - WT_UPDATE *old_upd, *upd; + WT_UPDATE *last_upd, *old_upd, *upd; wt_timestamp_t prev_upd_ts; size_t ins_size, upd_size; u_int i, skipdepth; @@ -141,19 +141,33 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U #endif WT_ERR(__wt_txn_modify(session, upd)); logged = true; + + /* Avoid a data copy in WT_CURSOR.update. */ + __wt_upd_value_assign(cbt->modify_update, upd); } else { - upd = upd_arg; - upd_size = WT_UPDATE_MEMSIZE(upd); - } + upd_size = __wt_update_list_memsize(upd); - /* Avoid a data copy in WT_CURSOR.update. */ - __wt_upd_value_assign(cbt->modify_update, upd); + /* If there are existing updates, append them after the new updates. */ + for (last_upd = upd; last_upd->next != NULL; last_upd = last_upd->next) + ; + last_upd->next = old_upd; - /* - * If we restore an update chain in update restore eviction, there should be no update on - * the existing update chain. - */ - WT_ASSERT(session, !restore || old_upd == NULL); + /* + * If we restore an update chain in update restore eviction, there should be no update + * on the existing update chain. + */ + WT_ASSERT(session, !restore || old_upd == NULL); + + /* + * We can either put multiple new updates or a single update on the update chain. + * + * Set the "old" entry to the second update in the list so that the serialization + * function succeeds in swapping the first update into place. + */ + if (upd->next != NULL) + cbt->ins->upd = upd->next; + old_upd = cbt->ins->upd; + } /* * Point the new WT_UPDATE item to the next element in the list. If we get it right, the diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index 5ecbd33fe3b..c96b963cb1f 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -206,7 +206,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, WT_ERR(__wt_txn_modify(session, upd)); logged = true; - /* Avoid WT_CURSOR.update data copy. */ + /* Avoid a data copy in WT_CURSOR.update. */ __wt_upd_value_assign(cbt->modify_update, upd); } else { /* diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 168c3824d8e..693bf9aa554 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -227,8 +227,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_commit_transaction[] = { {"commit_timestamp", "string", NULL, NULL, NULL, 0}, {"durable_timestamp", "string", NULL, NULL, NULL, 0}, {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, - {"sync", "string", NULL, "choices=[\"background\",\"off\",\"on\"]", NULL, 0}, - {NULL, NULL, NULL, NULL, NULL, 0}}; + {"sync", "string", NULL, "choices=[\"off\",\"on\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_SESSION_compact[] = { {"timeout", "int", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -343,8 +342,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_join[] = { {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_SESSION_log_flush[] = { - {"sync", "string", NULL, "choices=[\"background\",\"off\",\"on\"]", NULL, 0}, - {NULL, NULL, NULL, NULL, NULL, 0}}; + {"sync", "string", NULL, "choices=[\"off\",\"on\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor_debug_subconfigs[] = { {"release_evict", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -405,9 +403,6 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_timestamp_transaction[] = { {"prepare_timestamp", "string", NULL, NULL, NULL, 0}, {"read_timestamp", "string", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; -static const WT_CONFIG_CHECK confchk_WT_SESSION_transaction_sync[] = { - {"timeout_ms", "int", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; - static const WT_CONFIG_CHECK confchk_WT_SESSION_verify[] = { {"dump_address", "boolean", NULL, NULL, NULL, 0}, {"dump_blocks", "boolean", NULL, NULL, NULL, 0}, {"dump_layout", "boolean", NULL, NULL, NULL, 0}, {"dump_offsets", "list", NULL, NULL, NULL, 0}, @@ -1244,7 +1239,6 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "commit_timestamp=,durable_timestamp=,prepare_timestamp=," "read_timestamp=", confchk_WT_SESSION_timestamp_transaction, 4}, - {"WT_SESSION.transaction_sync", "timeout_ms=1200000", confchk_WT_SESSION_transaction_sync, 1}, {"WT_SESSION.truncate", "", NULL, 0}, {"WT_SESSION.upgrade", "", NULL, 0}, {"WT_SESSION.verify", "dump_address=false,dump_blocks=false,dump_layout=false," diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index 8012c5d84fc..e4b3c5e981d 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -555,9 +555,8 @@ __log_file_server(void *arg) WT_DECL_RET; WT_FH *close_fh; WT_LOG *log; - WT_LSN close_end_lsn, min_lsn; + WT_LSN close_end_lsn; WT_SESSION_IMPL *session; - uint64_t yield_count; uint32_t filenum; bool locked; @@ -565,7 +564,6 @@ __log_file_server(void *arg) conn = S2C(session); log = conn->log; locked = false; - yield_count = 0; while (FLD_ISSET(conn->server_flags, WT_CONN_SERVER_LOG)) { /* * If there is a log file to close, make sure any outstanding write operations have @@ -617,54 +615,6 @@ __log_file_server(void *arg) __wt_spin_unlock(session, &log->log_sync_lock); } } - /* - * If a later thread asked for a background sync, do it now. - */ - if (__wt_log_cmp(&log->bg_sync_lsn, &log->sync_lsn) > 0) { - /* - * Save the latest write LSN which is the minimum we will have written to disk. - */ - WT_ASSIGN_LSN(&min_lsn, &log->write_lsn); - /* - * We have to wait until the LSN we asked for is written. If it isn't signal the wrlsn - * thread to get it written. - * - * We also have to wait for the written LSN and the sync LSN to be in the same file so - * that we know we have synchronized all earlier log files. - */ - if (__wt_log_cmp(&log->bg_sync_lsn, &min_lsn) <= 0) { - /* - * If the sync file is behind either the one wanted for a background sync or the - * write LSN has moved to another file continue to let this worker thread process - * that older file immediately. - */ - if ((log->sync_lsn.l.file < log->bg_sync_lsn.l.file) || - (log->sync_lsn.l.file < min_lsn.l.file)) - continue; - WT_ERR(__wt_fsync(session, log->log_fh, true)); - __wt_spin_lock(session, &log->log_sync_lock); - WT_NOT_READ(locked, true); - /* - * The sync LSN could have advanced while we were writing to disk. - */ - if (__wt_log_cmp(&log->sync_lsn, &min_lsn) <= 0) { - WT_ASSERT(session, min_lsn.l.file == log->sync_lsn.l.file); - WT_ASSIGN_LSN(&log->sync_lsn, &min_lsn); - __wt_cond_signal(session, log->log_sync_cond); - } - locked = false; - __wt_spin_unlock(session, &log->log_sync_lock); - } else { - __wt_cond_signal(session, conn->log_wrlsn_cond); - /* - * We do not want to wait potentially a second to process this. Yield to give the - * wrlsn thread a chance to run and try again in this case. - */ - yield_count++; - __wt_yield(); - continue; - } - } /* Wait until the next event. */ __wt_cond_wait(session, conn->log_file_cond, 100000, NULL); @@ -674,7 +624,6 @@ __log_file_server(void *arg) err: WT_IGNORE_RET(__wt_panic(session, ret, "log close server error")); } - WT_STAT_CONN_INCRV(session, log_server_sync_blocked, yield_count); if (locked) __wt_spin_unlock(session, &log->log_sync_lock); return (WT_THREAD_RET_VALUE); diff --git a/src/third_party/wiredtiger/src/docs/Doxyfile b/src/third_party/wiredtiger/src/docs/Doxyfile index 8bd05ca0bfa..cfef83d0445 100644 --- a/src/third_party/wiredtiger/src/docs/Doxyfile +++ b/src/third_party/wiredtiger/src/docs/Doxyfile @@ -297,6 +297,7 @@ ALIASES = "arch_page_table{2}=<div class= arch_head><table><tr><t "row{7}=<tr><td>\1</td><td>\2</td><td>\3</td><td>\4</td><td>\5</td><td>\6</td><td>\7</td></tr>" \ "row{8}=<tr><td>\1</td><td>\2</td><td>\3</td><td>\4</td><td>\5</td><td>\6</td><td>\7</td><td>\8</td></tr>" \ "row{9}=<tr><td>\1</td><td>\2</td><td>\3</td><td>\4</td><td>\5</td><td>\6</td><td>\7</td><td>\8</td><td>\9</td></tr>" \ + "sic=" \ "subpage_single=@subpage" # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources diff --git a/src/third_party/wiredtiger/src/docs/arch-backup.dox b/src/third_party/wiredtiger/src/docs/arch-backup.dox new file mode 100644 index 00000000000..7969e11190b --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-backup.dox @@ -0,0 +1,69 @@ +/*! @arch_page arch-backup Backup + +# Overview # + +Backup in WiredTiger can be performed using the \c "backup:" cursor. WiredTiger +supports four types of backups. + +1. Full database +2. Block-based incremental backup +3. Log-based incremental backup +4. Target backup + +The following page describes how backup works inside the WiredTiger. Please refer to +@ref backup for details on how to use each of these types of backups. + +# Full database # + +When the application opens the backup cursor, internally WiredTiger generates a list of all +files in the database that are necessary for the backup. WiredTiger takes both the checkpoint +and schema locks to block database file modifications while generating the list files. There +is a contract that all files that exist at the time the backup starts exist for the entire time +the backup cursor is open. This contract applies to all files in the database, not just files +that are included in the generated list. + +WiredTiger log files are also part of the generated file list for backup. There is a contract +that the log files do not contain or make visible operations that are added to the log after +the backup started that could have adverse affects during recovery and restore. Therefore +WiredTiger forces a log file switch when opening the backup cursor. To fulfill the earlier +contract that all files must exist during backup, WiredTiger does not use any pre-allocated +log files nor does it archive or remove old log files during the time the backup cursor is open. + +Once the backup is opened successfully, any checkpoints that exist on the database before +backup starts must be retained until the backup cursor is closed. All the newer checkpoints +that are created during the backup in progress are cleaned whenever a newer checkpoint is +created. + +# Block-based incremental backup # + +Block-based incremental backup is performed by tracking the modified blocks in the checkpoint. +Whenever a new checkpoint occurs on a file, any new or modified blocks in this checkpoint are +recorded as a bit string. This bit string information is updated with every new checkpoint. + +Whenever the incremental backup cursor is opened to perform the backup, WiredTiger returns all +the file offsets and sizes that are modified from the previous source backup identifier. +WiredTiger returns the full file information to be copied for all the new files that are created, +renamed or imported into the database after the incremental backup cursor is opened. Refer to +@ref backup_incremental-block for more information on how to use the block-based incremental backup. + +# Log-based incremental backup # + +When the backup cursor is opened with the \c target configuration string as \c "target=(\"log:\\")" +the log-based incremental backup is performed by adding all the existing log files in the database +to the list of files that needs to be copied. Applications wanting to use log files for incremental +backup must first disable automatic log file removal using the \c log=(archive=false) configuration +to ::wiredtiger_open. By default, WiredTiger automatically removes log files no longer required +for recovery. Refer to @ref backup_incremental for more information on how to use the log-based +incremental backup. + +# Target backup # + +When the backup cursor is opened with the \c target configuration string as \c "target=", +WiredTiger internally generates a list of targeted files instead of all files like the full backup. +If the targeted object is a table, all the indexes and column groups of the table are also backed up. + +Note: There can only be one backup cursor open at a time unless using duplicate backup +cursors. The duplicate backup cursors are used for catching up with the log files or with +block-based incremental backup. Please refer to @ref backup_duplicate for more details. + +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-column.dox b/src/third_party/wiredtiger/src/docs/arch-column.dox deleted file mode 100644 index 4b3b399349a..00000000000 --- a/src/third_party/wiredtiger/src/docs/arch-column.dox +++ /dev/null @@ -1,23 +0,0 @@ -/*! @arch_page arch-column Column Store - -Column Stores are Btrees stored in WiredTiger that have as their -key a record id, that is, a 64 bit unsigned integer. Thus, they implement -a specialized version of a Btree, where the key is a predictable length. - -A particular kind of column store is the fixed length column store. -As its name implies, the value is fixed length, and furthermore the -value is restricted to 1 and 8 bits in length. The bit length is specified -when the column store is created. The fixed length column store -has specialized use cases like bitmaps. - -The more general case is the variable length column store which allows -for values that have any length, and may have arbitrary types, including -aggregates of various types. - -Internally, row stores and both kinds of column stores all use a common -\c WT_BTREE structure. Column stores are distinguished in that -<code>WT_BTREE->type == BTREE_COL_VAR</code> for variable length column stores -and <code>WT_BTREE->type == BTREE_COL_FIX</code> for fixed length column stores. -Internal functions that navigate, access and manipulate Btrees have -code sprinkled throughout that is conditional on <code>WT_BTREE->type</code>. -*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-data-file.dox b/src/third_party/wiredtiger/src/docs/arch-data-file.dox index 80a32dd4525..580b355d838 100644 --- a/src/third_party/wiredtiger/src/docs/arch-data-file.dox +++ b/src/third_party/wiredtiger/src/docs/arch-data-file.dox @@ -1,6 +1,6 @@ /*! @arch_page arch-data-file Data File Format -@section Database Files +@section data_file_database_files Database Files WiredTiger stores databases in a durable format through writing data to files it has created in the database directory. These include data files representing @@ -12,7 +12,7 @@ Data files are denoted by the .wt suffix. The history store file (WiredTigerHS.wt) and the metadata file (WiredTiger.wt) are special WiredTiger files, but have the same underlying structure as normal data files. -@section Format & Layout of Data Files +@section data_file_format Data File Format and Layout In memory, a database table is represented by a B-Tree data structure, which is made up of nodes that are page structures. The root and internal pages will only @@ -30,7 +30,7 @@ contents. This information is used to verify that the file is a legitimate WiredTiger data file with a compatible WiredTiger version, and that its contents are not corrupted. -@section Format & Layout of On-Disk Pages +@section data_file_on_disk_page_format On-Disk Page Format and Layout Pages consist of a header (\c WT_PAGE_HEADER and \c WT_BLOCK_HEADER) followed by a variable number of cells, which encode keys, values or addresses (see @@ -56,7 +56,7 @@ described here. The encoding strikes a balance between data that can be compacted efficiently in time and space, extensibility, and compatibility with previous versions. -@section Page Types +@section data_file_page_types Page Types Different types of pages are made up of different types of cells. @@ -94,7 +94,7 @@ is used (the identical key prefix is stored only once per page to reduce size). In addition, each block written may be compressed and/or encrypted. The header portion of each block remains uncompressed and unencrypted. -@section Data File Functions +@section data_file_functions Data File Functions When data needs to be read from a data file, it is unpacked into in-memory structures, and vice versa, we pack the contents when we want to write a diff --git a/src/third_party/wiredtiger/src/docs/arch-glossary.dox b/src/third_party/wiredtiger/src/docs/arch-glossary.dox index ea71fe5893b..f12468aaaaf 100644 --- a/src/third_party/wiredtiger/src/docs/arch-glossary.dox +++ b/src/third_party/wiredtiger/src/docs/arch-glossary.dox @@ -4,31 +4,98 @@ WiredTiger has a lot of domain specific nomenclature - this page attempts to decode it. This is intended for those navigating the WiredTiger source tree - it describes terms internal to the storage engine. -@section arch-glossary-general General Terms Used in WiredTiger +The definitions list how the terms are used in WiredTiger, which may not match how +they are used in other settings. -<table> -<caption id="general_terms">General Term Table</caption> -<tr><th>Term <th>Definition -<tr><td>Hello<td>The typical next word is World -</table> - -@section arch-glossary-checkpoint Terms Related to Checkpoints +@section arch-glossary-transactions Glossary (click headings to sort) -<table> -<caption id="checkpoint_terms">Checkpoint Term Table</caption> -<tr><th>Term <th>Definition -<tr><td>Hello<td>The typical next word is World -</table> -@section arch-glossary-transactions Terms Related to Transactions +<!-- Please keep glossary formatted a single entry to a line, and sorted alphabetically (case insensitive). --> -<table> -<caption id="transaction_terms">Transaction Term Table</caption> -<tr><th>Term <th>Definition -<tr><td>transaction<td>A unit of work performed in WiredTiger that is atomic and consistent with the specified isolation levels, and durability levels. -<tr><td>session<td>The container that manages transactions and performs the transactional operations on behalf of the users in a single thread. -<tr><td>atomicity<td>Atomicity is a guarantee provided within the context of a transaction that all operations made within that transaction either succeed or fail. -<tr><td>isolation<td>Isolation determines which versions of data are visible to a running transaction, and whether it can see changes made by concurrently running transactions. -<tr><td>durability<td>Durability is the property that guarantees that transactions that have been committed will survive permanently. +<table class="doxtable sortable"> +<tr><th>Term<th>Category<th>Definition +<tr><td>address cookie<td>block manager<td>an opaque set of bytes returned by the block manager to reference a block in a Btree file, it includes an offset, size, checksum, and object id. +<tr><td>atomicity<td>transactions<td>a guarantee provided within the context of a transaction that all operations made within that transaction either succeed or fail. +<tr><td>barrier<td>general<td>a call that forces the CPU and compiler to enforce ordering constraints on memory operations, used in multithreaded code. +<tr><td>block<td>block manager<td>the smallest granularity of data that is read from or written to a Btree file. A btree page, when it appears on disk, may consist of one or more blocks. +<tr><td>block compression<td>btree<td>after pages of btrees are encoded to be written, they may be compressed according to a configured algorithm. Compressors may be added to WiredTiger via the extension mechanism. +<tr><td>block manager<td>block manager<td>an internal API that abstracts writing, reading and allocating blocks to and from files. A block manager is associated with a Btree. +<tr><td>Bloom filter<td>lsm<td>a data structure that identifies that a key cannot be present. Used by LSM. +<tr><td>btree<td>btree<td>a data structure on disk or in memory that stores keys, it with its associated value. In WiredTiger, all btrees are in fact B+trees, meaning that adjacent keys and values are grouped into pages. +<tr><td>bulk insert<td>btree<td>the capability to rapidly insert a set of ordered key value pairs into a Btree. In WiredTiger, a Btree's dhandle must be obtained exclusively, guaranteeing that the inserts can be done single threaded. +<tr><td>cell<td>file format<td>a key or value packed as it appears on disk for a Btree. +<tr><td>checkpoint<td>checkpoint<td>a stable point that bounds how much work recovery must do. Data committed before a checkpoint is guaranteed to reside in data files. Thus, recovery must replay log files starting at the last completed checkpoint. +<tr><td>chunk<td>lsm<td>a single file within an LSM tree. +<tr><td>collator<td>general<td>a comparison function that determines how keys will be ordered in a Btree. The default collator is byte oriented. A custom collator may be packaged as an extension and attached to a Btree. +<tr><td>column<td>schema<td>a single field in a record, which must be unpacked from a raw key or value. +<tr><td>column group<td>schema<td>a set of column store Btrees, whose record ids logically correspond. Together, a column group represents a table of data, with its columns being the union of columns in the individual Btrees. +<tr><td>column store<td>btree<td>a Btrees that has as its key a record id +<tr><td>commit timestamp<td>transactions<td>values in this transaction become visible at this timestamp. +<tr><td>compare and swap (CAS)<td>general<td>a CPU instruction that can be used to coordinate the activity of multiple threads. Specifically, the instruction atomically changes a memory location to a new value only if the existing value matches a second value. +<tr><td>compression<td>general<td>one of several techniques to reduce the size of data on disk and memory. See block compression, run length encoding, Huffman encoding, key prefix compression. +<tr><td>connection<td>general<td>an encapsulation of information stored for an application's use of a WiredTiger instance attached to a specific home directory. This encapsulation is kept in a handle that is used in the API. +<tr><td>cursor<td>general<td>a handle in the WiredTiger API that encapsulates a position in a Btree, and may hold the key and value associated with that position. +<tr><td>data handle<td>general<td>an abstraction of a named file, table or other data source in WiredTiger +<tr><td>data source<td>general<td>an abstraction that extends the uri name space. A custom data source defines APIs that are invoked when objects are created in, or cursors are opened on, the new name space. +<tr><td>dhandle<td>general<td>see data handle +<tr><td>diagnostic build<td>general<td>a build of WiredTiger that includes extra code to check invariants, and abort if they fail. +<tr><td>durability<td>transactions<td>the property that guarantees that transactions that have been committed will survive permanently. +<tr><td>encryption<td>btree<td>in WiredTiger, encryption is applied to data files, log files and metadata. Encryption algorithms may be added to WiredTiger via the extension mechanism. +<tr><td>encryptor<td>btree<td>an encryption algorithm packaged as an extension. +<tr><td>eviction<td>general<td>the process of freeing memory in the cache, which often includes reconciling and writing dirty pages. This term generally includes the process of deciding which pages should be evicted. +<tr><td>extension<td>general<td>a module that uses the WiredTiger extension interface, allowing it to be added in an encapsulated way, without altering the core of the WiredTiger library. Extensions are typically compiled into shared libraries. +<tr><td>extent list<td>block manager<td>a list of contiguous sets of blocks in a file. Extent lists are used to track available blocks and/or used blocks. +<tr><td>fast truncate<td>btree<td>a truncate that spans a key range over more than one leaf page may be performed quickly by deleting entire data pages at a time. +<tr><td>frag<td>verify<td>in verify, a part of a file that is the minimum allocation size and aligned the same. The smallest blocks may be the same as a frag, other blocks may be composed of multiple contiguous frags. +<tr><td>hazard pointer<td>general<td>a data structure that assists in lock free algorithms, used by WiredTiger Btrees +<tr><td>history store<td>general<td>storage in WiredTiger (currently implemented as a Btree) that keeps previous values associated with a key. +<tr><td>home directory<td>general<td>the directory containing all the data for a WiredTiger connection. The connection is attached to this directory via a call to ::wiredtiger_open. +<tr><td>Huffman encoding<td>btree<td>data in a btree may optionally be encoded using a Huffman encoding, based on either English letter frequencies or a custom encoding. See @subpage_single huffman for details. +<tr><td>index<td>schema<td>a Btree with records having values that are keys to another Btree. An index (or set of indices) become associated with a primary Btree via the table construct. +<tr><td>insert<td>btree<td>an in-memory structure holding a key and a set of updates for a Btree. +<tr><td>internal page<td>btree<td>either in-memory or on disk, a page in a Btree that has a set of keys that reference pages beneath it. +<tr><td>isolation<td>transactions<td>determines which versions of data are visible to a running transaction, and whether it can see changes made by concurrently running transactions. +<tr><td>key prefix compression<td>btree<td>a technique to store identical key prefixes only once per page. See @ref file_formats_compression +<tr><td>kv or k/v<td>btree<td>a key/value pair in a Btree. +<tr><td>leaf page<td>btree<td>either in-memory or on disk, a page in a Btree that has a pairs of keys and values, and no pages beneath it. +<tr><td>log structured merge tree (or LSM tree)<td>lsm<td>a data structure that stores keys and values (as a Btree), but composed of potentially many trees (Btrees in WiredTiger). +<tr><td>LSM<td>lsm<td>see log structured merge tree +<tr><td>merge<td>btree<td>the process of making a single page out of two adjacent smaller pages in a Btree. This can occur with leaf pages or internal pages. +<tr><td>metadata<td>general<td>a set of data that is used to help manage files, tables, indices and column groups in WiredTiger. +<tr><td>metadata tracking<td>general<td>a technique to track a set of changes to the metadata, so they can be "rolled back" when an error occurs. +<tr><td>mutex<td>general<td>a locking algorithm that waits, allowing other threads to execute, when the lock cannot be immediately acquired, and is woken to retry when the lock is available. Although more heavyweight than a spin lock, it consumes fewer resources while waiting. Appropriate for resources that may be in contention and may be held for longer periods. +<tr><td>object id<td>tiered storage<td>an integer that indicates an individual file that is a part of a Btree. +<tr><td>oldest timestamp<td>transactions<td>a connection-wide value, specifies the oldest timestamp that must be tracked by the system. Values with stop timestamps older than this value do not have to be retained. +<tr><td>on disk<td>general<td>a shorthand meaning "in the file system", and may or may not be stored on a physical disk. +<tr><td>overflow page<td>btree<td>either in-memory or on disk, a page in a Btree that has a key or value that is too large to appear on a leaf or internal page. +<tr><td>pack<td>general<td>convert in-memory representations of keys or values to a compact format for file storage +<tr><td>page<td>btree<td>one logical node of a Btree, in memory, or on disk, that may store keys and/or values. See internal page, leaf page, overflow page. +<tr><td>page reference<td>btree<td>an indirect reference to a key on a page in a Btree. The reference struct (WT_REF) acts as the indirection and may include a pointer to an in memory object or an address cookie to find the object on disk. "ref" for short. +<tr><td>page split<td>btree<td>the process of breaking up a large page in a Btree into multiple smaller pages. This may occur with either leaf pages or internal pages. +<tr><td>pthread<td>general<td>a thread implementation used on POSIX systems +<tr><td>raw<td>schema<td>data as it appears on-disk, typically in a packed format. +<tr><td>read timestamp<td>transactions<td>a timestamp that specifies what data should be visible. +<tr><td>recno<td>general<td>a 64 bit integer. When used as a key, the Btree is said to be a column store, and the keys are assigned incrementally by WiredTiger when new records are inserted. +<tr><td>reconcile<td>general<td>convert the in-memory version of a page to a more compact form for file storage. +<tr><td>recovery<td>general<td>a procedure used on opening WiredTiger whereby, starting at the last checkpoint written, log records are played back to bring the database up to date. +<tr><td>ref<td>btree<td>see page reference +<tr><td>RLE<td>general<td>see run length encoding +<tr><td>run length encoding<td>general<td>also known as "RLE". A technique to save space whereby repeated items can be represented by a count followed by the item. For example the string of letters \sic "AAAAAAABBBCDDDD" might be encoded as \sic "7A3BC4D" using "out of band" digits. +<tr><td>salvage<td>general<td>functionality to repair Btree disk files. +<tr><td>schema<td>schema<td>the algorithms that use metadata to map the raw keys and values in Btrees to higher level abstractions like tables with typed columns. +<tr><td>session<td>general<td>an encapsulation of information stored for single thread of control in the application. This encapsulation is kept in a handle that is used in the API. Sessions manage transactions and performs the transactional operations. +<tr><td>skip list<td>general<td>a variation of a sorted linked list that allows for faster (O(log(N))) searching. It is used in WiredTiger as a lock-free data structure for keys inserted on a page. +<tr><td>spin lock<td>general<td>a simple locking algorithm that continually attempts to acquire a lock, appropriate for resources that are likely to be available and may be held for a short time. This lock is CPU intensive while it is waiting. Contrast with mutex +<tr><td>split<td>btree<td>see page split +<tr><td>stable timestamp<td>transactions<td>a connection-wide value, checkpoints will not include commits newer than this value. +<tr><td>table<td>schema<td>an abstraction that allows one or more btrees to be used in a coordinated way, which is specified by the schema. The table coordinates indices and column groups. +<tr><td>thread<td>general<td>a thread of control. WiredTiger uses pthreads for POSIX systems. +<tr><td>thread group<td>general<td>a group of threads that perform a common task. Eviction is performed by a thread group. +<tr><td>timestamp<td>transactions<td>refers to one of several types of 64 bit values used to control transactional visibility of data in WiredTiger. See oldest timestamp, stable timestamp, commit timestamp, read timestamp. +<tr><td>transaction<td>transactions<td>a unit of work performed in WiredTiger that is atomic and consistent with the specified isolation level and durability level. +<tr><td>truncate<td>btree<td>an operation to remove a range of key/values from a Btree. See also fast truncate +<tr><td>update<td>btree<td>a value associated with a key in a Btree. A key may have many updates, these correspond to values written to the key previously. +<tr><td>update chain<td>btree<td>a list of updates. +<tr><td>verify<td>general<td>functionality to check integrity of Btree disk files. </table> */ diff --git a/src/third_party/wiredtiger/src/docs/arch-hs.dox b/src/third_party/wiredtiger/src/docs/arch-hs.dox index 6ed50c5fbf6..e1d0a88c677 100644 --- a/src/third_party/wiredtiger/src/docs/arch-hs.dox +++ b/src/third_party/wiredtiger/src/docs/arch-hs.dox @@ -1,8 +1,272 @@ /*! @arch_page arch-hs History Store -The History Store in WiredTiger tracks old (all but the latest committed) -versions of records. By having these records in storage separate from -the current version, they can be used to service long running transactions, +The history store in WiredTiger tracks historical versions of records required to +service older readers. By having these records in storage separate from +the current version, they can be used to service long running transactions and be evicted as necessary, without interfering with activity that uses -the most recent committed versions. +the most recent committed versions. With the introduction of history store, only the +newest update to a key is written to the user table while the older updates +for the key are written to the history store. All user tables in the database are +backed by a single history store table. The history store has no outward +visibility to the application code and only WiredTiger's internal modules can +perform operations on the history store table using a predefined cursor API. + +@section arch_hs_table History store table structure +WiredTiger writes the history store on the disk as a standard row-store table. The key for +the history store table is a combination of: +- btree ID of user table this update belongs to +- record key (byte-string for row-store, record number for column-store) +- start timestamp for the update +- counter value + +This key format allows us to search efficiently for a given record key and read +timestamp combination. The last part of the key is a monotonically increasing counter +to keep the key unique in scenarios where a key has multiple updates with the same commit timestamp. +As the key for the history store table is different for row- and column-store, +we store both key types in a byte string otherwise we'd need two history store files with +different key formats. + +The corresponding value for each key is stored as a combination of: +- stop timestamp +- durable timestamp +- update type +- value + +@subsection arch_hs_table_tombstone Tombstones in the history store table +The stop timestamp refers to the point in time after which the update is no longer visible. This can +either be a result of deletion or of a new update being committed. An update with a valid stop time +is also called a \c tombstone. Every update in the history store has a valid stop timestamp and +transaction id associated with it, except for cases where the update preceded a prepared update. +The stop timestamp of the update immediately before the prepared update is set to the maximum +possible timestamp (\c WT_TS_MAX). The stop timestamp of the latest update in the history store is +updated once the prepared update is resolved. A tombstone becomes globally visible if the stop +timestamp is less than the oldest timestamp and the stop transaction id is visible to all concurrent +transactions in the system. A checkpoint can delete history store pages that only contain globally +visible tombstones. The garbage collection process is discussed in @ref arch-checkpoint. + +@section arch_hs_initialize History store initialization +WiredTiger checks for an existing history store table at startup and creates a new table if it is +not able to find one. The history store table is usually created when WiredTiger creates a +new database, with one exception where WiredTiger is opening an existing database created by an +older version of WiredTiger that doesn't support the history store. WiredTiger uses +\c WiredTigerHS.wt as the filename for the history store table. + +@section arch_hs_cursor History store cursor interface +WiredTiger uses a cursor interface for history store table to make it easier for different modules +to perform data operations on the history store table. A new cursor can be opened by calling +\c __wt_curhs_open(). History store cursor implementation supports most of the standard \c WT_CURSOR +methods except for \c WT_CURSOR::search() method. Recall that the key contains a +counter and timestamp value and it is expected that the caller can not possibly know the exact +values for both fields beforehand. Therefore, \c WT_CURSOR::search_near() is used for all search +operations. Two helper functions, \c __wt_curhs_search_near_before() and +\c __wt_curhs_search_near_after(), +have been provided to facilitate searching for the required record in the history store table. + +@section arch_hs_cursor_visibility History store cursor interface and visibility rules +When using the history store cursor interface, the user can configure the type of visibility checks +that are performed on the records. The behavior is controlled by a set of cursor flags: + +- By default, snapshot based visibility checks are performed on all records returned to the API +user. + +- When the flag \c WT_CURSTD_HS_READ_ALL is set on the cursor, no visibility checks are performed +on the records returned to the API user. This means that cursor interface can even return a record +with a globally visible tombstone. When this flag is set, it suppresses the effect of other +visibility flags. + +- When the flag \c WT_CURSTD_HS_READ_COMMITTED is set, the cursor interface can return any record +except for globally deleted records. This flag has no effect when the flag +\c WT_CURSTD_HS_READ_ALL is set. + +History store cursor users must use one of the flags if the caller thread is running in a lower +isolation level and doesn't hold a valid snapshot. + +@section arch_hs_reconciliation History store and reconciliation +When a dirty page is reconciled on a user file btree, the update chain is examined and the latest +committed update is chosen as the on-disk value. All older updates are added to the history store +table assuming they are not yet obsolete. Additionally any out of order timestamps will +be corrected. + +Consider the following update chain for a user table with btree id 1000 and data store key "AAA": + +@plantuml_start{hs_update_chain.png} +@startuml{hs_update_chain.png} + +together { +rectangle " U1 \n @ts 70" as U1 +rectangle " U2 \n @ts 80" as U2 +rectangle " U3 \n @ts 90" as U3 +rectangle " U4 \n @ts 100" as U4 + +U2-right->U1 +U3-right->U2 +U4-right->U3 + +} +@enduml +@plantuml_end + +Assuming all updates are committed, updates \c U1, \c U2 and \c U3 will be added to the history +store table as shown in the table below. + +<table style="height: 195px; width: 811px; border-color: Black;" border="4"> + <tr style="height: 32px;"> + <td style="width: 353.391px; height: 32px; text-align: center;" colspan="4"> + <strong>KEY</strong> + </td> + <td style="width: 456.609px; height: 32px; text-align: center;" colspan="4"> + <strong>VALUE</strong> + </td> + </tr> + <tr style="height: 32px;"> + <td style="width: 72.5156px; height: 32px; text-align: center;"> + <strong>Btree ID</strong> + </td> + <td style="width: 89.875px; height: 32px; text-align: center;"> + <strong>User Key</strong> + </td> + <td style="width: 97.0312px; height: 32px; text-align: center;"> + <strong>Start ts</strong> + </td> + <td style="width: 93.9688px; height: 32px; text-align: center;"> + <strong>Counter</strong> + </td> + <td style="width: 92.4375px; height: 32px; text-align: center;"> + <strong>Stop ts</strong> + </td> + <td style="width: 105.719px; height: 32px; text-align: center;"> + <strong>Durable ts</strong> + </td> + <td style="width: 110.312px; height: 32px; text-align: center;"> + <strong>Type</strong> + </td> + <td style="width: 148.141px; height: 32px; text-align: center;"> + <strong>Value</strong> + </td> + </tr> + <tr style="height: 32.5625px;"> + <td style="width: 72.5156px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 89.875px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 97.0312px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 93.9688px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 92.4375px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 105.719px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 110.312px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 148.141px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + </tr> + <tr style="height: 32px;"> + <td style="width: 72.5156px; height: 32px; text-align: center;">1000</td> + <td style="width: 89.875px; height: 32px; text-align: center;">"AAA"</td> + <td style="width: 97.0312px; height: 32px; text-align: center;">70</td> + <td style="width: 93.9688px; height: 32px; text-align: center;">0</td> + <td style="width: 92.4375px; height: 32px; text-align: center;">80</td> + <td style="width: 105.719px; height: 32px; text-align: center;">70</td> + <td style="width: 110.312px; height: 32px; text-align: center;">STANDARD</td> + <td style="width: 148.141px; height: 32px; text-align: center;">Value from U1</td> + </tr> + <tr style="height: 32px;"> + <td style="width: 72.5156px; height: 32px; text-align: center;">1000</td> + <td style="width: 89.875px; height: 32px; text-align: center;">"AAA"</td> + <td style="width: 97.0312px; height: 32px; text-align: center;">80</td> + <td style="width: 93.9688px; height: 32px; text-align: center;">0</td> + <td style="width: 92.4375px; height: 32px; text-align: center;">90</td> + <td style="width: 105.719px; height: 32px; text-align: center;">80</td> + <td style="width: 110.312px; height: 32px; text-align: center;">STANDARD</td> + <td style="width: 148.141px; height: 32px; text-align: center;">Value from U2</td> + </tr> + <tr style="height: 32px;"> + <td style="width: 72.5156px; height: 32px; text-align: center;">1000</td> + <td style="width: 89.875px; height: 32px; text-align: center;">"AAA"</td> + <td style="width: 97.0312px; height: 32px; text-align: center;">90</td> + <td style="width: 93.9688px; height: 32px; text-align: center;">0</td> + <td style="width: 92.4375px; height: 32px; text-align: center;">100</td> + <td style="width: 105.719px; height: 32px; text-align: center;">90</td> + <td style="width: 110.312px; height: 32px; text-align: center;">STANDARD</td> + <td style="width: 148.141px; height: 32px; text-align: center;">Value from U3</td> + </tr> + <tr style="height: 32px;"> + <td style="width: 72.5156px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 89.875px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 97.0312px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 93.9688px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 92.4375px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 105.719px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 110.312px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 148.141px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + </tr> +</table> + +For the history store table entry corresponding to the update \c U1, the start timestamp in the key +is the start timestamp for the update \c U1 and the stop timestamp in the value is the start +timestamp for the update \c U2. + +When a modified history store table page is processed by reconciliation, a new page image is +created without records which have globally visible tombstones. This ensures that WiredTiger is +only keeping the relevant historical data required to serve the oldest reader in the system or +as dictated by the oldest timestamp set by the application. Once all records on a page are obsolete, +the page itself can be removed to reduce the size of the history store table +(see @ref arch-transaction for more details about oldest timestamp). + +@section arch_hs_read Searching for older versions of a key in History Store +When looking for an update visible to the current transaction, WiredTiger first searches the update +chain for any visible updates. If there is no visible update in the chain, WiredTiger then +inspects the on-disk version of the key. If that version is not visible to the transaction, +WiredTiger searches the history store table for a visible update for the key. When +searching for an update that satisfies the read timestamp constraint, WiredTiger starts +with the newest update of the key in the history store table and then iterates through the older +updates until there are no updates left to process. Note that although there can be multiple updates +in the history store for a key and read timestamp combination, each update would have different +visibility based on transaction and timestamp based visibility rules. In case there are no records +visible in the history store table, \c WT_NOTFOUND error is returned to the reader. + +@section arch_hs_rts History store and rollback-to-stable +Rollback-to-stable searches the history store table for valid updates that are stable according to +the supplied stable timestamp and replaces the on-disk version with an update from the history store +table that fulfills the criteria. Rollback-to-stable also removes updates from the +history store table that are not stable according to the stable timestamp and transaction snapshot +being used for rollback-to-stable operation. More details about the rollback-to-stable operation +can be found in the @ref arch-rts page. + +@section arch_hs_prepared History store and prepared transactions +When there is a prepared update for a key and the page is evicted, the prepared update is written +to the on-disk page and any older updates are written to the history store table. There can be +no update from a different transaction that is newer than the prepared update for the key and +therefore the history store should never contain a prepared update. When a prepared update is +committed, the stop timestamp of the newest update in the history store table is updated from +\c WT_TS_MAX to the commit timestamp of the prepared update. When a prepared update is rolled back, +the newest update from the history store table is restored as the on-disk version and then removed +from the history store table. + */ diff --git a/src/third_party/wiredtiger/src/docs/arch-index.dox b/src/third_party/wiredtiger/src/docs/arch-index.dox index 75d5a8abf5c..3d916e675aa 100644 --- a/src/third_party/wiredtiger/src/docs/arch-index.dox +++ b/src/third_party/wiredtiger/src/docs/arch-index.dox @@ -51,8 +51,7 @@ rectangle "**WiredTiger Engine** rectangle "[[arch-transaction.html Transactions]]" as txn rectangle "[[arch-metadata.html Metadata]]" as meta rectangle "[[arch-dhandle.html dhandle/\nBtree]]" as btree - rectangle "[[arch-row.html Row\nStorage]]" as row - rectangle "[[arch-column.html Column\nStorage]]" as column + rectangle "[[arch-row-column.html Row/Column\nStorage]]" as row rectangle "[[arch-hs.html History\nStore]]" as history rectangle "[[arch-snapshot.html Snapshots]]" as snapshot rectangle "[[arch-cache.html Cache]]" as cache @@ -123,6 +122,10 @@ wt_file -[hidden]right-> log_file We go into some detail for some of the internal components. +@subpage arch-backup + +Hot backup uses a type of cursor to backup the database. + @subpage arch-block The Block Manager manages the reading and writing of disk blocks. @@ -136,10 +139,6 @@ make up in-memory Btrees and subordinate data structures. A checkpoint is created by WiredTiger to serve as a point from which it can recover. -@subpage arch-column - -Column Stores are Btrees that have as their key a record id. - @subpage arch-cursor Cursors are used to get and modify data. @@ -184,9 +183,10 @@ Metadata is stored as <code>uri, config</code> K/V pairs in a designated table. WiredTiger has a Python API that is useful for scripting and experimentation. -@subpage arch-row +@subpage arch-row-column -Row Stores are Btrees that have a variable size key and data. +Row stores and column store are B-Trees. Row stores have a variable size key and data while column +stores have as their key a record id. @subpage arch-rts diff --git a/src/third_party/wiredtiger/src/docs/arch-row-column.dox b/src/third_party/wiredtiger/src/docs/arch-row-column.dox new file mode 100644 index 00000000000..d7d10f54795 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-row-column.dox @@ -0,0 +1,72 @@ +/*! @arch_page arch-row-column Row Store & Column Store + +@section row_column_definition Definition + +WiredTiger supports both row-stores and column-stores and each access method is represented +internally by a B-Tree. Keys and values in row-stores can be of variable length. Furthermore, keys +are explicitly stored. Column-stores use a 64-bit unsigned integer record id as keys and they are +implied given the row's position in the B-Tree. + +There are two types of column-stores in WiredTiger: +- The fixed length column-store, as its name implies, the value has a fixed length, and furthermore +the value is restricted to between 1 and 8 bits in length. The bit length is specified when the +column store is created. The fixed length column store has specialized use cases like bitmaps. +- The variable length is a more general case which allows for values to have any length similarly to +row-stores. + +Please refer to @ref arch-data-file to learn more about the on-disk format for row-stores and +column-stores. + +@section row_column_rs Row-stores + +Row-stores are probably seen as the traditional access method and they serve a general purpose. In +WiredTiger, row-store keys are explicitly stored and they can be duplicated within the same single +file which directly impacts the on-disk storage size. However, keys are only duplicated if they are +the first key on the page, which gets them copied/promoted to an internal page. They are not +necessarily duplicated, too, they can be prefix/suffix compressed to discard unnecessary bytes on +the internal pages. + +@section row_column_vlcs Variable length column-stores + +Keys in column-stores are not stored but derived from the row's position in the B-Tree which saves +disk space. In fact, there is a starting key on each page which is used to figure out the rest of +the keys present on that same page. Column-stores values also present an advantage to further save +more disk space as they can be written in more compact forms through encoding techniques. In +WiredTiger, the run-length encoding is used to replace any sequence of the same value with a count +and value indicator. However, this makes the access to a specific record more complicated, it makes +it impossible to jump to a record in a leaf page. + +@section row_column_flcs Fixed length column-stores + +Fixed length column stores are very different internally and usually serve specific purposes as it +is a <a href="https://en.wikipedia.org/wiki/Bitmap"><b>Bitmap</b></a>. It makes fixed length +column-stores efficient to retrieve a value at a given offset, the use of +<a href="https://en.wikipedia.org/wiki/Bloom_filter"><b>Bloom filter</b></a> is probably one of the +best examples. + +@section row_column_internal_use Internal usage + +Internally, row-stores and column-stores use a common \c WT_BTREE structure. The fundamental +difference is that the \c WT_BTREE->type field is set to \c BTREE_ROW for row-stores and +\c BTREE_COL for column-stores. Internal functions that navigate, access and manipulate B-Trees have +code sprinkled throughout that is conditional on \c WT_BTREE->type. + +<table> +@hrow{, Row Store, Column Store - Variable Length, Column Store - Fixed Length} +@row{Characteristics,,,,} +@row{Internal representation, B-Tree (\c WT_BTREE), B-Tree (\c WT_BTREE), B-Tree (\c WT_BTREE)} +@row{B-Tree type, Generalized (\c BTREE_ROW), Specialized (\c BTREE_COL_VAR), Specialized (\c BTREE_COL_FIX)} +@row{Leaf pages, Key count equal to half the number of physical entries (unless all empty values flag is set where key count is equal to the number of physical entries), Only the first key is stored, Only the first key is stored} +@row{Record key, Anything (byte-string) prefix compressed or overflow objects, Record id (64 bit unsigned integer), Record id (64 bit unsigned integer)} +@row{Record value, Variable byte string length, Variable byte string length, Fixed bit string length (up to 8 bits)} +@row{Features,,,,} +@row{Random cursor, Yes, No, No} +@row{Block compression, Yes, Yes, Yes} +@row{Dictionary compression, Yes, Yes, No} +@row{Fast truncate, Yes, No, No} +@row{Huffman encoding, Yes, Yes, No} +@row{Prefix and suffix compression, Yes, No, No} +@row{RLE compression, No, Yes, No} +</table> + +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-row.dox b/src/third_party/wiredtiger/src/docs/arch-row.dox deleted file mode 100644 index ae3ffe22589..00000000000 --- a/src/third_party/wiredtiger/src/docs/arch-row.dox +++ /dev/null @@ -1,12 +0,0 @@ -/*! @arch_page arch-row Row Store - -Row Stores are Btrees stored in WiredTiger that do not have a record id -as key. Thus, they implement a generalized version of a Btree, where -the key and data can be arbitrary length. - -Internally, a row store and column store both use a common \c WT_BTREE -structure. The fundamental difference is that -<code>WT_BTREE->type == BTREE_ROW</code> for row stores. -Internal functions that navigate, access and manipulate Btrees have -code sprinkled throughout that is conditional on <code>WT_BTREE->type</code>. -*/ diff --git a/src/third_party/wiredtiger/src/docs/backup.dox b/src/third_party/wiredtiger/src/docs/backup.dox index 72188570706..01ecee137c7 100644 --- a/src/third_party/wiredtiger/src/docs/backup.dox +++ b/src/third_party/wiredtiger/src/docs/backup.dox @@ -77,6 +77,8 @@ the backup is available in the backup when that log file is included in the list of files. WiredTiger offers a mechanism to gather additional log files that may be created during the backup. +@section backup_duplicate Duplicate backup cursors + Since backups can take a long time, it may be desirable to catch up at the end of a backup with the log files so that operations that occurred during backup can be recovered. WiredTiger provides the ability to open a duplicate diff --git a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/hs_update_chain.png b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/hs_update_chain.png Binary files differnew file mode 100644 index 00000000000..75b7261110d --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/hs_update_chain.png diff --git a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx index 2fdeef6a039..69ba9be858a 100644 --- a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx +++ b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx @@ -12,8 +12,7 @@ <area shape="rect" id="id11" href="arch-snapshot.html" title="arch-snapshot.html" alt="" coords="381,441,455,457"/> <area shape="rect" id="id12" href="arch-transaction.html" title="arch-transaction.html" alt="" coords="339,328,428,344"/> <area shape="rect" id="id13" href="arch-hs.html" title="arch-hs.html" alt="" coords="93,642,140,674"/> -<area shape="rect" id="id14" href="arch-row.html" title="arch-row.html" alt="" coords="163,433,217,465"/> -<area shape="rect" id="id15" href="arch-column.html" title="arch-column.html" alt="" coords="272,433,326,465"/> +<area shape="rect" id="id14" href="arch-row-column.html" title="arch-row-column.html" alt="" coords="163,433,217,465"/> <area shape="rect" id="id16" href="arch-block.html" title="arch-block.html" alt="" coords="196,642,256,674"/> <area shape="rect" id="id17" href="arch-dhandle.html" title="arch-dhandle.html" alt="" coords="181,320,242,352"/> <area shape="rect" id="id18" href="arch-data-file.html" title="arch-data-file.html" alt="" coords="179,865,245,897"/> diff --git a/src/third_party/wiredtiger/src/docs/js/sorttable.js b/src/third_party/wiredtiger/src/docs/js/sorttable.js new file mode 100644 index 00000000000..8b2d629c63b --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/js/sorttable.js @@ -0,0 +1,501 @@ +/* + SortTable + version 2 + 7th April 2007 + Stuart Langridge, http://www.kryogenix.org/code/browser/sorttable/ + + Instructions: + Download this file + Add <script src="sorttable.js"></script> to your HTML + Add class="sortable" to any table you'd like to make sortable + Click on the headers to sort + + Thanks to many, many people for contributions and suggestions. + Licenced as X11: http://www.kryogenix.org/code/browser/licence.html + This basically means: do what you want with it. +*/ +/* Modified for WiredTiger to sort alphabetic items with insensitive case. */ + + +var stIsIE = /*@cc_on!@*/false; + +sorttable = { + init: function() { + // quit if this function has already been called + if (arguments.callee.done) return; + // flag this function so we don't do the same thing twice + arguments.callee.done = true; + // kill the timer + if (_timer) clearInterval(_timer); + + if (!document.createElement || !document.getElementsByTagName) return; + + sorttable.DATE_RE = /^(\d\d?)[\/\.-](\d\d?)[\/\.-]((\d\d)?\d\d)$/; + + forEach(document.getElementsByTagName('table'), function(table) { + if (table.className.search(/\bsortable\b/) != -1) { + sorttable.makeSortable(table); + } + }); + + }, + + makeSortable: function(table) { + if (table.getElementsByTagName('thead').length == 0) { + // table doesn't have a tHead. Since it should have, create one and + // put the first table row in it. + the = document.createElement('thead'); + the.appendChild(table.rows[0]); + table.insertBefore(the,table.firstChild); + } + // Safari doesn't support table.tHead, sigh + if (table.tHead == null) table.tHead = table.getElementsByTagName('thead')[0]; + + if (table.tHead.rows.length != 1) return; // can't cope with two header rows + + // Sorttable v1 put rows with a class of "sortbottom" at the bottom (as + // "total" rows, for example). This is B&R, since what you're supposed + // to do is put them in a tfoot. So, if there are sortbottom rows, + // for backwards compatibility, move them to tfoot (creating it if needed). + sortbottomrows = []; + for (var i=0; i<table.rows.length; i++) { + if (table.rows[i].className.search(/\bsortbottom\b/) != -1) { + sortbottomrows[sortbottomrows.length] = table.rows[i]; + } + } + if (sortbottomrows) { + if (table.tFoot == null) { + // table doesn't have a tfoot. Create one. + tfo = document.createElement('tfoot'); + table.appendChild(tfo); + } + for (var i=0; i<sortbottomrows.length; i++) { + tfo.appendChild(sortbottomrows[i]); + } + delete sortbottomrows; + } + + // work through each column and calculate its type + headrow = table.tHead.rows[0].cells; + for (var i=0; i<headrow.length; i++) { + // manually override the type with a sorttable_type attribute + if (!headrow[i].className.match(/\bsorttable_nosort\b/)) { // skip this col + mtch = headrow[i].className.match(/\bsorttable_([a-z0-9]+)\b/); + if (mtch) { override = mtch[1]; } + if (mtch && typeof sorttable["sort_"+override] == 'function') { + headrow[i].sorttable_sortfunction = sorttable["sort_"+override]; + } else { + headrow[i].sorttable_sortfunction = sorttable.guessType(table,i); + } + // make it clickable to sort + headrow[i].sorttable_columnindex = i; + headrow[i].sorttable_tbody = table.tBodies[0]; + dean_addEvent(headrow[i],"click", sorttable.innerSortFunction = function(e) { + + if (this.className.search(/\bsorttable_sorted\b/) != -1) { + // if we're already sorted by this column, just + // reverse the table, which is quicker + sorttable.reverse(this.sorttable_tbody); + this.className = this.className.replace('sorttable_sorted', + 'sorttable_sorted_reverse'); + this.removeChild(document.getElementById('sorttable_sortfwdind')); + sortrevind = document.createElement('span'); + sortrevind.id = "sorttable_sortrevind"; + sortrevind.innerHTML = stIsIE ? ' <font face="webdings">5</font>' : ' ▴'; + this.appendChild(sortrevind); + return; + } + if (this.className.search(/\bsorttable_sorted_reverse\b/) != -1) { + // if we're already sorted by this column in reverse, just + // re-reverse the table, which is quicker + sorttable.reverse(this.sorttable_tbody); + this.className = this.className.replace('sorttable_sorted_reverse', + 'sorttable_sorted'); + this.removeChild(document.getElementById('sorttable_sortrevind')); + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' <font face="webdings">6</font>' : ' ▾'; + this.appendChild(sortfwdind); + return; + } + + // remove sorttable_sorted classes + theadrow = this.parentNode; + forEach(theadrow.childNodes, function(cell) { + if (cell.nodeType == 1) { // an element + cell.className = cell.className.replace('sorttable_sorted_reverse',''); + cell.className = cell.className.replace('sorttable_sorted',''); + } + }); + sortfwdind = document.getElementById('sorttable_sortfwdind'); + if (sortfwdind) { sortfwdind.parentNode.removeChild(sortfwdind); } + sortrevind = document.getElementById('sorttable_sortrevind'); + if (sortrevind) { sortrevind.parentNode.removeChild(sortrevind); } + + this.className += ' sorttable_sorted'; + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' <font face="webdings">6</font>' : ' ▾'; + this.appendChild(sortfwdind); + + // build an array to sort. This is a Schwartzian transform thing, + // i.e., we "decorate" each row with the actual sort key, + // sort based on the sort keys, and then put the rows back in order + // which is a lot faster because you only do getInnerText once per row + row_array = []; + col = this.sorttable_columnindex; + rows = this.sorttable_tbody.rows; + for (var j=0; j<rows.length; j++) { + row_array[row_array.length] = [sorttable.getInnerText(rows[j].cells[col]), rows[j]]; + } + /* If you want a stable sort, uncomment the following line */ + //sorttable.shaker_sort(row_array, this.sorttable_sortfunction); + /* and comment out this one */ + row_array.sort(this.sorttable_sortfunction); + + tb = this.sorttable_tbody; + for (var j=0; j<row_array.length; j++) { + tb.appendChild(row_array[j][1]); + } + + delete row_array; + }); + } + } + }, + + guessType: function(table, column) { + // guess the type of a column based on its first non-blank row + sortfn = sorttable.sort_alpha; + for (var i=0; i<table.tBodies[0].rows.length; i++) { + text = sorttable.getInnerText(table.tBodies[0].rows[i].cells[column]); + if (text != '') { + if (text.match(/^-?[£$¤]?[\d,.]+%?$/)) { + return sorttable.sort_numeric; + } + // check for a date: dd/mm/yyyy or dd/mm/yy + // can have / or . or - as separator + // can be mm/dd as well + possdate = text.match(sorttable.DATE_RE) + if (possdate) { + // looks like a date + first = parseInt(possdate[1]); + second = parseInt(possdate[2]); + if (first > 12) { + // definitely dd/mm + return sorttable.sort_ddmm; + } else if (second > 12) { + return sorttable.sort_mmdd; + } else { + // looks like a date, but we can't tell which, so assume + // that it's dd/mm (English imperialism!) and keep looking + sortfn = sorttable.sort_ddmm; + } + } + } + } + return sortfn; + }, + + getInnerText: function(node) { + // gets the text we want to use for sorting for a cell. + // strips leading and trailing whitespace. + // this is *not* a generic getInnerText function; it's special to sorttable. + // for example, you can override the cell text with a customkey attribute. + // it also gets .value for <input> fields. + + if (!node) return ""; + + hasInputs = (typeof node.getElementsByTagName == 'function') && + node.getElementsByTagName('input').length; + + if (node.getAttribute("sorttable_customkey") != null) { + return node.getAttribute("sorttable_customkey"); + } + else if (typeof node.textContent != 'undefined' && !hasInputs) { + return node.textContent.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.innerText != 'undefined' && !hasInputs) { + return node.innerText.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.text != 'undefined' && !hasInputs) { + return node.text.replace(/^\s+|\s+$/g, ''); + } + else { + switch (node.nodeType) { + case 3: + if (node.nodeName.toLowerCase() == 'input') { + return node.value.replace(/^\s+|\s+$/g, ''); + } + case 4: + return node.nodeValue.replace(/^\s+|\s+$/g, ''); + break; + case 1: + case 11: + var innerText = ''; + for (var i = 0; i < node.childNodes.length; i++) { + innerText += sorttable.getInnerText(node.childNodes[i]); + } + return innerText.replace(/^\s+|\s+$/g, ''); + break; + default: + return ''; + } + } + }, + + reverse: function(tbody) { + // reverse the rows in a tbody + newrows = []; + for (var i=0; i<tbody.rows.length; i++) { + newrows[newrows.length] = tbody.rows[i]; + } + for (var i=newrows.length-1; i>=0; i--) { + tbody.appendChild(newrows[i]); + } + delete newrows; + }, + + /* sort functions + each sort function takes two parameters, a and b + you are comparing a[0] and b[0] */ + sort_numeric: function(a,b) { + aa = parseFloat(a[0].replace(/[^0-9.-]/g,'')); + if (isNaN(aa)) aa = 0; + bb = parseFloat(b[0].replace(/[^0-9.-]/g,'')); + if (isNaN(bb)) bb = 0; + return aa-bb; + }, + sort_alpha: function(a,b) { + /* + * Now we sort case insensitive. This code was: + + if (a[0]==b[0]) return 0; + if (a[0]<b[0]) return -1; + */ + if (a[0].toLowerCase()==b[0].toLowerCase()) return 0; + if (a[0].toLowerCase()<b[0].toLowerCase()) return -1; + return 1; + }, + sort_ddmm: function(a,b) { + mtch = a[0].match(sorttable.DATE_RE); + y = mtch[3]; m = mtch[2]; d = mtch[1]; + if (m.length == 1) m = '0'+m; + if (d.length == 1) d = '0'+d; + dt1 = y+m+d; + mtch = b[0].match(sorttable.DATE_RE); + y = mtch[3]; m = mtch[2]; d = mtch[1]; + if (m.length == 1) m = '0'+m; + if (d.length == 1) d = '0'+d; + dt2 = y+m+d; + if (dt1==dt2) return 0; + if (dt1<dt2) return -1; + return 1; + }, + sort_mmdd: function(a,b) { + mtch = a[0].match(sorttable.DATE_RE); + y = mtch[3]; d = mtch[2]; m = mtch[1]; + if (m.length == 1) m = '0'+m; + if (d.length == 1) d = '0'+d; + dt1 = y+m+d; + mtch = b[0].match(sorttable.DATE_RE); + y = mtch[3]; d = mtch[2]; m = mtch[1]; + if (m.length == 1) m = '0'+m; + if (d.length == 1) d = '0'+d; + dt2 = y+m+d; + if (dt1==dt2) return 0; + if (dt1<dt2) return -1; + return 1; + }, + + shaker_sort: function(list, comp_func) { + // A stable sort function to allow multi-level sorting of data + // see: http://en.wikipedia.org/wiki/Cocktail_sort + // thanks to Joseph Nahmias + var b = 0; + var t = list.length - 1; + var swap = true; + + while(swap) { + swap = false; + for(var i = b; i < t; ++i) { + if ( comp_func(list[i], list[i+1]) > 0 ) { + var q = list[i]; list[i] = list[i+1]; list[i+1] = q; + swap = true; + } + } // for + t--; + + if (!swap) break; + + for(var i = t; i > b; --i) { + if ( comp_func(list[i], list[i-1]) < 0 ) { + var q = list[i]; list[i] = list[i-1]; list[i-1] = q; + swap = true; + } + } // for + b++; + + } // while(swap) + } +} + +/* ****************************************************************** + Supporting functions: bundled here to avoid depending on a library + ****************************************************************** */ + +// Dean Edwards/Matthias Miller/John Resig + +/* for Mozilla/Opera9 */ +if (document.addEventListener) { + document.addEventListener("DOMContentLoaded", sorttable.init, false); +} + +/* for Internet Explorer */ +/*@cc_on @*/ +/*@if (@_win32) + document.write("<script id=__ie_onload defer src=javascript:void(0)><\/script>"); + var script = document.getElementById("__ie_onload"); + script.onreadystatechange = function() { + if (this.readyState == "complete") { + sorttable.init(); // call the onload handler + } + }; +/*@end @*/ + +/* for Safari */ +if (/WebKit/i.test(navigator.userAgent)) { // sniff + var _timer = setInterval(function() { + if (/loaded|complete/.test(document.readyState)) { + sorttable.init(); // call the onload handler + } + }, 10); +} + +/* for other browsers */ +window.onload = sorttable.init; + +// written by Dean Edwards, 2005 +// with input from Tino Zijdel, Matthias Miller, Diego Perini + +// http://dean.edwards.name/weblog/2005/10/add-event/ + +function dean_addEvent(element, type, handler) { + if (element.addEventListener) { + element.addEventListener(type, handler, false); + } else { + // assign each event handler a unique ID + if (!handler.$$guid) handler.$$guid = dean_addEvent.guid++; + // create a hash table of event types for the element + if (!element.events) element.events = {}; + // create a hash table of event handlers for each element/event pair + var handlers = element.events[type]; + if (!handlers) { + handlers = element.events[type] = {}; + // store the existing event handler (if there is one) + if (element["on" + type]) { + handlers[0] = element["on" + type]; + } + } + // store the event handler in the hash table + handlers[handler.$$guid] = handler; + // assign a global event handler to do all the work + element["on" + type] = handleEvent; + } +}; +// a counter used to create unique IDs +dean_addEvent.guid = 1; + +function removeEvent(element, type, handler) { + if (element.removeEventListener) { + element.removeEventListener(type, handler, false); + } else { + // delete the event handler from the hash table + if (element.events && element.events[type]) { + delete element.events[type][handler.$$guid]; + } + } +}; + +function handleEvent(event) { + var returnValue = true; + // grab the event object (IE uses a global event object) + event = event || fixEvent(((this.ownerDocument || this.document || this).parentWindow || window).event); + // get a reference to the hash table of event handlers + var handlers = this.events[event.type]; + // execute each event handler + for (var i in handlers) { + this.$$handleEvent = handlers[i]; + if (this.$$handleEvent(event) === false) { + returnValue = false; + } + } + return returnValue; +}; + +function fixEvent(event) { + // add W3C standard event methods + event.preventDefault = fixEvent.preventDefault; + event.stopPropagation = fixEvent.stopPropagation; + return event; +}; +fixEvent.preventDefault = function() { + this.returnValue = false; +}; +fixEvent.stopPropagation = function() { + this.cancelBubble = true; +} + +// Dean's forEach: http://dean.edwards.name/base/forEach.js +/* + forEach, version 1.0 + Copyright 2006, Dean Edwards + License: http://www.opensource.org/licenses/mit-license.php +*/ + +// array-like enumeration +if (!Array.forEach) { // mozilla already supports this + Array.forEach = function(array, block, context) { + for (var i = 0; i < array.length; i++) { + block.call(context, array[i], i, array); + } + }; +} + +// generic enumeration +Function.prototype.forEach = function(object, block, context) { + for (var key in object) { + if (typeof this.prototype[key] == "undefined") { + block.call(context, object[key], key, object); + } + } +}; + +// character enumeration +String.forEach = function(string, block, context) { + Array.forEach(string.split(""), function(chr, index) { + block.call(context, chr, index, string); + }); +}; + +// globally resolve forEach enumeration +var forEach = function(object, block, context) { + if (object) { + var resolve = Object; // default + if (object instanceof Function) { + // functions have a "length" property + resolve = Function; + } else if (object.forEach instanceof Function) { + // the object implements a custom forEach method so use that + object.forEach(block, context); + return; + } else if (typeof object == "string") { + // the object is a string + resolve = String; + } else if (typeof object.length == "number") { + // the object is array-like + resolve = Array; + } + resolve.forEach(object, block, context); + } +}; diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index da63c933d3f..1d302409f2e 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -8,6 +8,7 @@ Adler's Atomicity BLOBs BLRrVv +CAS CFLAGS CMake COV @@ -108,9 +109,11 @@ PPC PRELOAD PROFDATA README +RLE RedHat RepMgr Riak +RLE RocksDB Roelofs RrVv @@ -205,6 +208,7 @@ clickable cmake colgroup colgroups +colspan combinatorial command's comparator @@ -228,6 +232,7 @@ cryptographically csuite ctest curfile +curhs cursortype curtable customerABC @@ -267,6 +272,7 @@ dl dll dlp dontlock +doxtable dp ds dsrc @@ -316,6 +322,7 @@ fillfactor firstname flamegraph flamegraphs +flcs fnv forw fprofile @@ -362,6 +369,7 @@ insn intl intpack inuse +invariants io ip je @@ -515,6 +523,7 @@ putKey putKeyString putValue putValueString +px py qnx qqq @@ -565,6 +574,7 @@ sess sid skinparam skiplist +sortable spinlock spinlocks sql @@ -642,6 +652,7 @@ valuev vec versa viewable +vlcs vm vpmsum warmup diff --git a/src/third_party/wiredtiger/src/docs/style/header-web.html b/src/third_party/wiredtiger/src/docs/style/header-web.html index 63b22bb763e..ec15f09bba3 100644 --- a/src/third_party/wiredtiger/src/docs/style/header-web.html +++ b/src/third_party/wiredtiger/src/docs/style/header-web.html @@ -8,6 +8,7 @@ <link href="$relpath$tabs.css" rel="stylesheet" type="text/css"/> <script type="text/javascript" src="$relpath$jquery.js"></script> <script type="text/javascript" src="$relpath$dynsections.js"></script> +<script type="text/javascript" src="$relpath$sorttable.js"></script> $treeview $search $mathjax diff --git a/src/third_party/wiredtiger/src/docs/style/header.html b/src/third_party/wiredtiger/src/docs/style/header.html index 422268b7a47..559e3e690da 100644 --- a/src/third_party/wiredtiger/src/docs/style/header.html +++ b/src/third_party/wiredtiger/src/docs/style/header.html @@ -8,6 +8,7 @@ <link href="$relpath$tabs.css" rel="stylesheet" type="text/css"/> <script type="text/javascript" src="$relpath$jquery.js"></script> <script type="text/javascript" src="$relpath$dynsections.js"></script> +<script type="text/javascript" src="$relpath$sorttable.js"></script> $treeview $search $mathjax diff --git a/src/third_party/wiredtiger/src/docs/tune-durability.dox b/src/third_party/wiredtiger/src/docs/tune-durability.dox index a16c2938d9c..5350c617d3b 100644 --- a/src/third_party/wiredtiger/src/docs/tune-durability.dox +++ b/src/third_party/wiredtiger/src/docs/tune-durability.dox @@ -79,12 +79,6 @@ by the \c transaction_sync=(method) setting before returning. If \c sync=off is configured then this commit operation will write its records into the in-memory buffer and return immediately. -If \c sync=background is configured then this commit operation will write -its record to an in-memory buffer, and will return. Prior to returning it -will signal an internal WiredTiger worker thread to synchronize this log -record. The caller may then check the status of that background -synchronization with the WT_SESSION::transaction_sync method. - The durability of the write-ahead log can be controlled independently as well via the WT_SESSION::log_flush method. The WT_SESSION::log_flush supports several durability modes with @@ -99,10 +93,6 @@ If \c sync=off is configured then this flush operation will force the logging subsystem to write any outstanding in-memory buffers to the file system before returning. -If \c sync=background is configured then this flush operation will force -the signaling of a background synchronization operation. -The caller may then check the status of that background -synchronization with the WT_SESSION::transaction_sync method. */ /*! @page tune_durability Commit-level durability diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index e92dde30464..c5f4b2d013e 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -1,6 +1,16 @@ /*! @page upgrading Upgrading WiredTiger applications </dl><hr> +@section version_1001 Upgrading to Version 10.0.1 +<dl> + +<dt>Background sync APIs</dt> +<dd> +The WT_SESSION::transaction_sync API and the sync=background configuration option to +WT_SESSION::log_flush and WT_SESSION::commit_transaction have all been removed from WiredTiger. +</dd> + +</dl><hr> @section version_1000 Upgrading to Version 10.0.0 <dl> diff --git a/src/third_party/wiredtiger/src/include/btree_inline.h b/src/third_party/wiredtiger/src/include/btree_inline.h index 6e2add93605..adcf81b83bb 100644 --- a/src/third_party/wiredtiger/src/include/btree_inline.h +++ b/src/third_party/wiredtiger/src/include/btree_inline.h @@ -724,6 +724,13 @@ static inline void __wt_page_modify_set(WT_SESSION_IMPL *session, WT_PAGE *page) { /* + * Prepared records in the datastore require page updates, even for read-only handles, don't + * mark the tree or page dirty. + */ + if (F_ISSET(S2BT(session), WT_BTREE_READONLY)) + return; + + /* * Mark the tree dirty (even if the page is already marked dirty), newly created pages to * support "empty" files are dirty, but the file isn't marked dirty until there's a real change * needing to be written. @@ -1992,34 +1999,6 @@ __wt_page_swap_func(WT_SESSION_IMPL *session, WT_REF *held, WT_REF *want, uint32 } /* - * __wt_bt_col_var_cursor_walk_txn_read -- - * Set the value for variable-length column-store cursor walk. - */ -static inline int -__wt_bt_col_var_cursor_walk_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page, - WT_CELL_UNPACK_KV *unpack, WT_COL *cip) -{ - WT_UPDATE *upd; - - upd = NULL; - - if (cbt->ins) - upd = cbt->ins->upd; - - cbt->slot = WT_COL_SLOT(page, cip); - WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, upd, unpack)); - if (cbt->upd_value->type == WT_UPDATE_INVALID || cbt->upd_value->type == WT_UPDATE_TOMBSTONE) - return (0); - - WT_RET(__wt_value_return(cbt, cbt->upd_value)); - - cbt->tmp->data = cbt->iface.value.data; - cbt->tmp->size = cbt->iface.value.size; - cbt->cip_saved = cip; - return (0); -} - -/* * __wt_btcur_skip_page -- * Return if the cursor is pointing to a page with deleted records and can be skipped for cursor * traversal. diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 3b092857ed3..29c576defd8 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -91,23 +91,22 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_WT_SESSION_salvage 37 #define WT_CONFIG_ENTRY_WT_SESSION_strerror 38 #define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 39 -#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 40 -#define WT_CONFIG_ENTRY_WT_SESSION_truncate 41 -#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 42 -#define WT_CONFIG_ENTRY_WT_SESSION_verify 43 -#define WT_CONFIG_ENTRY_colgroup_meta 44 -#define WT_CONFIG_ENTRY_file_config 45 -#define WT_CONFIG_ENTRY_file_meta 46 -#define WT_CONFIG_ENTRY_index_meta 47 -#define WT_CONFIG_ENTRY_lsm_meta 48 -#define WT_CONFIG_ENTRY_object_meta 49 -#define WT_CONFIG_ENTRY_table_meta 50 -#define WT_CONFIG_ENTRY_tier_meta 51 -#define WT_CONFIG_ENTRY_tiered_meta 52 -#define WT_CONFIG_ENTRY_wiredtiger_open 53 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 54 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 55 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 56 +#define WT_CONFIG_ENTRY_WT_SESSION_truncate 40 +#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 41 +#define WT_CONFIG_ENTRY_WT_SESSION_verify 42 +#define WT_CONFIG_ENTRY_colgroup_meta 43 +#define WT_CONFIG_ENTRY_file_config 44 +#define WT_CONFIG_ENTRY_file_meta 45 +#define WT_CONFIG_ENTRY_index_meta 46 +#define WT_CONFIG_ENTRY_lsm_meta 47 +#define WT_CONFIG_ENTRY_object_meta 48 +#define WT_CONFIG_ENTRY_table_meta 49 +#define WT_CONFIG_ENTRY_tier_meta 50 +#define WT_CONFIG_ENTRY_tiered_meta 51 +#define WT_CONFIG_ENTRY_wiredtiger_open 52 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 53 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 54 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 55 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index 9425292ad8a..569da6aabaa 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -154,7 +154,10 @@ struct __wt_cursor_btree { * Variable-length column-store values are run-length encoded and may be overflow values or * Huffman encoded. To avoid repeatedly reading overflow values or decompressing encoded values, * process it once and store the result in a temporary buffer. The cip_saved field is used to - * determine if we've switched columns since our last cursor call. + * determine if we've switched columns since our last cursor call. Note however that this result + * caching is not necessarily safe for all RLE cells. The flag WT_CBT_CACHEABLE_RLE_CELL + * indicates that the value is uniform across the whole cell. If it is not set (e.g. if the cell + * is not globally visible yet), the cached values should not be used. */ WT_COL *cip_saved; /* Last iteration reference */ @@ -214,16 +217,17 @@ struct __wt_cursor_btree { /* AUTOMATIC FLAG VALUE GENERATION START 0 */ #define WT_CBT_ACTIVE 0x001u /* Active in the tree */ -#define WT_CBT_ITERATE_APPEND 0x002u /* Col-store: iterating append list */ -#define WT_CBT_ITERATE_NEXT 0x004u /* Next iteration configuration */ -#define WT_CBT_ITERATE_PREV 0x008u /* Prev iteration configuration */ -#define WT_CBT_ITERATE_RETRY_NEXT 0x010u /* Prepare conflict by next. */ -#define WT_CBT_ITERATE_RETRY_PREV 0x020u /* Prepare conflict by prev. */ -#define WT_CBT_NO_TRACKING 0x040u /* Non tracking cursor. */ -#define WT_CBT_NO_TXN 0x080u /* Non-txn cursor (e.g. a checkpoint) */ -#define WT_CBT_READ_ONCE 0x100u /* Page in with WT_READ_WONT_NEED */ -#define WT_CBT_SEARCH_SMALLEST 0x200u /* Row-store: small-key insert list */ -#define WT_CBT_VAR_ONPAGE_MATCH 0x400u /* Var-store: on-page recno match */ +#define WT_CBT_CACHEABLE_RLE_CELL 0x002u /* Col-store: value in RLE cell valid for all its keys */ +#define WT_CBT_ITERATE_APPEND 0x004u /* Col-store: iterating append list */ +#define WT_CBT_ITERATE_NEXT 0x008u /* Next iteration configuration */ +#define WT_CBT_ITERATE_PREV 0x010u /* Prev iteration configuration */ +#define WT_CBT_ITERATE_RETRY_NEXT 0x020u /* Prepare conflict by next. */ +#define WT_CBT_ITERATE_RETRY_PREV 0x040u /* Prepare conflict by prev. */ +#define WT_CBT_NO_TRACKING 0x080u /* Non tracking cursor. */ +#define WT_CBT_NO_TXN 0x100u /* Non-txn cursor (e.g. a checkpoint) */ +#define WT_CBT_READ_ONCE 0x200u /* Page in with WT_READ_WONT_NEED */ +#define WT_CBT_SEARCH_SMALLEST 0x400u /* Row-store: small-key insert list */ +#define WT_CBT_VAR_ONPAGE_MATCH 0x800u /* Var-store: on-page recno match */ /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ #define WT_CBT_POSITION_MASK /* Flags associated with position */ \ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index ef7ac6d6929..68b89d99b0e 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -1175,7 +1175,9 @@ extern int __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t fla #endif ) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, const void *image, uint32_t flags, - WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); + WT_PAGE **pagep, bool *preparedp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_page_inmem_prepare(WT_SESSION_IMPL *session, WT_REF *ref) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) @@ -1760,7 +1762,6 @@ extern void __wt_hs_close(WT_SESSION_IMPL *session); extern void __wt_hs_upd_time_window(WT_CURSOR *hs_cursor, WT_TIME_WINDOW **twp); extern void __wt_huffman_close(WT_SESSION_IMPL *session, void *huffman_arg); extern void __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor); -extern void __wt_log_background(WT_SESSION_IMPL *session, WT_LSN *lsn); extern void __wt_log_ckpt(WT_SESSION_IMPL *session, WT_LSN *ckpt_lsn); extern void __wt_log_slot_activate(WT_SESSION_IMPL *session, WT_LOGSLOT *slot); extern void __wt_log_slot_free(WT_SESSION_IMPL *session, WT_LOGSLOT *slot); @@ -1966,9 +1967,6 @@ static inline bool __wt_txn_visible_all(WT_SESSION_IMPL *session, uint64_t id, wt_timestamp_t timestamp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline double __wt_eviction_dirty_target(WT_CACHE *cache) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline int __wt_bt_col_var_cursor_walk_txn_read(WT_SESSION_IMPL *session, - WT_CURSOR_BTREE *cbt, WT_PAGE *page, WT_CELL_UNPACK_KV *unpack, WT_COL *cip) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_btree_block_free(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_buf_extend(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) @@ -2141,8 +2139,7 @@ static inline int __wt_txn_modify_page_delete(WT_SESSION_IMPL *session, WT_REF * static inline int __wt_txn_op_set_key(WT_SESSION_IMPL *session, const WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, - uint64_t recno, WT_UPDATE *upd, WT_CELL_UNPACK_KV *vpack) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); + uint64_t recno, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_read_upd_list(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_read_upd_list_internal(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index 132a1512b37..641c91babc8 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -15,11 +15,10 @@ /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ /* AUTOMATIC FLAG VALUE GENERATION START 0 */ -#define WT_LOG_BACKGROUND 0x01u -#define WT_LOG_DSYNC 0x02u -#define WT_LOG_FLUSH 0x04u -#define WT_LOG_FSYNC 0x08u -#define WT_LOG_SYNC_ENABLED 0x10u +#define WT_LOG_DSYNC 0x1u +#define WT_LOG_FLUSH 0x2u +#define WT_LOG_FSYNC 0x4u +#define WT_LOG_SYNC_ENABLED 0x8u /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ #define WT_LOGOP_IGNORE 0x80000000 @@ -257,7 +256,6 @@ struct __wt_log { * System LSNs */ WT_LSN alloc_lsn; /* Next LSN for allocation */ - WT_LSN bg_sync_lsn; /* Latest background sync LSN */ WT_LSN ckpt_lsn; /* Last checkpoint LSN */ WT_LSN dirty_lsn; /* LSN of last non-synced write */ WT_LSN first_lsn; /* First LSN */ diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 1bddc6a193e..465f641954b 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -138,9 +138,6 @@ struct __wt_session_impl { WT_TXN_ISOLATION isolation; WT_TXN *txn; /* Transaction state */ -#define WT_SESSION_BG_SYNC_MSEC 1200000 - WT_LSN bg_sync_lsn; /* Background sync operation LSN. */ - void *block_manager; /* Block-manager support */ int (*block_manager_cleanup)(WT_SESSION_IMPL *); @@ -200,17 +197,16 @@ struct __wt_session_impl { #define WT_SESSION_IGNORE_CACHE_SIZE 0x00020u #define WT_SESSION_IMPORT 0x00040u #define WT_SESSION_IMPORT_REPAIR 0x00080u -#define WT_SESSION_INSTANTIATE_PREPARE 0x00100u -#define WT_SESSION_INTERNAL 0x00200u -#define WT_SESSION_LOGGING_INMEM 0x00400u -#define WT_SESSION_NO_DATA_HANDLES 0x00800u -#define WT_SESSION_NO_LOGGING 0x01000u -#define WT_SESSION_NO_RECONCILE 0x02000u -#define WT_SESSION_QUIET_CORRUPT_FILE 0x04000u -#define WT_SESSION_READ_WONT_NEED 0x08000u -#define WT_SESSION_RESOLVING_TXN 0x10000u -#define WT_SESSION_ROLLBACK_TO_STABLE 0x20000u -#define WT_SESSION_SCHEMA_TXN 0x40000u +#define WT_SESSION_INTERNAL 0x00100u +#define WT_SESSION_LOGGING_INMEM 0x00200u +#define WT_SESSION_NO_DATA_HANDLES 0x00400u +#define WT_SESSION_NO_LOGGING 0x00800u +#define WT_SESSION_NO_RECONCILE 0x01000u +#define WT_SESSION_QUIET_CORRUPT_FILE 0x02000u +#define WT_SESSION_READ_WONT_NEED 0x04000u +#define WT_SESSION_RESOLVING_TXN 0x08000u +#define WT_SESSION_ROLLBACK_TO_STABLE 0x10000u +#define WT_SESSION_SCHEMA_TXN 0x20000u /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 8aa75787f99..893179db355 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -717,7 +717,6 @@ struct __wt_connection_stats { int64_t conn_close_blocked_lsm; int64_t dhandle_lock_blocked; int64_t page_index_slot_ref_blocked; - int64_t log_server_sync_blocked; int64_t prepared_transition_blocked_page; int64_t page_busy_blocked; int64_t page_forcible_evict_blocked; @@ -792,7 +791,6 @@ struct __wt_connection_stats { int64_t txn_pinned_timestamp_oldest; int64_t txn_timestamp_oldest_active_read; int64_t txn_rollback_to_stable_running; - int64_t txn_sync; int64_t txn_walk_sessions; int64_t txn_commit; int64_t txn_rollback; diff --git a/src/third_party/wiredtiger/src/include/time_inline.h b/src/third_party/wiredtiger/src/include/time_inline.h index 5b7c28da5fb..611e38eab24 100644 --- a/src/third_party/wiredtiger/src/include/time_inline.h +++ b/src/third_party/wiredtiger/src/include/time_inline.h @@ -29,6 +29,13 @@ __wt_rdtsc(void) __asm__ volatile("rdtsc" : "=a"(a), "=d"(d)); return ((d << 32) | a); } +#elif defined(__aarch64__) + { + uint64_t t; + + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(t)); + return (t); + } #else return (0); #endif diff --git a/src/third_party/wiredtiger/src/include/txn_inline.h b/src/third_party/wiredtiger/src/include/txn_inline.h index 97fea21a14c..df81b29fe23 100644 --- a/src/third_party/wiredtiger/src/include/txn_inline.h +++ b/src/third_party/wiredtiger/src/include/txn_inline.h @@ -973,8 +973,8 @@ __wt_txn_read_upd_list(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE * function will search the history store for a visible update. */ static inline int -__wt_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint64_t recno, - WT_UPDATE *upd, WT_CELL_UNPACK_KV *vpack) +__wt_txn_read( + WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint64_t recno, WT_UPDATE *upd) { WT_TIME_WINDOW tw; WT_UPDATE *prepare_upd, *restored_upd; @@ -1013,14 +1013,7 @@ retry: have_stop_tw = WT_TIME_WINDOW_HAS_STOP(&cbt->upd_value->tw); /* Check the ondisk value. */ - if (vpack == NULL) { - WT_TIME_WINDOW_INIT(&tw); - WT_RET(__wt_value_return_buf(cbt, cbt->ref, &cbt->upd_value->buf, &tw)); - } else { - WT_TIME_WINDOW_COPY(&tw, &vpack->tw); - cbt->upd_value->buf.data = vpack->data; - cbt->upd_value->buf.size = vpack->size; - } + WT_RET(__wt_value_return_buf(cbt, cbt->ref, &cbt->upd_value->buf, &tw)); /* * If the stop time point is set, that means that there is a tombstone at that time. If it diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 4ca3eeb1f72..2da0d297fb1 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1431,11 +1431,9 @@ struct __wt_session { * @param session the session handle * @configstart{WT_SESSION.log_flush, see dist/api_data.py} * @config{sync, forcibly flush the log and wait for it to achieve the synchronization level - * specified. The \c background setting initiates a background synchronization intended to - * be used with a later call to WT_SESSION::transaction_sync. The \c off setting forces any - * buffered log records to be written to the file system. The \c on setting forces log - * records to be written to the storage device., a string\, chosen from the following - * options: \c "background"\, \c "off"\, \c "on"; default \c on.} + * specified. The \c off setting forces any buffered log records to be written to the file + * system. The \c on setting forces log records to be written to the storage device., a + * string\, chosen from the following options: \c "off"\, \c "on"; default \c on.} * @configend * @errors */ @@ -1719,12 +1717,10 @@ struct __wt_session { * operation may return a WT_ROLLBACK error. Default is to have no limit., an integer * greater than or equal to 1; default \c 0.} * @config{sync, override whether to sync log records when the transaction commits\, - * inherited from ::wiredtiger_open \c transaction_sync. The \c background setting - * initiates a background synchronization intended to be used with a later call to - * WT_SESSION::transaction_sync. The \c off setting does not wait for record to be written - * or synchronized. The \c on setting forces log records to be written to the storage - * device., a string\, chosen from the following options: \c "background"\, \c "off"\, \c - * "on"; default empty.} + * inherited from ::wiredtiger_open \c transaction_sync. The \c off setting does not wait + * for record to be written or synchronized. The \c on setting forces log records to be + * written to the storage device., a string\, chosen from the following options: \c "off"\, + * \c "on"; default empty.} * @configend * @errors */ @@ -1916,26 +1912,6 @@ struct __wt_session { */ int __F(transaction_pinned_range)(WT_SESSION* session, uint64_t *range); - /*! - * Wait for a transaction to become synchronized. This method is - * only useful when ::wiredtiger_open is configured with the - * \c transaction_sync setting disabled. - * - * @requires_notransaction - * - * @snippet ex_all.c Transaction sync - * - * @param session the session handle - * @configstart{WT_SESSION.transaction_sync, see dist/api_data.py} - * @config{timeout_ms, maximum amount of time to wait for background sync to complete in - * milliseconds. A value of zero disables the timeout and returns immediately., an integer; - * default \c 1200000.} - * @configend - * @errors - */ - int __F(transaction_sync)(WT_SESSION *session, const char *config); - /*! @} */ - #ifndef DOXYGEN /*! * Call into the library. @@ -5974,202 +5950,198 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); * (usecs) */ #define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1403 -/*! thread-yield: log server sync yielded for log write */ -#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1404 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1405 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1404 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1406 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1405 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1407 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1406 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1408 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1407 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1409 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1408 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1410 +#define WT_STAT_CONN_PAGE_SLEEP 1409 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1411 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1410 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1412 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1411 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES 1413 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES 1412 /*! transaction: Number of prepared updates committed */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COMMITTED 1414 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COMMITTED 1413 /*! transaction: Number of prepared updates repeated on the same key */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_KEY_REPEATED 1415 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_KEY_REPEATED 1414 /*! transaction: Number of prepared updates rolled back */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_ROLLEDBACK 1416 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_ROLLEDBACK 1415 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1417 +#define WT_STAT_CONN_TXN_PREPARE 1416 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1418 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1417 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1419 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1418 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1420 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1419 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1421 +#define WT_STAT_CONN_TXN_QUERY_TS 1420 /*! transaction: race to read prepared update retry */ -#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1422 +#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1421 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_RTS 1423 +#define WT_STAT_CONN_TXN_RTS 1422 /*! * transaction: rollback to stable history store records with stop * timestamps older than newer records */ -#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1424 +#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1423 /*! transaction: rollback to stable inconsistent checkpoint */ -#define WT_STAT_CONN_TXN_RTS_INCONSISTENT_CKPT 1425 +#define WT_STAT_CONN_TXN_RTS_INCONSISTENT_CKPT 1424 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1426 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1425 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1427 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1426 /*! transaction: rollback to stable pages visited */ -#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1428 +#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1427 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1429 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1428 /*! transaction: rollback to stable restored updates from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1430 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1429 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1431 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1430 /*! transaction: rollback to stable tree walk skipping pages */ -#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1432 +#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1431 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1433 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1432 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1434 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1433 /*! transaction: sessions scanned in each walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1435 +#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1434 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1436 +#define WT_STAT_CONN_TXN_SET_TS 1435 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1437 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1436 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1438 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1437 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1439 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1438 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1440 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1439 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1441 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1440 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1442 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1441 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1443 +#define WT_STAT_CONN_TXN_BEGIN 1442 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1444 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1443 /*! * transaction: transaction checkpoint currently running for history * store file */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1445 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1444 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1446 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1445 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1447 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1446 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1448 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1447 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1449 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1448 /*! * transaction: transaction checkpoint most recent duration for gathering * all handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1450 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1449 /*! * transaction: transaction checkpoint most recent duration for gathering * applied handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1451 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1450 /*! * transaction: transaction checkpoint most recent duration for gathering * skipped handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1452 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1451 /*! transaction: transaction checkpoint most recent handles applied */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1453 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1452 /*! transaction: transaction checkpoint most recent handles skipped */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1454 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1453 /*! transaction: transaction checkpoint most recent handles walked */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1455 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1454 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1456 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1455 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1457 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1456 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1458 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1457 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1459 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1458 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1460 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1459 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1461 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1460 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1462 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1461 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1463 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1462 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1464 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1463 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1465 +#define WT_STAT_CONN_TXN_CHECKPOINT 1464 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1466 +#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1465 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1467 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1466 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1468 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1467 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1469 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1468 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1470 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1469 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1471 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1470 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1472 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1471 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1473 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1472 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1474 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1473 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1475 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1474 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1476 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1475 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1477 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1476 /*! transaction: transaction rollback to stable currently running */ -#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1478 -/*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1479 +#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1477 /*! transaction: transaction walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_WALK_SESSIONS 1480 +#define WT_STAT_CONN_TXN_WALK_SESSIONS 1478 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1481 +#define WT_STAT_CONN_TXN_COMMIT 1479 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1482 +#define WT_STAT_CONN_TXN_ROLLBACK 1480 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1483 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1481 /*! * @} diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index bfa252ce4b8..6b136c6cc84 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -269,35 +269,6 @@ __wt_log_flush_lsn(WT_SESSION_IMPL *session, WT_LSN *lsn, bool start) } /* - * __wt_log_background -- - * Record the given LSN as the background LSN and signal the thread as needed. - */ -void -__wt_log_background(WT_SESSION_IMPL *session, WT_LSN *lsn) -{ - WT_CONNECTION_IMPL *conn; - WT_LOG *log; - - conn = S2C(session); - log = conn->log; - /* - * If a thread already set the LSN to a bigger LSN, we're done. - */ - if (__wt_log_cmp(&session->bg_sync_lsn, lsn) > 0) - return; - WT_ASSIGN_LSN(&session->bg_sync_lsn, lsn); - - /* - * Advance the logging subsystem background sync LSN if needed. - */ - __wt_spin_lock(session, &log->log_sync_lock); - if (__wt_log_cmp(lsn, &log->bg_sync_lsn) > 0) - WT_ASSIGN_LSN(&log->bg_sync_lsn, lsn); - __wt_spin_unlock(session, &log->log_sync_lock); - __wt_cond_signal(session, conn->log_file_cond); -} - -/* * __wt_log_force_sync -- * Force a sync of the log and files. */ @@ -2715,12 +2686,6 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, ui __wt_cond_wait(session, log->log_sync_cond, 10000, NULL); } - /* - * Advance the background sync LSN if needed. - */ - if (LF_ISSET(WT_LOG_BACKGROUND)) - __wt_log_background(session, &lsn); - err: if (ret == 0 && lsnp != NULL) *lsnp = lsn; @@ -2836,9 +2801,7 @@ __wt_log_flush(WT_SESSION_IMPL *session, uint32_t flags) * If the user wants write-no-sync, there is nothing more to do. If the user wants background * sync, set the LSN and we're done. If the user wants sync, force it now. */ - if (LF_ISSET(WT_LOG_BACKGROUND)) - __wt_log_background(session, &lsn); - else if (LF_ISSET(WT_LOG_FSYNC)) + if (LF_ISSET(WT_LOG_FSYNC)) WT_RET(__wt_log_force_sync(session, &lsn)); return (0); } diff --git a/src/third_party/wiredtiger/src/log/log_sys.c b/src/third_party/wiredtiger/src/log/log_sys.c index a922cb9b56a..56a831f7275 100644 --- a/src/third_party/wiredtiger/src/log/log_sys.c +++ b/src/third_party/wiredtiger/src/log/log_sys.c @@ -128,7 +128,6 @@ __wt_verbose_dump_log(WT_SESSION_IMPL *session) WT_RET(__wt_msg(session, "Current log file number: %" PRIu32, log->fileid)); WT_RET(__wt_msg(session, "Current log version number: %" PRIu16, log->log_version)); WT_RET(WT_LSN_MSG(&log->alloc_lsn, "Next allocation")); - WT_RET(WT_LSN_MSG(&log->bg_sync_lsn, "Last background sync")); WT_RET(WT_LSN_MSG(&log->ckpt_lsn, "Last checkpoint")); WT_RET(WT_LSN_MSG(&log->sync_dir_lsn, "Last directory sync")); WT_RET(WT_LSN_MSG(&log->sync_lsn, "Last sync")); diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index 417bafebb50..9518a3b63c3 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -573,17 +573,20 @@ __wt_rec_col_var( WT_CELL *cell; WT_CELL_UNPACK_KV *vpack, _vpack; WT_COL *cip; + WT_CURSOR *hs_cursor; WT_CURSOR_BTREE *cbt; WT_DECL_ITEM(orig); WT_DECL_RET; WT_INSERT *ins; + WT_ITEM hs_recno_key; WT_PAGE *page; WT_TIME_WINDOW clear_tw, *twp; WT_UPDATE *upd; WT_UPDATE_SELECT upd_select; uint64_t n, nrepeat, repeat_count, rle, skip, src_recno; uint32_t i, size; - bool deleted, orig_deleted, update_no_copy; + uint8_t *p, hs_recno_key_buf[WT_INTPACK64_MAXSIZE]; + bool deleted, hs_clear, orig_deleted, update_no_copy; const void *data; btree = S2BT(session); @@ -603,6 +606,23 @@ __wt_rec_col_var( WT_TIME_WINDOW_INIT(&last.tw); last.deleted = false; + /* + * When removing a key due to a tombstone with a durable timestamp of "none", also remove the + * history store contents associated with that key. It's safe to do even if we fail + * reconciliation after the removal, the history store content must be obsolete in order for us + * to consider removing the key. + * + * Ignore if this is metadata, as metadata doesn't have any history. + * + * Some code paths, such as schema removal, involve deleting keys in metadata and assert that + * they shouldn't open new dhandles. In those cases we won't ever need to blow away history + * store content, so we can skip this. + */ + hs_cursor = NULL; + hs_clear = F_ISSET(S2C(session), WT_CONN_HS_OPEN) && + !F_ISSET(session, WT_SESSION_NO_DATA_HANDLES) && !WT_IS_HS(btree->dhandle) && + !WT_IS_METADATA(btree->dhandle); + WT_RET(__wt_rec_split_init(session, r, page, pageref->ref_recno, btree->maxleafpage_precomp)); WT_RET(__wt_scr_alloc(session, 0, &orig)); @@ -789,8 +809,30 @@ record_loop: size = upd->size; break; case WT_UPDATE_TOMBSTONE: - twp = &clear_tw; + /* + * When removing a key due to a tombstone with a durable timestamp of "none", + * also remove the history store contents associated with that key. + */ + if (twp->durable_stop_ts == WT_TS_NONE && hs_clear) { + p = hs_recno_key_buf; + WT_ERR(__wt_vpack_uint(&p, 0, src_recno)); + hs_recno_key.data = hs_recno_key_buf; + hs_recno_key.size = WT_PTRDIFF(p, hs_recno_key_buf); + + /* Open a history store cursor if we don't yet have one. */ + if (hs_cursor == NULL) + WT_ERR(__wt_curhs_open(session, NULL, &hs_cursor)); + + /* From WT_TS_NONE to delete all the history store content of the key. */ + WT_ERR(__wt_hs_delete_key_from_ts(session, hs_cursor, btree->id, + &hs_recno_key, WT_TS_NONE, false, F_ISSET(r, WT_REC_CHECKPOINT_RUNNING))); + + WT_STAT_CONN_INCR(session, cache_hs_key_truncate_onpage_removal); + WT_STAT_DATA_INCR(session, cache_hs_key_truncate_onpage_removal); + } + deleted = true; + twp = &clear_tw; break; default: WT_ERR(__wt_illegal_value(session, upd->type)); @@ -799,19 +841,21 @@ record_loop: compare: /* - * If we have a record against which to compare, and the records compare equal, - * increment the rle counter and continue. If the records don't compare equal, output - * the last record and swap the last and current buffers: do NOT update the starting - * record number, we've been doing that all along. + * If we have a record against which to compare and the records compare equal, increment + * the RLE and continue. If the records don't compare equal, output the last record and + * swap the last and current buffers: do NOT update the starting record number, we've + * been doing that all along. */ if (rle != 0) { - if (WT_TIME_WINDOWS_EQUAL(twp, &last.tw) && + if (WT_TIME_WINDOWS_EQUAL(&last.tw, twp) && ((deleted && last.deleted) || (!deleted && !last.deleted && last.value->size == size && - memcmp(last.value->data, data, size) == 0))) { + (size == 0 || memcmp(last.value->data, data, size) == 0)))) { + /* The time window for deleted keys must be empty. */ WT_ASSERT( session, (!deleted && !last.deleted) || WT_TIME_WINDOW_IS_EMPTY(&last.tw)); + rle += repeat_count; continue; } @@ -951,13 +995,13 @@ compare: if (rle != 0) { if (WT_TIME_WINDOWS_EQUAL(&last.tw, twp) && ((deleted && last.deleted) || - (!deleted && !last.deleted && size != 0 && last.value->size == size && - memcmp(last.value->data, data, size) == 0))) { - /* - * The time window for deleted keys must be empty. - */ + (!deleted && !last.deleted && last.value->size == size && + (size == 0 || memcmp(last.value->data, data, size) == 0)))) { + + /* The time window for deleted keys must be empty. */ WT_ASSERT( session, (!deleted && !last.deleted) || WT_TIME_WINDOW_IS_EMPTY(&last.tw)); + ++rle; goto next; } @@ -1012,6 +1056,8 @@ next: ret = __wt_rec_split_finish(session, r); err: + if (hs_cursor != NULL) + WT_TRET(hs_cursor->close(hs_cursor)); __wt_scr_free(session, &orig); return (ret); } diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 513b466d0a9..62464af3c6a 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -203,8 +203,6 @@ __session_clear(WT_SESSION_IMPL *session) */ memset(session, 0, WT_SESSION_CLEAR_SIZE); - WT_INIT_LSN(&session->bg_sync_lsn); - session->hazard_inuse = 0; session->nhazard = 0; } @@ -808,9 +806,7 @@ __session_log_flush(WT_SESSION *wt_session, const char *config) WT_ERR_MSG(session, EINVAL, "logging not enabled"); WT_ERR(__wt_config_gets_def(session, cfg, "sync", 0, &cval)); - if (WT_STRING_MATCH("background", cval.str, cval.len)) - flags = WT_LOG_BACKGROUND; - else if (WT_STRING_MATCH("off", cval.str, cval.len)) + if (WT_STRING_MATCH("off", cval.str, cval.len)) flags = WT_LOG_FLUSH; else if (WT_STRING_MATCH("on", cval.str, cval.len)) flags = WT_LOG_FSYNC; @@ -1783,114 +1779,6 @@ err: } /* - * __transaction_sync_run_chk -- - * Check to decide if the transaction sync call should continue running. - */ -static bool -__transaction_sync_run_chk(WT_SESSION_IMPL *session) -{ - WT_CONNECTION_IMPL *conn; - - conn = S2C(session); - - return (FLD_ISSET(conn->server_flags, WT_CONN_SERVER_LOG)); -} - -/* - * __session_transaction_sync -- - * WT_SESSION->transaction_sync method. - */ -static int -__session_transaction_sync(WT_SESSION *wt_session, const char *config) -{ - WT_CONFIG_ITEM cval; - WT_CONNECTION_IMPL *conn; - WT_DECL_RET; - WT_LOG *log; - WT_SESSION_IMPL *session; - uint64_t remaining_usec, timeout_ms, waited_ms; - uint64_t time_start, time_stop; - - session = (WT_SESSION_IMPL *)wt_session; - SESSION_API_CALL_PREPARE_NOT_ALLOWED(session, transaction_sync, config, cfg); - WT_STAT_CONN_INCR(session, txn_sync); - - conn = S2C(session); - WT_ERR(__wt_txn_context_check(session, false)); - - /* - * If logging is not enabled there is nothing to do. - */ - if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) - WT_ERR_MSG(session, EINVAL, "logging not enabled"); - - log = conn->log; - - /* - * If there is no background sync LSN in this session, there is nothing to do. - */ - if (WT_IS_INIT_LSN(&session->bg_sync_lsn)) - goto err; - - /* - * If our LSN is smaller than the current sync LSN then our transaction is stable. We're done. - */ - if (__wt_log_cmp(&session->bg_sync_lsn, &log->sync_lsn) <= 0) - goto err; - - /* - * Our LSN is not yet stable. Wait and check again depending on the timeout. - */ - WT_ERR(__wt_config_gets_def(session, cfg, "timeout_ms", (int)WT_SESSION_BG_SYNC_MSEC, &cval)); - timeout_ms = (uint64_t)cval.val; - - if (timeout_ms == 0) - WT_ERR(ETIMEDOUT); - - /* - * Keep checking the LSNs until we find it is stable or we reach our timeout, or there's some - * other reason to quit. - */ - time_start = __wt_clock(session); - while (__wt_log_cmp(&session->bg_sync_lsn, &log->sync_lsn) > 0) { - if (!__transaction_sync_run_chk(session)) - WT_ERR(ETIMEDOUT); - - __wt_cond_signal(session, conn->log_file_cond); - time_stop = __wt_clock(session); - waited_ms = WT_CLOCKDIFF_MS(time_stop, time_start); - if (waited_ms < timeout_ms) { - remaining_usec = (timeout_ms - waited_ms) * WT_THOUSAND; - __wt_cond_wait(session, log->log_sync_cond, remaining_usec, __transaction_sync_run_chk); - } else - WT_ERR(ETIMEDOUT); - } - -err: - API_END_RET(session, ret); -} - -/* - * __session_transaction_sync_readonly -- - * WT_SESSION->transaction_sync method; readonly version. - */ -static int -__session_transaction_sync_readonly(WT_SESSION *wt_session, const char *config) -{ - WT_DECL_RET; - WT_SESSION_IMPL *session; - - WT_UNUSED(config); - - session = (WT_SESSION_IMPL *)wt_session; - SESSION_API_CALL_NOCONF(session, transaction_sync); - - ret = __wt_session_notsup(session); -err: - API_END_RET(session, ret); -} - -/* * __session_checkpoint -- * WT_SESSION->checkpoint method. */ @@ -2009,8 +1897,7 @@ __open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const __session_truncate, __session_upgrade, __session_verify, __session_begin_transaction, __session_commit_transaction, __session_prepare_transaction, __session_reset_snapshot, __session_rollback_transaction, __session_timestamp_transaction, __session_query_timestamp, - __session_checkpoint, __session_transaction_pinned_range, __session_transaction_sync, - __wt_session_breakpoint}, + __session_checkpoint, __session_transaction_pinned_range, __wt_session_breakpoint}, stds_readonly = {NULL, NULL, __session_close, __session_reconfigure, __session_flush_tier, __wt_session_strerror, __session_open_cursor, __session_alter_readonly, __session_create_readonly, __wt_session_compact_readonly, __session_drop_readonly, @@ -2020,8 +1907,7 @@ __open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const __session_begin_transaction, __session_commit_transaction, __session_prepare_transaction_readonly, __session_reset_snapshot, __session_rollback_transaction, __session_timestamp_transaction, __session_query_timestamp, - __session_checkpoint_readonly, __session_transaction_pinned_range, - __session_transaction_sync_readonly, __wt_session_breakpoint}; + __session_checkpoint_readonly, __session_transaction_pinned_range, __wt_session_breakpoint}; WT_DECL_RET; WT_SESSION_IMPL *session, *session_ret; uint32_t i; diff --git a/src/third_party/wiredtiger/src/support/global.c b/src/third_party/wiredtiger/src/support/global.c index 10bb0e93c7a..dc038c2b53c 100644 --- a/src/third_party/wiredtiger/src/support/global.c +++ b/src/third_party/wiredtiger/src/support/global.c @@ -52,7 +52,7 @@ __global_calibrate_ticks(void) __wt_process.tsc_nsec_ratio = WT_TSC_DEFAULT_RATIO; __wt_process.use_epochtime = true; -#if defined(__i386) || defined(__amd64) +#if defined(__i386) || defined(__amd64) || defined(__aarch64__) { struct timespec start, stop; double ratio; diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 9ec45ccd6ba..bc13dd8aab7 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -1392,7 +1392,6 @@ static const char *const __stats_connection_desc[] = { "thread-yield: connection close yielded for lsm manager shutdown", "thread-yield: data handle lock yielded", "thread-yield: get reference for page index and slot time sleeping (usecs)", - "thread-yield: log server sync yielded for log write", "thread-yield: page access yielded due to prepare state change", "thread-yield: page acquire busy blocked", "thread-yield: page acquire eviction blocked", @@ -1469,7 +1468,6 @@ static const char *const __stats_connection_desc[] = { "transaction: transaction range of timestamps pinned by the oldest timestamp", "transaction: transaction read timestamp of the oldest active reader", "transaction: transaction rollback to stable currently running", - "transaction: transaction sync calls", "transaction: transaction walk of concurrent sessions", "transaction: transactions committed", "transaction: transactions rolled back", @@ -1918,7 +1916,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->conn_close_blocked_lsm = 0; stats->dhandle_lock_blocked = 0; stats->page_index_slot_ref_blocked = 0; - stats->log_server_sync_blocked = 0; stats->prepared_transition_blocked_page = 0; stats->page_busy_blocked = 0; stats->page_forcible_evict_blocked = 0; @@ -1993,7 +1990,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing txn_pinned_timestamp_oldest */ /* not clearing txn_timestamp_oldest_active_read */ /* not clearing txn_rollback_to_stable_running */ - stats->txn_sync = 0; stats->txn_walk_sessions = 0; stats->txn_commit = 0; stats->txn_rollback = 0; @@ -2453,7 +2449,6 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->conn_close_blocked_lsm += WT_STAT_READ(from, conn_close_blocked_lsm); to->dhandle_lock_blocked += WT_STAT_READ(from, dhandle_lock_blocked); to->page_index_slot_ref_blocked += WT_STAT_READ(from, page_index_slot_ref_blocked); - to->log_server_sync_blocked += WT_STAT_READ(from, log_server_sync_blocked); to->prepared_transition_blocked_page += WT_STAT_READ(from, prepared_transition_blocked_page); to->page_busy_blocked += WT_STAT_READ(from, page_busy_blocked); to->page_forcible_evict_blocked += WT_STAT_READ(from, page_forcible_evict_blocked); @@ -2532,7 +2527,6 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->txn_pinned_timestamp_oldest += WT_STAT_READ(from, txn_pinned_timestamp_oldest); to->txn_timestamp_oldest_active_read += WT_STAT_READ(from, txn_timestamp_oldest_active_read); to->txn_rollback_to_stable_running += WT_STAT_READ(from, txn_rollback_to_stable_running); - to->txn_sync += WT_STAT_READ(from, txn_sync); to->txn_walk_sessions += WT_STAT_READ(from, txn_walk_sessions); to->txn_commit += WT_STAT_READ(from, txn_commit); to->txn_rollback += WT_STAT_READ(from, txn_rollback); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 9876159c4be..68e0e45a8ea 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -874,14 +874,24 @@ __txn_commit_timestamps_usage_check(WT_SESSION_IMPL *session, WT_TXN_OP *op, WT_ #ifdef HAVE_DIAGNOSTIC prev_op_durable_ts = upd->prev_durable_ts; + /* + * Exit abnormally as the key consistency mode dictates all updates must use timestamps once + * they have been used. + */ if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_KEY_CONSISTENT) && prev_op_durable_ts != WT_TS_NONE && - !txn_has_ts) + !txn_has_ts) { WT_RET(__wt_msg(session, WT_COMMIT_TS_VERB_PREFIX "no timestamp provided for an update to a " "table configured to always use timestamps once they are first used")); + WT_ASSERT(session, false); + } - if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_ORDERED) && txn_has_ts && prev_op_durable_ts > op_ts) + /* + * Exit abnormally as we don't allow out of order timestamps on a table configured for strict + * ordering. + */ + if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_ORDERED) && txn_has_ts && prev_op_durable_ts > op_ts) { WT_RET(__wt_msg(session, WT_COMMIT_TS_VERB_PREFIX "committing a transaction that updates a " @@ -889,18 +899,27 @@ __txn_commit_timestamps_usage_check(WT_SESSION_IMPL *session, WT_TXN_OP *op, WT_ "update (%s) on a table configured for strict ordering", __wt_timestamp_to_string(op_ts, ts_string[0]), __wt_timestamp_to_string(prev_op_durable_ts, ts_string[1]))); + WT_ASSERT(session, false); + } + /* + * Exit abnormally as we don't allow an update without a timestamp if the previous update had an + * associated timestamp. This applies to both tables configured for strict and mixed mode + * orderings. + */ if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_ORDERED) && prev_op_durable_ts != WT_TS_NONE && - !txn_has_ts) + !txn_has_ts) { WT_RET( __wt_msg(session, WT_COMMIT_TS_VERB_PREFIX "committing a transaction that updates a value without " "a timestamp while the previous update (%s) is timestamped " "on a table configured for strict ordering", __wt_timestamp_to_string(prev_op_durable_ts, ts_string[1]))); + WT_ASSERT(session, false); + } if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_MIXED_MODE) && F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) && - op_ts != WT_TS_NONE && prev_op_durable_ts > op_ts) + op_ts != WT_TS_NONE && prev_op_durable_ts > op_ts) { WT_RET(__wt_msg(session, WT_COMMIT_TS_VERB_PREFIX "committing a transaction that updates a " @@ -908,6 +927,8 @@ __txn_commit_timestamps_usage_check(WT_SESSION_IMPL *session, WT_TXN_OP *op, WT_ "update (%s) on a table configured for mixed mode ordering", __wt_timestamp_to_string(op_ts, ts_string[0]), __wt_timestamp_to_string(prev_op_durable_ts, ts_string[1]))); + WT_ASSERT(session, false); + } #else WT_UNUSED(prev_op_durable_ts); #endif @@ -1052,16 +1073,18 @@ __txn_search_prepared_op( static int __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, WT_CURSOR **cursorp) { + WT_BTREE *btree; WT_CURSOR *hs_cursor; WT_CURSOR_BTREE *cbt; WT_DECL_RET; + WT_ITEM hs_recno_key; WT_TXN *txn; WT_UPDATE *fix_upd, *tombstone, *upd; #ifdef HAVE_DIAGNOSTIC WT_UPDATE *head_upd; #endif size_t not_used; - uint32_t hs_btree_id; + uint8_t *p, hs_recno_key_buf[WT_INTPACK64_MAXSIZE]; char ts_string[3][WT_TS_INT_STRING_SIZE]; bool upd_appended; @@ -1118,17 +1141,24 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, (upd->type != WT_UPDATE_TOMBSTONE || (!commit && upd->next != NULL && upd->durable_ts == upd->next->durable_ts && upd->txnid == upd->next->txnid && upd->start_ts == upd->next->start_ts))) { + btree = S2BT(session); cbt = (WT_CURSOR_BTREE *)(*cursorp); - hs_btree_id = S2BT(session)->id; - /* Open a history store table cursor. */ - WT_ERR(__wt_curhs_open(session, NULL, &hs_cursor)); - F_SET(hs_cursor, WT_CURSTD_HS_READ_COMMITTED); /* - * Scan the history store for the given btree and key with maximum start timestamp to let - * the search point to the last version of the key. + * Open a history store table cursor and scan the history store for the given btree and key + * with maximum start timestamp to let the search point to the last version of the key. */ - hs_cursor->set_key(hs_cursor, 4, hs_btree_id, &op->u.op_row.key, WT_TS_MAX, UINT64_MAX); + WT_ERR(__wt_curhs_open(session, NULL, &hs_cursor)); + F_SET(hs_cursor, WT_CURSTD_HS_READ_COMMITTED); + if (btree->type == BTREE_ROW) + hs_cursor->set_key(hs_cursor, 4, btree->id, &cbt->iface.key, WT_TS_MAX, UINT64_MAX); + else { + p = hs_recno_key_buf; + WT_ERR(__wt_vpack_uint(&p, 0, cbt->recno)); + hs_recno_key.data = hs_recno_key_buf; + hs_recno_key.size = WT_PTRDIFF(p, hs_recno_key_buf); + hs_cursor->set_key(hs_cursor, 4, btree->id, &hs_recno_key, WT_TS_MAX, UINT64_MAX); + } WT_ERR_NOTFOUND_OK(__wt_curhs_search_near_before(session, hs_cursor), true); if (ret == WT_NOTFOUND && !commit) { /* @@ -1139,12 +1169,15 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, WT_ERR(__wt_upd_alloc_tombstone(session, &tombstone, ¬_used)); #ifdef HAVE_DIAGNOSTIC WT_WITH_BTREE(session, op->btree, - ret = __wt_row_modify( - cbt, &cbt->iface.key, NULL, tombstone, WT_UPDATE_INVALID, false, false)); + ret = btree->type == BTREE_ROW ? + __wt_row_modify( + cbt, &cbt->iface.key, NULL, tombstone, WT_UPDATE_INVALID, false, false) : + __wt_col_modify(cbt, cbt->recno, NULL, tombstone, WT_UPDATE_INVALID, false, false)); #else WT_WITH_BTREE(session, op->btree, - ret = - __wt_row_modify(cbt, &cbt->iface.key, NULL, tombstone, WT_UPDATE_INVALID, false)); + ret = btree->type == BTREE_ROW ? + __wt_row_modify(cbt, &cbt->iface.key, NULL, tombstone, WT_UPDATE_INVALID, false) : + __wt_col_modify(cbt, cbt->recno, NULL, tombstone, WT_UPDATE_INVALID, false)); #endif WT_ERR(ret); tombstone = NULL; @@ -1537,9 +1570,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) */ if (F_ISSET(txn, WT_TXN_SYNC_SET)) WT_ERR_MSG(session, EINVAL, "Sync already set during begin_transaction"); - if (WT_STRING_MATCH("background", cval.str, cval.len)) - txn->txn_logsync = WT_LOG_BACKGROUND; - else if (WT_STRING_MATCH("off", cval.str, cval.len)) + if (WT_STRING_MATCH("off", cval.str, cval.len)) txn->txn_logsync = 0; /* * We don't need to check for "on" here because that is the default to inherit from the diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index d6b683dd4f2..1a7090cd2c6 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -1991,9 +1991,6 @@ __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) !WT_IS_METADATA(session->dhandle) || FLD_ISSET(session->lock_flags, WT_SESSION_LOCKED_METADATA)); - /* Discard the cached checkpoint list when checkpointing a single file by itself. */ - __wt_meta_saved_ckptlist_free(session); - WT_RET(__wt_config_gets_def(session, cfg, "force", 0, &cval)); force = cval.val != 0; WT_SAVE_DHANDLE(session, ret = __checkpoint_lock_dirty_tree(session, true, force, true, cfg)); @@ -2066,9 +2063,6 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) (!F_ISSET(S2C(session), WT_CONN_FILE_CLOSE_SYNC) && !metadata))) return (__wt_set_return(session, EBUSY)); - /* Discard the cached checkpoint list when checkpointing a single file by itself. */ - __wt_meta_saved_ckptlist_free(session); - /* * Make sure there isn't a potential race between backup copying the metadata and a checkpoint * changing the metadata. Backup holds both the checkpoint and schema locks. Checkpoint should @@ -2096,7 +2090,7 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) if (ret == 0 && !F_ISSET(btree, WT_BTREE_SKIP_CKPT)) ret = __checkpoint_tree(session, false, NULL); - /* Do not store the cached checkpoint list when checkpointing a single file alone. */ + /* Do not store the cached checkpoint list when closing the handle. */ __wt_meta_saved_ckptlist_free(session); if (need_tracking) diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 97da70f3b6b..5bc0f245497 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -339,11 +339,11 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page WT_CELL *kcell; WT_CELL_UNPACK_KV *unpack, _unpack; WT_CURSOR *hs_cursor; + WT_DECL_ITEM(full_value); WT_DECL_ITEM(hs_key); WT_DECL_ITEM(hs_value); WT_DECL_ITEM(key); WT_DECL_RET; - WT_ITEM full_value; WT_TIME_WINDOW *hs_tw; WT_UPDATE *tombstone, *upd; wt_timestamp_t hs_durable_ts, hs_start_ts, hs_stop_durable_ts, newer_hs_durable_ts; @@ -371,7 +371,6 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page tombstone = upd = NULL; hs_durable_ts = hs_start_ts = hs_stop_durable_ts = WT_TS_NONE; hs_btree_id = S2BT(session)->id; - WT_CLEAR(full_value); valid_update_found = false; #ifdef HAVE_DIAGNOSTIC first_record = true; @@ -392,18 +391,19 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page } else { /* Unpack a column cell. */ WT_ERR(__wt_scr_alloc(session, WT_INTPACK64_MAXSIZE, &key)); + memp = key->mem; + WT_ERR(__wt_vpack_uint(&memp, 0, recno)); + key->size = WT_PTRDIFF(memp, key->data); /* Get the full update value from the data store. */ unpack = &_unpack; kcell = WT_COL_PTR(page, cip); __wt_cell_unpack_kv(session, page->dsk, kcell, unpack); - memp = key->mem; - WT_ERR(__wt_vpack_uint(&memp, 0, recno)); - key->size = WT_PTRDIFF(memp, key->data); } - WT_ERR(__wt_page_cell_data_ref(session, page, unpack, &full_value)); - WT_ERR(__wt_buf_set(session, &full_value, full_value.data, full_value.size)); + WT_ERR(__wt_scr_alloc(session, 0, &full_value)); + WT_ERR(__wt_page_cell_data_ref(session, page, unpack, full_value)); + WT_ERR(__wt_buf_set(session, full_value, full_value->data, full_value->size)); newer_hs_durable_ts = unpack->tw.durable_start_ts; /* Open a history store table cursor. */ @@ -447,10 +447,10 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page if (hs_start_ts <= unpack->tw.start_ts || unpack->tw.prepare) { if (type == WT_UPDATE_MODIFY) WT_ERR(__wt_modify_apply_item( - session, S2BT(session)->value_format, &full_value, hs_value->data)); + session, S2BT(session)->value_format, full_value, hs_value->data)); else { WT_ASSERT(session, type == WT_UPDATE_STANDARD); - WT_ERR(__wt_buf_set(session, &full_value, hs_value->data, hs_value->size)); + WT_ERR(__wt_buf_set(session, full_value, hs_value->data, hs_value->size)); } } else __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), @@ -532,7 +532,7 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page __wt_hs_upd_time_window(hs_cursor, &hs_tw); WT_ASSERT(session, hs_tw->start_ts < unpack->tw.start_ts || hs_tw->start_txn < unpack->tw.start_txn); - WT_ERR(__wt_upd_alloc(session, &full_value, WT_UPDATE_STANDARD, &upd, NULL)); + WT_ERR(__wt_upd_alloc(session, full_value, WT_UPDATE_STANDARD, &upd, NULL)); /* * Set the transaction id of updates to WT_TXN_NONE when called from recovery, because the @@ -624,10 +624,10 @@ err: WT_ASSERT(session, tombstone == NULL || upd == tombstone); __wt_free_update_list(session, &upd); } + __wt_scr_free(session, &full_value); __wt_scr_free(session, &hs_key); __wt_scr_free(session, &hs_value); __wt_scr_free(session, &key); - __wt_buf_free(session, &full_value); if (hs_cursor != NULL) WT_TRET(hs_cursor->close(hs_cursor)); return (ret); @@ -643,8 +643,8 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W { WT_CELL *kcell; WT_CELL_UNPACK_KV *vpack, _vpack; + WT_DECL_ITEM(tmp); WT_DECL_RET; - WT_ITEM buf; WT_PAGE *page; WT_UPDATE *upd; char ts_string[5][WT_TS_INT_STRING_SIZE]; @@ -652,7 +652,6 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W page = ref->page; vpack = &_vpack; - WT_CLEAR(buf); upd = NULL; /* @@ -712,12 +711,6 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W (vpack->tw.durable_stop_ts > rollback_timestamp || __rollback_check_if_txnid_non_committed(session, vpack->tw.stop_txn) || prepared)) { /* - * Clear the remove operation from the key by inserting the original on-disk value as a - * standard update. - */ - WT_RET(__wt_page_cell_data_ref(session, page, vpack, &buf)); - - /* * For prepared transactions, it is possible that both the on-disk key start and stop time * windows can be the same. To abort these updates, check for any stable update from history * store or remove the key. @@ -738,7 +731,16 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W WT_STAT_CONN_DATA_INCR(session, txn_rts_keys_removed); } } else { - WT_ERR(__wt_upd_alloc(session, &buf, WT_UPDATE_STANDARD, &upd, NULL)); + /* + * Clear the remove operation from the key by inserting the original on-disk value as a + * standard update. + */ + WT_RET(__wt_scr_alloc(session, 0, &tmp)); + if ((ret = __wt_page_cell_data_ref(session, page, vpack, tmp)) == 0) + ret = __wt_upd_alloc(session, tmp, WT_UPDATE_STANDARD, &upd, NULL); + __wt_scr_free(session, &tmp); + WT_RET(ret); + /* * Set the transaction id of updates to WT_TXN_NONE when called from recovery, because * the connections write generation will be initialized after rollback to stable and the @@ -774,10 +776,9 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W WT_ERR(__rollback_row_modify(session, page, rip, upd)); else WT_ERR(__rollback_col_modify(session, ref, upd, recno)); - upd = NULL; + return (0); err: - __wt_buf_free(session, &buf); __wt_free(session, upd); return (ret); } diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 9191a9447e8..5222df53fc0 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -435,15 +435,15 @@ functions: "upload artifact": - command: archive.targz_pack params: - target: "wiredtiger.tgz" - source_dir: "wiredtiger" + target: ${upload_filename|wiredtiger.tgz} + source_dir: ${upload_source_dir|wiredtiger} include: - "./**" - command: s3.put params: aws_secret: ${aws_secret} aws_key: ${aws_key} - local_file: wiredtiger.tgz + local_file: ${upload_filename|wiredtiger.tgz} bucket: build_external permissions: public-read content_type: application/tar @@ -2151,18 +2151,31 @@ tasks: script: | set -o errexit set -o verbose - - GCOV=/opt/mongodbtoolchain/v3/bin/gcov gcovr -r .. -e '.*/bt_(debug|dump|misc|salvage|vrfy).*' -e '.*/(log|progress|verify_build|strerror|env_msg|err_file|cur_config|os_abort)\..*' -e '.*_stat\..*' --html -o ../coverage_report.html + mkdir -p ../coverage_report + GCOV=/opt/mongodbtoolchain/v3/bin/gcov gcovr -r .. -f ../src -e '.*/bt_(debug|dump|misc|salvage|vrfy).*' -e '.*/(log|progress|verify_build|strerror|env_msg|err_file|cur_config|os_abort)\..*' -e '.*_stat\..*' -e 'bench' -e 'examples' -e 'test' -e 'ext' -e 'dist' -e 'tools' -j 4 --html-details --html-self-contained -o ../coverage_report/2_coverage_report.html + - command: s3.put + params: + aws_secret: ${aws_secret} + aws_key: ${aws_key} + local_files_include_filter: wiredtiger/coverage_report/* + bucket: build_external + permissions: public-read + content_type: text/html + remote_file: wiredtiger/${build_variant}/${revision}/coverage_report_${build_id}-${execution}/ - command: s3.put params: aws_secret: ${aws_secret} aws_key: ${aws_key} - local_file: wiredtiger/coverage_report.html + local_file: wiredtiger/coverage_report/2_coverage_report.html bucket: build_external permissions: public-read content_type: text/html - display_name: Coverage report - remote_file: wiredtiger/${build_variant}/${revision}/coverage_report/coverage_report_${build_id}-${execution}.html + # Ensure that the first character of the display_name is a space + # This will ensure that it sorts before the per-file report pages which also get a space + # at the start of their display name (why this happens is not yet clear). + display_name: " 1 Coverage report main page" + remote_file: wiredtiger/${build_variant}/${revision}/coverage_report_${build_id}-${execution}/1_coverage_report_main.html + - name: spinlock-gcc-test commands: @@ -2606,8 +2619,6 @@ tasks: script: | set -o errexit set -o verbose - sudo su - ulimit -n 1000000 export "PATH=/opt/mongodbtoolchain/v3/bin:$PATH" virtualenv -p python3 venv source venv/bin/activate @@ -2819,6 +2830,8 @@ buildvariants: --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH + upload_source_dir: mongo-tests/largescale/many-collection/dbpath/diagnostic.data + upload_filename: diagnostic.data.tgz tasks: - name: many-dhandle-stress-test - name: many-collection-test @@ -2908,7 +2921,7 @@ buildvariants: - name: code-statistics display_name: "Code statistics" - batchtime: 1440 # 1 day + batchtime: 10080 # 7 days run_on: - ubuntu1804-test expansions: @@ -2958,6 +2971,7 @@ buildvariants: windows_configure_flags: -vcvars_bat "'C:\Program Files (x86)\Microsoft Visual Studio\2017\Professional\VC\Auxiliary\Build\vcvars64.bat'" tasks: - name: compile + - name: make-check-test - name: unit-test - name: macos-1014 diff --git a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh index 521a9787145..b686b6a90ba 100755 --- a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh +++ b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh @@ -53,44 +53,134 @@ build_branch() } ############################################################# +# get_config_file_name: +# arg1: branch name +############################################################# +get_config_file_name() +{ + local file_name="" + branch_name=$1 + if [ "${wt_standalone}" = true ] || [ $older = true ] ; then + file_name="${branch_name}/test/format/CONFIG_default" + echo $file_name + return + fi + file_name="CONFIG_${branch_name}" + + echo $file_name +} + +############################################################# +# create_configs: +# arg1: branch name +############################################################# +create_configs() +{ + branch_name=$1 + + file_name=$(get_config_file_name $branch_name) + + if [ -f $file_name ] ; then + echo " WARNING - ${file_name} already exists, overwriting it." + fi + + echo "##################################################" > $file_name + echo "runs.type=row" >> $file_name # WT-7379 - Temporarily disable column store tests + echo "btree.prefix=0" >> $file_name # WT-7579 - Prefix testing isn't portable between releases + echo "cache=80" >> $file_name # Medium cache so there's eviction + echo "checksum=on" >> $file_name # WT-7851 Fix illegal checksum configuration + echo "checkpoints=1" >> $file_name # Force periodic writes + echo "compression=snappy" >> $file_name # We only built with snappy, force the choice + echo "data_source=table" >> $file_name + echo "huffman_key=0" >> $file_name # WT-6893 - Not supoprted by newer releases + echo "in_memory=0" >> $file_name # Interested in the on-disk format + echo "leak_memory=1" >> $file_name # Faster runs + echo "logging=1" >> $file_name # Test log compatibility + echo "logging_compression=snappy" >> $file_name # We only built with snappy, force the choice + echo "rows=1000000" >> $file_name + echo "salvage=0" >> $file_name # Faster runs + echo "timer=4" >> $file_name + echo "verify=1" >> $file_name # Faster runs + + # Append older release configs for newer compatibility release test + if [ $newer = true ]; then + for i in "${compatible_upgrade_downgrade_release_branches[@]}" + do + if [ "$i" == "$branch_name" ] ; then + echo "transaction.isolation=snapshot" >> $file_name # WT-7545 - Older releases can't do lower isolation levels + echo "transaction.timestamps=1" >> $file_name # WT-7545 - Older releases can't do non-timestamp transactions + break + fi + done + fi + echo "##################################################" >> $file_name +} + +############################################################# +# create_default_configs: +# This function will create the default configs for older and standalone +# release branches. +############################################################# +create_default_configs() +{ + # Iterate over the release branches and create configuration files + for b in `ls`; do + if [ -d "$b" ]; then + (create_configs $b) + fi + done +} + +############################################################# +# create_configs_for_newer_release_branches: +############################################################# +create_configs_for_newer_release_branches() +{ + # Create configs for all the newer releases + for b in ${newer_release_branches[@]}; do + (create_configs $b) + done + + # Copy per-release configs in the newer release branches + for b in ${newer_release_branches[@]}; do + cp -rf CONFIG* $b/test/format/ + done + + # Delete configs from the top folder + rm -rf CONFIG* +} + +############################################################# # run_format: # arg1: branch name # arg2: access methods list ############################################################# run_format() { + branch_name=$1 echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - echo "Running format in branch: \"$1\"" + echo "Running format in branch: \"$branch_name\"" echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - cd "$1/test/format" - - flags="-1q $(bflag $1)" - - args="" - args+="btree.prefix=0 " # Prefix testing isn't portable between releases - args+="cache=80 " # Medium cache so there's eviction - args+="checkpoints=1 " # Force periodic writes - args+="checksum=on " # Force checksums. - args+="compression=snappy " # We only built with snappy, force the choice - args+="data_source=table " - args+="huffman_key=0 " # Not supoprted by newer releases - args+="in_memory=0 " # Interested in the on-disk format - args+="leak_memory=1 " # Faster runs - args+="logging=1 " # Test log compatibility - args+="logging_compression=snappy " # We only built with snappy, force the choice - args+="rows=1000000 " - args+="runs.type=row " # Temporarily disable column store tests - args+="salvage=0 " # Faster runs - args+="timer=4 " - args+="transaction.isolation=snapshot " # Older releases can't do lower isolation levels - args+="transaction.timestamps=1 " # Older releases can't do non-timestamp transactions - args+="verify=0 " # Faster runs + cd "$branch_name/test/format" + flags="-1q $(bflag $branch_name)" + + config_file="" + + # Compatibility test for newer releases will have CONFIG file for each release + # branches for the upgrade/downgrade testing. + # + # Compatibility test for older and standalone releases will have the default config. + if [ "$newer" = true ]; then + config_file="-c CONFIG_${branch_name}" + else + config_file="-c CONFIG_default" + fi for am in $2; do dir="RUNDIR.$am" echo "./t running $am access method..." - ./t $flags -h $dir "file_type=$am" $args + ./t $flags ${config_file} -h $dir "file_type=$am" # Remove the version string from the base configuration file. (MongoDB does not create # a base configuration file, but format does, so we need to remove its version string @@ -145,6 +235,9 @@ upgrade_downgrade() echo "Upgrade/downgrade testing with \"$1\" and \"$2\"" echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + cfg_file_branch1=$(get_config_file_name $1) + cfg_file_branch2=$(get_config_file_name $2) + # Alternate running each branch format test program on the second branch's build. # Loop twice, that is, run format twice using each branch. top="$PWD" @@ -152,13 +245,13 @@ upgrade_downgrade() for reps in {1..2}; do echo "$1 format running on $2 access method $am..." cd "$top/$1/test/format" - flags="-1qR $(bflag $1)" - ./t $flags -h "$top/$2/test/format/RUNDIR.$am" timer=2 + flags="-1Rq $(bflag $1)" + ./t $flags -c "$top/$2/test/format/${cfg_file_branch1}" -h "$top/$2/test/format/RUNDIR.$am" timer=2 echo "$2 format running on $2 access method $am..." cd "$top/$2/test/format" - flags="-1qR $(bflag $2)" - ./t $flags -h "RUNDIR.$am" timer=2 + flags="-1Rq $(bflag $2)" + ./t $flags -c $cfg_file_branch2 -h "RUNDIR.$am" timer=2 done done } @@ -178,6 +271,11 @@ wt_standalone=false newer_release_branches=(develop mongodb-5.0 mongodb-4.4 mongodb-4.2) older_release_branches=(mongodb-4.2 mongodb-4.0 mongodb-3.6) +# This array is used to generate compatible configuration files between releases, because +# upgrade/downgrade test runs each build's format test program on the second build's +# configuration file. +compatible_upgrade_downgrade_release_branches=(mongodb-4.4 mongodb-4.2) + declare -A scopes scopes[newer]="newer stable release branches" scopes[older]="older stable release branches" @@ -229,6 +327,7 @@ top="test-compatibility-run" rm -rf "$top" && mkdir "$top" cd "$top" + # Build the branches. if [ "$newer" = true ]; then for b in ${newer_release_branches[@]}; do @@ -253,6 +352,13 @@ if [ "${wt_standalone}" = true ]; then (build_branch "$wt2") fi +if [ "$newer" = true ]; then + create_configs_for_newer_release_branches +else + create_default_configs +fi + + # Run format in each branch for supported access methods. if [ "$newer" = true ]; then for b in ${newer_release_branches[@]}; do @@ -267,8 +373,8 @@ if [ "$older" = true ]; then fi if [ "${wt_standalone}" = true ]; then - (run_format "$wt1" "fix row var") - (run_format "$wt2" "fix row var") + (run_format "$wt1" "row") + (run_format "$wt2" "row") fi # Verify backward compatibility for supported access methods. diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 9ea615cb519..81d21111ec3 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -149,6 +149,7 @@ tinfo_init(void) tinfo->session = NULL; tinfo->cursor = NULL; + memset(tinfo->insert_list, 0, sizeof(tinfo->insert_list)); tinfo->insert_list_cnt = 0; tinfo->state = TINFO_RUNNING; @@ -685,7 +686,7 @@ ops(void *arg) WT_SESSION *session; iso_level_t iso_level; thread_op op; - uint64_t reset_op, session_op, truncate_op; + uint64_t max_rows, reset_op, session_op, truncate_op; uint32_t range, rnd; u_int i, j; const char *iso_config; @@ -814,8 +815,12 @@ ops(void *arg) op = UPDATE; } - /* Select a row. */ - tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows); + /* + * Select a row. Column-store extends the object, explicitly read the maximum row count and + * then use a local variable so the value won't change inside the loop. + */ + max_rows = (volatile uint64_t)g.rows; + tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)max_rows); /* * Inserts, removes and updates can be done following a cursor set-key, or based on a cursor @@ -938,7 +943,7 @@ ops(void *arg) } if (!positioned) - tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows); + tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)max_rows); /* * Truncate up to 5% of the table. If the range overlaps the beginning/end of the table, @@ -950,7 +955,7 @@ ops(void *arg) * (truncating from lower keys to higher keys or vice-versa). */ greater_than = mmrand(&tinfo->rnd, 0, 1) == 1; - range = g.rows < 20 ? 0 : mmrand(&tinfo->rnd, 0, (u_int)g.rows / 20); + range = max_rows < 20 ? 0 : mmrand(&tinfo->rnd, 0, (u_int)max_rows / 20); tinfo->last = tinfo->keyno; if (greater_than) { if (g.c_reverse) { @@ -960,13 +965,13 @@ ops(void *arg) tinfo->last -= range; } else { tinfo->last += range; - if (tinfo->last > g.rows) + if (tinfo->last > max_rows) tinfo->last = 0; } } else { if (g.c_reverse) { tinfo->keyno += range; - if (tinfo->keyno > g.rows) + if (tinfo->keyno > max_rows) tinfo->keyno = 0; } else { if (tinfo->keyno <= range) @@ -1700,7 +1705,7 @@ col_insert_resolve(TINFO *tinfo) */ do { WT_ORDERED_READ(v, g.rows); - for (i = 0, p = tinfo->insert_list; i < WT_ELEMENTS(tinfo->insert_list); ++i) { + for (i = 0, p = tinfo->insert_list; i < WT_ELEMENTS(tinfo->insert_list); ++i, ++p) { if (*p == v + 1) { testutil_assert(__wt_atomic_casv64(&g.rows, v, v + 1)); *p = 0; @@ -1746,6 +1751,7 @@ col_insert(TINFO *tinfo, WT_CURSOR *cursor) else cursor->set_value(cursor, tinfo->value); + /* Create a record, then add the key to our list of new records for later resolution. */ if ((ret = cursor->insert(cursor)) != 0) return (ret); diff --git a/src/third_party/wiredtiger/test/manydbs/manydbs.c b/src/third_party/wiredtiger/test/manydbs/manydbs.c index 2dc0e5a1f8f..1dec47c5f16 100644 --- a/src/third_party/wiredtiger/test/manydbs/manydbs.c +++ b/src/third_party/wiredtiger/test/manydbs/manydbs.c @@ -54,6 +54,27 @@ static const char *const uri = "table:main"; #define MAX_KV 100 #define MAX_VAL 128 +/* + * Maximum expected condition variable wakeups. POSIX allows arbitrarily many spurious wakeups to + * happen, so we need to be able to adjust this expectation per-platform. There are two cases: when + * completely idle, and when running a light workload. The latter is expressed as a fraction of the + * total number of condition variable sleeps; the former is a constant. + */ +#if defined(__NetBSD__) || defined(_WIN32) +/* + * NetBSD should never generate spurious wakeups, but does: see https://gnats.netbsd.org/56275. + * Windows can also generate spurious wakeups: + * https://docs.microsoft.com/en-us/windows/win32/sync/condition-variables These values allow the + * test to complete in spite of that. + */ +#define CV_RESET_THRESHOLD_IDLE 20 +#define CV_RESET_THRESHOLD_DENOM 10 +#else +/* Default values: should be no wakeups when idle and allow 1/20 otherwise. */ +#define CV_RESET_THRESHOLD_IDLE 0 +#define CV_RESET_THRESHOLD_DENOM 20 +#endif + static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) @@ -208,13 +229,13 @@ main(int argc, char *argv[]) * On an idle workload there should be no resets of condition variables during the idle * period. Even with a light workload, resets should not be very common. We look for 5%. */ - if (idle && cond_reset != cond_reset_orig[i]) - testutil_die( - ERANGE, "condition reset on idle connection %d of %" PRIu64, i, cond_reset); - if (!idle && cond_reset > cond_wait / 20) + if (idle && cond_reset > cond_reset_orig[i] + CV_RESET_THRESHOLD_IDLE) + testutil_die(ERANGE, "condition reset on idle connection %d of %" PRIu64 " exceeds %d", + i, cond_reset, CV_RESET_THRESHOLD_IDLE); + if (!idle && cond_reset > cond_wait / CV_RESET_THRESHOLD_DENOM) testutil_die(ERANGE, - "connection %d condition reset %" PRIu64 " exceeds 5%% of %" PRIu64, i, cond_reset, - cond_wait); + "connection %d condition reset %" PRIu64 " exceeds %d%% of %" PRIu64, i, cond_reset, + 100 / CV_RESET_THRESHOLD_DENOM, cond_wait); testutil_check(connections[i]->close(connections[i], NULL)); } diff --git a/src/third_party/wiredtiger/test/suite/test_assert01.py b/src/third_party/wiredtiger/test/suite/test_assert01.py index a91fd1700e0..4793ef26e72 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert01.py +++ b/src/third_party/wiredtiger/test/suite/test_assert01.py @@ -32,9 +32,7 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): base = 'assert01' @@ -43,13 +41,18 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): uri_def = base_uri + '.def.wt' uri_never = base_uri + '.never.wt' uri_none = base_uri + '.none.wt' - cfg = 'key_format=S,value_format=S,' cfg_always = 'verbose=[write_timestamp],write_timestamp_usage=always,assert=(write_timestamp=on)' cfg_def = '' cfg_never = 'verbose=(write_timestamp=true),write_timestamp_usage=never,assert=(write_timestamp=on)' cfg_none = 'assert=(write_timestamp=off)' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + count = 1 # # Commit a k/v pair making sure that it detects an error if needed, when @@ -57,13 +60,13 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): # def insert_check(self, uri, use_ts): c = self.session.open_cursor(uri) - key = 'key' + str(self.count) + key = 'key' + str(self.count) if self.usestrings else self.count val = 'value' + str(self.count) # Commit with a timestamp self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(self.count)) + 'commit_timestamp=' + self.timestamp_str(self.count)) c[key] = val # All settings other than never should commit successfully if (use_ts != 'never'): @@ -77,7 +80,7 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): self.count += 1 # Commit without a timestamp - key = 'key' + str(self.count) + key = 'key' + str(self.count) if self.usestrings else self.count val = 'value' + str(self.count) c = self.session.open_cursor(uri) self.session.begin_transaction() @@ -94,11 +97,13 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): c.close() def test_commit_timestamp(self): + cfg = 'key_format={},value_format=S,'.format(self.key_format) + # Create a data item at a timestamp - self.session.create(self.uri_always, self.cfg + self.cfg_always) - self.session.create(self.uri_def, self.cfg + self.cfg_def) - self.session.create(self.uri_never, self.cfg + self.cfg_never) - self.session.create(self.uri_none, self.cfg + self.cfg_none) + self.session.create(self.uri_always, cfg + self.cfg_always) + self.session.create(self.uri_def, cfg + self.cfg_def) + self.session.create(self.uri_never, cfg + self.cfg_never) + self.session.create(self.uri_none, cfg + self.cfg_none) # Check inserting into each table self.insert_check(self.uri_always, 'always') diff --git a/src/third_party/wiredtiger/test/suite/test_assert02.py b/src/third_party/wiredtiger/test/suite/test_assert02.py index ba871eb01c9..d3d5e885e87 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert02.py +++ b/src/third_party/wiredtiger/test/suite/test_assert02.py @@ -32,13 +32,17 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def test_read_timestamp(self): #if not wiredtiger.diagnostic_build(): # self.skipTest('requires a diagnostic build') @@ -50,29 +54,32 @@ class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): uri_never = base_uri + '.never.wt' uri_none = base_uri + '.none.wt' - cfg = 'key_format=S,value_format=S' + cfg = 'key_format={},value_format=S'.format(self.key_format) cfg_always = cfg + ',write_timestamp_usage=always,assert=(read_timestamp=always)' cfg_def = cfg cfg_never = cfg + ',assert=(read_timestamp=never)' cfg_none = cfg + ',assert=(read_timestamp=none)' - # Create a data item at a timestamp + # Create a data item at a timestamp. self.session.create(uri_always, cfg_always) self.session.create(uri_def, cfg_def) self.session.create(uri_never, cfg_never) self.session.create(uri_none, cfg_none) + # Make a key. + key1 = 'key1' if self.usestrings else 1 + # Insert a data item at timestamp 1. This should work for all. c_always = self.session.open_cursor(uri_always) c_def = self.session.open_cursor(uri_def) c_never = self.session.open_cursor(uri_never) c_none = self.session.open_cursor(uri_none) self.session.begin_transaction() - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(1)) - c_always['key1'] = 'value1' - c_def['key1'] = 'value1' - c_never['key1'] = 'value1' - c_none['key1'] = 'value1' + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(1)) + c_always[key1] = 'value1' + c_def[key1] = 'value1' + c_never[key1] = 'value1' + c_none[key1] = 'value1' self.session.commit_transaction() c_always.close() c_def.close() @@ -86,12 +93,12 @@ class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): c_never = self.session.open_cursor(uri_never) c_none = self.session.open_cursor(uri_none) - c_always.set_key('key1') - c_def.set_key('key1') - c_never.set_key('key1') - c_none.set_key('key1') + c_always.set_key(key1) + c_def.set_key(key1) + c_never.set_key(key1) + c_none.set_key(key1) - self.session.begin_transaction('read_timestamp=' + timestamp_str(1)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1)) c_always.search() c_def.search() c_none.search() @@ -114,10 +121,10 @@ class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): c_never = self.session.open_cursor(uri_never) c_none = self.session.open_cursor(uri_none) - c_always.set_key('key1') - c_def.set_key('key1') - c_never.set_key('key1') - c_none.set_key('key1') + c_always.set_key(key1) + c_def.set_key(key1) + c_never.set_key(key1) + c_none.set_key(key1) self.session.begin_transaction() c_never.search() diff --git a/src/third_party/wiredtiger/test/suite/test_assert03.py b/src/third_party/wiredtiger/test/suite/test_assert03.py index d1ac3926269..51b219b2fac 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert03.py +++ b/src/third_party/wiredtiger/test/suite/test_assert03.py @@ -32,24 +32,36 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest +from wtscenario import make_scenarios class test_assert03(wttest.WiredTigerTestCase, suite_subprocess): conn_config = 'log=(enabled)' base_uri = 'file:assert03.wt' - cfg = 'key_format=S,value_format=S' always = 'write_timestamp_usage=always,assert=(write_timestamp=on)' never = 'write_timestamp_usage=never,assert=(write_timestamp=on)' none = 'assert=(write_timestamp=off)' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def test_assert03(self): #if not wiredtiger.diagnostic_build(): # self.skipTest('requires a diagnostic build') + cfg = 'key_format={},value_format=S'.format(self.key_format) + key0 = 'key0' if self.usestrings else 17 + key1 = 'key1' if self.usestrings else 18 + key2 = 'key2' if self.usestrings else 19 + key3 = 'key3' if self.usestrings else 20 + # Create a data item at the default setting - self.session.create(self.base_uri, self.cfg) + self.session.create(self.base_uri, cfg) c = self.session.open_cursor(self.base_uri) self.session.begin_transaction() - c['key0'] = 'value0' + c[key0] = 'value0' self.session.commit_transaction() c.close() @@ -58,7 +70,7 @@ class test_assert03(wttest.WiredTigerTestCase, suite_subprocess): self.session.alter(self.base_uri, self.always) c = self.session.open_cursor(self.base_uri) self.session.begin_transaction() - c['key1'] = 'value1' + c[key1] = 'value1' msg = "/none set on this transaction/" self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:self.assertEquals(self.session.commit_transaction(), 0), msg) @@ -68,14 +80,14 @@ class test_assert03(wttest.WiredTigerTestCase, suite_subprocess): self.session.alter(self.base_uri, self.never) c = self.session.open_cursor(self.base_uri) self.session.begin_transaction() - c['key2'] = 'value2' + c[key2] = 'value2' self.session.commit_transaction() c.close() self.session.alter(self.base_uri, self.none) c = self.session.open_cursor(self.base_uri) self.session.begin_transaction() - c['key3'] = 'value3' + c[key3] = 'value3' self.session.commit_transaction() c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_assert04.py b/src/third_party/wiredtiger/test/suite/test_assert04.py index 711e1d02b06..6c5d4623d4d 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert04.py +++ b/src/third_party/wiredtiger/test/suite/test_assert04.py @@ -32,13 +32,17 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def test_timestamp_alter(self): base = 'assert04' uri = 'file:' + base @@ -50,14 +54,17 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # Create the table without the key consistency checking turned on. # Create a few items breaking the rules. Then alter the setting and # verify the inconsistent usage is detected. - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + + key_nots = 'key_nots' if self.usestrings else 5 + key_ts1 = 'key_ts1' if self.usestrings else 16 # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)) - c['key_ts1'] = 'value2' + 'commit_timestamp=' + self.timestamp_str(2)) + c[key_ts1] = 'value2' self.session.commit_transaction() c.close() @@ -65,8 +72,8 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1)) - c['key_ts1'] = 'value1' + 'commit_timestamp=' + self.timestamp_str(1)) + c[key_ts1] = 'value1' self.session.commit_transaction() c.close() @@ -74,27 +81,27 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # again modify without a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots' + c[key_nots] = 'value_nots' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)) - c['key_nots'] = 'value2' + 'commit_timestamp=' + self.timestamp_str(2)) + c[key_nots] = 'value2' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots2' + c[key_nots] = 'value_nots2' self.session.commit_transaction() c.close() # We must move the oldest timestamp forward in order to alter. # Otherwise alter closing the file will fail with EBUSY. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(2)) # Now alter the setting and make sure we detect incorrect usage. self.session.alter(uri, cfg_on) @@ -103,16 +110,16 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(5)) - c['key_ts1'] = 'value5' + 'commit_timestamp=' + self.timestamp_str(5)) + c[key_ts1] = 'value5' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(4)) - c['key_ts1'] = 'value4' + 'commit_timestamp=' + self.timestamp_str(4)) + c[key_ts1] = 'value4' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() @@ -120,7 +127,7 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # Detect not using a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value_nots3' + c[key_ts1] = 'value_nots3' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() @@ -131,41 +138,41 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # following transaction. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots3' + c[key_nots] = 'value_nots3' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(3)) - c['key_nots'] = 'value3' + 'commit_timestamp=' + self.timestamp_str(3)) + c[key_nots] = 'value3' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts1'], 'value5') - self.assertEquals(c['key_nots'], 'value_nots3') + self.assertEquals(c[key_ts1], 'value5') + self.assertEquals(c[key_nots], 'value_nots3') c.close() # Now alter the setting again and detection is off. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) self.session.alter(uri, cfg_off) # Detection is off we can successfully change the same key with and # without a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots4' + c[key_nots] = 'value_nots4' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(6)) - c['key_nots'] = 'value6' + 'commit_timestamp=' + self.timestamp_str(6)) + c[key_nots] = 'value6' self.session.commit_transaction() c.close() @@ -179,14 +186,22 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # That checking will verify any individual key is always or never # used with a timestamp. And if it is used with a timestamp that # the timestamps are in increasing order for that key. - self.session.create(uri, 'key_format=S,value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)') + self.session.create(uri, 'key_format={},value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)'.format(self.key_format)) + + key_nots = 'key_nots' if self.usestrings else 5 + key_ts1 = 'key_ts1' if self.usestrings else 16 + key_ts2 = 'key_ts2' if self.usestrings else 17 + key_ts3 = 'key_ts3' if self.usestrings else 18 + key_ts4 = 'key_ts4' if self.usestrings else 19 + key_ts5 = 'key_ts5' if self.usestrings else 20 + key_ts6 = 'key_ts6' if self.usestrings else 21 # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)) - c['key_ts1'] = 'value2' + 'commit_timestamp=' + self.timestamp_str(2)) + c[key_ts1] = 'value2' self.session.commit_transaction() c.close() @@ -194,8 +209,8 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1)) - c['key_ts1'] = 'value1' + 'commit_timestamp=' + self.timestamp_str(1)) + c[key_ts1] = 'value1' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() @@ -204,8 +219,8 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1)) - c['key_ts2'] = 'value1' + 'commit_timestamp=' + self.timestamp_str(1)) + c[key_ts2] = 'value1' self.session.commit_transaction() c.close() @@ -218,28 +233,28 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(10)) - c['key_ts3'] = 'value10' + 'commit_timestamp=' + self.timestamp_str(10)) + c[key_ts3] = 'value10' self.session.commit_transaction() self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(15)) - c['key_ts4'] = 'value15' + 'commit_timestamp=' + self.timestamp_str(15)) + c[key_ts4] = 'value15' self.session.commit_transaction() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(13)) - c['key_ts3'] = 'value13' - c['key_ts4'] = 'value13' + 'commit_timestamp=' + self.timestamp_str(13)) + c[key_ts3] = 'value13' + c[key_ts4] = 'value13' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts3'], 'value10') - self.assertEquals(c['key_ts4'], 'value15') + self.assertEquals(c[key_ts3], 'value10') + self.assertEquals(c[key_ts4], 'value15') c.close() # @@ -249,15 +264,15 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(13)) - c['key_ts3'] = 'value13' + 'commit_timestamp=' + self.timestamp_str(13)) + c[key_ts3] = 'value13' self.session.commit_transaction() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(13)) - c['key_ts4'] = 'value13' + 'commit_timestamp=' + self.timestamp_str(13)) + c[key_ts4] = 'value13' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() @@ -268,43 +283,43 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(14)) - c['key_ts4'] = 'value14' + 'commit_timestamp=' + self.timestamp_str(14)) + c[key_ts4] = 'value14' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value15') + self.assertEquals(c[key_ts4], 'value15') c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(16)) - c['key_ts4'] = 'value16' + 'commit_timestamp=' + self.timestamp_str(16)) + c[key_ts4] = 'value16' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value16') + self.assertEquals(c[key_ts4], 'value16') c.close() # Now try to modify a key previously used with timestamps without # one. We should get the inconsistent usage message. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value_nots' + c[key_ts4] = 'value_nots' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value_nots' + c[key_ts4] = 'value_nots' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value16') + self.assertEquals(c[key_ts4], 'value16') c.close() # Now confirm the other way. Create a key without a timestamp and then @@ -312,82 +327,82 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # makes sense here is the inconsistent usage. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots' + c[key_nots] = 'value_nots' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(16)) - c['key_nots'] = 'value16' + 'commit_timestamp=' + self.timestamp_str(16)) + c[key_nots] = 'value16' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots1' + c[key_nots] = 'value_nots1' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(17)) - c['key_nots'] = 'value17' + 'commit_timestamp=' + self.timestamp_str(17)) + c[key_nots] = 'value17' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_nots'], 'value_nots1') + self.assertEquals(c[key_nots], 'value_nots1') c.close() # Confirm it is okay to set the timestamp in the middle or end of the # transaction. That should set the timestamp for the whole thing. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts5'] = 'value_notsyet' + c[key_ts5] = 'value_notsyet' self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(20)) - c['key_ts5'] = 'value20' + 'commit_timestamp=' + self.timestamp_str(20)) + c[key_ts5] = 'value20' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts5'], 'value20') + self.assertEquals(c[key_ts5], 'value20') c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts6'] = 'value_notsyet' - c['key_ts6'] = 'value21_after' + c[key_ts6] = 'value_notsyet' + c[key_ts6] = 'value21_after' self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(21)) + 'commit_timestamp=' + self.timestamp_str(21)) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts6'], 'value21_after') + self.assertEquals(c[key_ts6], 'value21_after') c.close() # Confirm it is okay to set the timestamp on the commit call. # That should set the timestamp for the whole thing. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts6'] = 'value_committs1' - c['key_ts6'] = 'value22' + c[key_ts6] = 'value_committs1' + c[key_ts6] = 'value22' self.session.commit_transaction('commit_timestamp=' + - timestamp_str(22)) + self.timestamp_str(22)) c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value23' + c[key_nots] = 'value23' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(23)), msg_usage) + 'commit_timestamp=' + self.timestamp_str(23)), msg_usage) c.close() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_assert05.py b/src/third_party/wiredtiger/test/suite/test_assert05.py index b5fe9205bd9..afa2acbcbfb 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert05.py +++ b/src/third_party/wiredtiger/test/suite/test_assert05.py @@ -32,9 +32,7 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): base = 'assert05' @@ -44,12 +42,17 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): uri_def = base_uri + '.def.wt' uri_never = base_uri + '.never.wt' uri_none = base_uri + '.none.wt' - cfg = 'key_format=S,value_format=S,' cfg_always = 'verbose=(write_timestamp=true),write_timestamp_usage=always,assert=(write_timestamp=on)' cfg_def = '' cfg_never = 'write_timestamp_usage=never,assert=(write_timestamp=on)' cfg_none = 'assert=(write_timestamp=off)' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + count = 1 # # Commit a k/v pair making sure that it detects an error if needed, when @@ -57,18 +60,18 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): # def insert_check(self, uri, use_ts): c = self.session.open_cursor(uri) - key = 'key' + str(self.count) + key = 'key' + str(self.count) if self.usestrings else self.count val = 'value' + str(self.count) # Commit with a timestamp self.session.begin_transaction() c[key] = val self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(self.count)) + 'prepare_timestamp=' + self.timestamp_str(self.count)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(self.count)) + 'commit_timestamp=' + self.timestamp_str(self.count)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(self.count)) + 'durable_timestamp=' + self.timestamp_str(self.count)) # All settings other than never should commit successfully if (use_ts != 'never'): self.session.commit_transaction() @@ -86,17 +89,17 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): self.count += 1 # Commit without a timestamp - key = 'key' + str(self.count) + key = 'key' + str(self.count) if self.usestrings else self.count val = 'value' + str(self.count) c = self.session.open_cursor(uri) self.session.begin_transaction() c[key] = val if (use_ts == 'always'): self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(self.count)) + 'prepare_timestamp=' + self.timestamp_str(self.count)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(self.count)) + 'commit_timestamp=' + self.timestamp_str(self.count)) # All settings other than always should commit successfully if (use_ts != 'always' and use_ts != 'never'): self.session.commit_transaction() @@ -114,11 +117,13 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): c.close() def test_durable_timestamp(self): + cfg = 'key_format={},value_format=S,'.format(self.key_format) + # Create a data item at a timestamp - self.session.create(self.uri_always, self.cfg + self.cfg_always) - self.session.create(self.uri_def, self.cfg + self.cfg_def) - self.session.create(self.uri_never, self.cfg + self.cfg_never) - self.session.create(self.uri_none, self.cfg + self.cfg_none) + self.session.create(self.uri_always, cfg + self.cfg_always) + self.session.create(self.uri_def, cfg + self.cfg_def) + self.session.create(self.uri_never, cfg + self.cfg_never) + self.session.create(self.uri_none, cfg + self.cfg_none) # Check inserting into each table self.insert_check(self.uri_always, 'always') diff --git a/src/third_party/wiredtiger/test/suite/test_assert06.py b/src/third_party/wiredtiger/test/suite/test_assert06.py index 24891445ab0..3ea620e638e 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert06.py +++ b/src/third_party/wiredtiger/test/suite/test_assert06.py @@ -32,20 +32,24 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def apply_timestamps(self, timestamp): self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(timestamp)) + 'prepare_timestamp=' + self.timestamp_str(timestamp)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(timestamp)) + 'commit_timestamp=' + self.timestamp_str(timestamp)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(timestamp)) + 'durable_timestamp=' + self.timestamp_str(timestamp)) def test_timestamp_alter(self): base = 'assert06' @@ -55,14 +59,17 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): msg_ooo='/out of order/' msg_usage='/used inconsistently/' + key_nots = 'key_nots' if self.usestrings else 5 + key_ts1 = 'key_ts1' if self.usestrings else 16 + # Create the table without the key consistency checking turned on. # Create a few items breaking the rules. Then alter the setting and # verify the inconsistent usage is detected. - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value2' + c[key_ts1] = 'value2' self.apply_timestamps(2) self.session.commit_transaction() c.close() @@ -70,7 +77,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Modify the data item at timestamp 1. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value1' + c[key_ts1] = 'value1' self.apply_timestamps(1) self.session.commit_transaction() c.close() @@ -79,26 +86,26 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # again modify without a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots' + c[key_nots] = 'value_nots' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value2' + c[key_nots] = 'value2' self.apply_timestamps(2) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots2' + c[key_nots] = 'value_nots2' self.session.commit_transaction() c.close() # We must move the oldest timestamp forward in order to alter. # Otherwise alter closing the file will fail with EBUSY. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(2)) # Now alter the setting and make sure we detect incorrect usage. self.session.alter(uri, cfg_on) @@ -106,7 +113,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Detect decreasing timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value5' + c[key_ts1] = 'value5' self.apply_timestamps(5) self.session.commit_transaction() c.close() @@ -116,7 +123,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value4' + c[key_ts1] = 'value4' self.apply_timestamps(4) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) @@ -126,7 +133,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Detect not using a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value_nots3' + c[key_ts1] = 'value_nots3' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() @@ -137,7 +144,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # following transaction. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots3' + c[key_nots] = 'value_nots3' self.session.commit_transaction() c.close() @@ -146,7 +153,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value3' + c[key_nots] = 'value3' self.apply_timestamps(3) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) @@ -155,26 +162,26 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): ''' c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts1'], 'value5') - self.assertEquals(c['key_nots'], 'value_nots3') + self.assertEquals(c[key_ts1], 'value5') + self.assertEquals(c[key_nots], 'value_nots3') c.close() # Test to make sure that key consistency can be turned off # after turning it on. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) self.session.alter(uri, cfg_off) # Detection is off we can successfully change the same key with and # without a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots4' + c[key_nots] = 'value_nots4' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value6' + c[key_nots] = 'value6' self.apply_timestamps(6) self.session.commit_transaction() c.close() @@ -185,16 +192,24 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): msg_ooo='/out of order/' msg_usage='/used inconsistently/' + key_nots = 'key_nots' if self.usestrings else 5 + key_ts1 = 'key_ts1' if self.usestrings else 16 + key_ts2 = 'key_ts2' if self.usestrings else 17 + key_ts3 = 'key_ts3' if self.usestrings else 18 + key_ts4 = 'key_ts4' if self.usestrings else 19 + key_ts5 = 'key_ts5' if self.usestrings else 20 + key_ts6 = 'key_ts6' if self.usestrings else 21 + # Create the table with the key consistency checking turned on. # That checking will verify any individual key is always or never # used with a timestamp. And if it is used with a timestamp that # the timestamps are in increasing order for that key. - self.session.create(uri, 'key_format=S,value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)') + self.session.create(uri, 'key_format={},value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)'.format(self.key_format)) # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value2' + c[key_ts1] = 'value2' self.apply_timestamps(2) self.session.commit_transaction() c.close() @@ -205,7 +220,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Modify the data item at timestamp 1. We should detect it is wrong. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value1' + c[key_ts1] = 'value1' self.apply_timestamps(1) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) @@ -215,7 +230,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Make sure we can successfully add a different key at timestamp 1. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts2'] = 'value1' + c[key_ts2] = 'value1' self.apply_timestamps(1) self.session.commit_transaction() c.close() @@ -228,11 +243,11 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts3'] = 'value10' + c[key_ts3] = 'value10' self.apply_timestamps(10) self.session.commit_transaction() self.session.begin_transaction() - c['key_ts4'] = 'value15' + c[key_ts4] = 'value15' self.apply_timestamps(15) self.session.commit_transaction() @@ -241,8 +256,8 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts3'] = 'value13' - c['key_ts4'] = 'value13' + c[key_ts3] = 'value13' + c[key_ts4] = 'value13' self.apply_timestamps(13) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) @@ -250,8 +265,8 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): ''' c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts3'], 'value10') - self.assertEquals(c['key_ts4'], 'value15') + self.assertEquals(c[key_ts3], 'value10') + self.assertEquals(c[key_ts4], 'value15') c.close() # @@ -260,7 +275,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts3'] = 'value13' + c[key_ts3] = 'value13' self.apply_timestamps(13) self.session.commit_transaction() @@ -269,7 +284,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value13' + c[key_ts4] = 'value13' self.apply_timestamps(13) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) @@ -284,43 +299,43 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # in the update chain are not considered for the timestamp check. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value14' + c[key_ts4] = 'value14' self.apply_timestamps(14) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value15') + self.assertEquals(c[key_ts4], 'value15') c.close() ''' c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value16' + c[key_ts4] = 'value16' self.apply_timestamps(16) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value16') + self.assertEquals(c[key_ts4], 'value16') c.close() # Now try to modify a key previously used with timestamps without # one. We should get the inconsistent usage message. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value_nots' + c[key_ts4] = 'value_nots' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value_nots' + c[key_ts4] = 'value_nots' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value16') + self.assertEquals(c[key_ts4], 'value16') c.close() # Now confirm the other way. Create a key without a timestamp and then @@ -328,7 +343,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # makes sense here is the inconsistent usage. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots' + c[key_nots] = 'value_nots' self.session.commit_transaction() c.close() @@ -337,7 +352,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value16' + c[key_nots] = 'value16' self.apply_timestamps(16) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) @@ -346,7 +361,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots1' + c[key_nots] = 'value_nots1' self.session.commit_transaction() c.close() @@ -355,7 +370,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value17' + c[key_nots] = 'value17' self.apply_timestamps(17) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) @@ -363,47 +378,47 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): ''' c = self.session.open_cursor(uri) - self.assertEquals(c['key_nots'], 'value_nots1') + self.assertEquals(c[key_nots], 'value_nots1') c.close() # Confirm it is okay to set the timestamp in the middle or end of the # transaction. That should set the timestamp for the whole thing. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts5'] = 'value_notsyet' - c['key_ts5'] = 'value20' + c[key_ts5] = 'value_notsyet' + c[key_ts5] = 'value20' self.apply_timestamps(20) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts5'], 'value20') + self.assertEquals(c[key_ts5], 'value20') c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts6'] = 'value_notsyet' - c['key_ts6'] = 'value21_after' + c[key_ts6] = 'value_notsyet' + c[key_ts6] = 'value21_after' self.apply_timestamps(21) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts6'], 'value21_after') + self.assertEquals(c[key_ts6], 'value21_after') c.close() # Confirm it is okay to set the durable timestamp on the commit call. # That should set the timestamp for the whole thing. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts6'] = 'value_committs1' - c['key_ts6'] = 'value22' + c[key_ts6] = 'value_committs1' + c[key_ts6] = 'value22' self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(22)) + 'prepare_timestamp=' + self.timestamp_str(22)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(22)) + 'commit_timestamp=' + self.timestamp_str(22)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(22)) + 'durable_timestamp=' + self.timestamp_str(22)) self.session.commit_transaction() c.close() @@ -412,14 +427,14 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value23' + c[key_nots] = 'value23' self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(23)) + 'prepare_timestamp=' + self.timestamp_str(23)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(23)) + 'commit_timestamp=' + self.timestamp_str(23)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'durable_timestamp=' + timestamp_str(23)), msg_usage) + 'durable_timestamp=' + self.timestamp_str(23)), msg_usage) c.close() ''' diff --git a/src/third_party/wiredtiger/test/suite/test_assert07.py b/src/third_party/wiredtiger/test/suite/test_assert07.py index e79dd1a27dc..687174b4796 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert07.py +++ b/src/third_party/wiredtiger/test/suite/test_assert07.py @@ -33,95 +33,101 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert07(wttest.WiredTigerTestCase, suite_subprocess): + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def apply_timestamps(self, timestamp): self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(timestamp)) + 'prepare_timestamp=' + self.timestamp_str(timestamp)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(timestamp)) + 'commit_timestamp=' + self.timestamp_str(timestamp)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(timestamp)) + 'durable_timestamp=' + self.timestamp_str(timestamp)) def test_timestamp_alter(self): base = 'assert07' uri = 'file:' + base + key_ts1 = 'key_ts1' if self.usestrings else 1 + # No reserved, single update. - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) c = self.session.open_cursor(uri) self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value1' + c[key_ts1] = 'value1' self.apply_timestamps(1) self.session.commit_transaction() # Reserved at the start of the chain, with one update. self.session.begin_transaction('isolation=snapshot') - c.set_key('key_ts1') + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value2' + c[key_ts1] = 'value2' self.apply_timestamps(2) self.session.commit_transaction() # Reserved at the end of the chain, with one update. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value3' - c.set_key('key_ts1') + c[key_ts1] = 'value3' + c.set_key(key_ts1) c.reserve() self.apply_timestamps(3) self.session.commit_transaction() # Reserved at the start of the chain, with multiple. self.session.begin_transaction('isolation=snapshot') - c.set_key('key_ts1') + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value4' - c['key_ts1'] = 'value5' + c[key_ts1] = 'value4' + c[key_ts1] = 'value5' self.apply_timestamps(4) self.session.commit_transaction() # Reserved at the end of the chain, with multiple updates. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value6' - c['key_ts1'] = 'value7' - c.set_key('key_ts1') + c[key_ts1] = 'value6' + c[key_ts1] = 'value7' + c.set_key(key_ts1) c.reserve() self.apply_timestamps(5) self.session.commit_transaction() # Reserved between two updates. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value8' - c.set_key('key_ts1') + c[key_ts1] = 'value8' + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value9' + c[key_ts1] = 'value9' self.apply_timestamps(6) self.session.commit_transaction() # Reserved update with multiple extra updates. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value10' - c.set_key('key_ts1') + c[key_ts1] = 'value10' + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value11' - c['key_ts1'] = 'value12' - c['key_ts1'] = 'value13' + c[key_ts1] = 'value11' + c[key_ts1] = 'value12' + c[key_ts1] = 'value13' self.apply_timestamps(7) self.session.commit_transaction() # Reserved updates with multiple extra updates. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value14' - c.set_key('key_ts1') + c[key_ts1] = 'value14' + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value15' - c['key_ts1'] = 'value16' - c.set_key('key_ts1') + c[key_ts1] = 'value15' + c[key_ts1] = 'value16' + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value17' + c[key_ts1] = 'value17' self.apply_timestamps(8) self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_backup08.py b/src/third_party/wiredtiger/test/suite/test_backup08.py index 1c674df5a54..95e50971408 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup08.py +++ b/src/third_party/wiredtiger/test/suite/test_backup08.py @@ -34,9 +34,6 @@ import os, shutil import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_backup08(wttest.WiredTigerTestCase): conn_config = 'config_base=false,create,log=(enabled)' session_config = 'isolation=snapshot' @@ -85,12 +82,12 @@ class test_backup08(wttest.WiredTigerTestCase): curs[i] = i self.pr("i: " + str(i)) self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(i)) + 'commit_timestamp=' + self.timestamp_str(i)) # Set the oldest and stable timestamp a bit earlier than the data # we inserted. Take a checkpoint to the stable timestamp. self.pr("stable ts: " + str(ts)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(ts) + - ',stable_timestamp=' + timestamp_str(ts)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(ts) + + ',stable_timestamp=' + self.timestamp_str(ts)) # This forces a different checkpoint timestamp for each table. # Default is to use the stable timestamp. expected = ts @@ -102,7 +99,7 @@ class test_backup08(wttest.WiredTigerTestCase): ckpt_config = 'use_timestamp=true' self.session.checkpoint(ckpt_config) q = self.conn.query_timestamp('get=last_checkpoint') - self.assertTimestampsEqual(q, timestamp_str(expected)) + self.assertTimestampsEqual(q, self.timestamp_str(expected)) return expected def backup_and_recover(self, expected_rec_ts): @@ -123,7 +120,7 @@ class test_backup08(wttest.WiredTigerTestCase): backup_conn = self.wiredtiger_open(self.dir) q = backup_conn.query_timestamp('get=recovery') self.pr("query recovery ts: " + q) - self.assertTimestampsEqual(q, timestamp_str(expected_rec_ts)) + self.assertTimestampsEqual(q, self.timestamp_str(expected_rec_ts)) def test_timestamp_backup(self): # Add some data and checkpoint using the timestamp or not diff --git a/src/third_party/wiredtiger/test/suite/test_backup15.py b/src/third_party/wiredtiger/test/suite/test_backup15.py index 6aa2f854b95..009fdf85e40 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup15.py +++ b/src/third_party/wiredtiger/test/suite/test_backup15.py @@ -25,13 +25,8 @@ # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger -import os, shutil +import os from wtbackup import backup_base -from wtdataset import simple_key -from wtscenario import make_scenarios -import glob # test_backup15.py # Test cursor backup with a block-based incremental cursor. @@ -118,6 +113,3 @@ class test_backup15(backup_base): self.take_full_backup(self.home_full) self.compare_backups(self.uri, self.home_full, self.home_incr, str(self.bkup_id)) self.setup_directories(self.home_incr, self.home_full) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug022.py b/src/third_party/wiredtiger/test/suite/test_bug022.py index fe7c2c32f28..31fb2387662 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug022.py +++ b/src/third_party/wiredtiger/test/suite/test_bug022.py @@ -31,9 +31,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_bug022(wttest.WiredTigerTestCase): uri = 'file:test_bug022' conn_config = 'cache_size=50MB' @@ -41,21 +38,21 @@ class test_bug022(wttest.WiredTigerTestCase): def test_apply_modifies_on_onpage_tombstone(self): self.session.create(self.uri, 'key_format=S,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) value = 'a' * 500 for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply tombstones for every key. for i in range(1, 10000): self.session.begin_transaction() cursor.set_key(str(i)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint03.py b/src/third_party/wiredtiger/test/suite/test_checkpoint03.py index 0f4fba34186..10f6138c2c0 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint03.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint03.py @@ -39,9 +39,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_checkpoint03(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_checkpoint03' conn_config = 'statistics=(all)' diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint04.py b/src/third_party/wiredtiger/test/suite/test_checkpoint04.py index 413f3333428..eda62c33f66 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint04.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint04.py @@ -33,9 +33,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - class test_checkpoint04(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' session_config = 'isolation=snapshot' diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint06.py b/src/third_party/wiredtiger/test/suite/test_checkpoint06.py index edd22434228..06795a3cd27 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint06.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint06.py @@ -29,9 +29,6 @@ import time import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_checkpoint06.py # Verify that we rollback the truncation that is committed after stable # timestamp in the checkpoint. @@ -44,14 +41,14 @@ class test_checkpoint06(wttest.WiredTigerTestCase): self.session.create(self.uri, 'key_format=i,value_format=S') value = "abcdefghijklmnopqrstuvwxyz" * 3 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() # Setup: Insert some data for i in range(10000): cursor[i] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2)) # Flush everything to disk self.reopen_conn() @@ -63,7 +60,7 @@ class test_checkpoint06(wttest.WiredTigerTestCase): end = self.session.open_cursor(self.uri) end.set_key(9995) self.session.truncate(None, start, None, None) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Do a checkpoint self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint07.py b/src/third_party/wiredtiger/test/suite/test_checkpoint07.py index c1c11dbad91..a876d1ac807 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint07.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint07.py @@ -33,9 +33,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - class test_checkpoint07(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' session_config = 'isolation=snapshot' diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint08.py b/src/third_party/wiredtiger/test/suite/test_checkpoint08.py index 5a8e8fe647e..49de9c5f163 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint08.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint08.py @@ -37,9 +37,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - class test_checkpoint08(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' session_config = 'isolation=snapshot' @@ -61,8 +58,8 @@ class test_checkpoint08(wttest.WiredTigerTestCase): self.session.create(self.uri2, 'key_format=i,value_format=i') # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) # Setup: Insert some data and checkpoint it. Then modify only # the data in the first table and checkpoint. Verify the clean skip @@ -73,23 +70,23 @@ class test_checkpoint08(wttest.WiredTigerTestCase): self.session.begin_transaction() c1[1] = 1 c2[1] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() c1[1] = 10 c2[1] = 10 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(3)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(3)) self.session.checkpoint(None) # Modify the both tables and reverify. self.session.begin_transaction() c1[3] = 3 c2[3] = 3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(4)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(4)) self.session.checkpoint(None) val = self.get_stat(self.uri1) @@ -100,13 +97,13 @@ class test_checkpoint08(wttest.WiredTigerTestCase): self.assertNotEqual(hsval, 0) # Modify the both tables and reverify when oldest timestamp moved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(4)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() c1[4] = 4 c2[4] = 4 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) self.session.checkpoint(None) val = self.get_stat(self.uri1) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py index ce839469ed1..0bc38abb5d2 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py @@ -42,7 +42,7 @@ from wiredtiger import stat # class test_checkpoint_snapshot01(wttest.WiredTigerTestCase): - conn_config = 'cache_size=50MB,statistics=(fast)' + conn_config = 'cache_size=50MB' # Create a table. uri = "table:test_checkpoint_snapshot01" diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py index 2f93fef82b2..9f796b28089 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py @@ -38,9 +38,6 @@ from wiredtiger import stat # This test is to run checkpoint and eviction in parallel with timing # stress for checkpoint and let eviction write more data than checkpoint. # - -def timestamp_str(t): - return '%x' % t class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): # Create a table. @@ -68,7 +65,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): if commit_ts == 0: session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, nrows, read_ts): @@ -76,7 +73,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): if read_ts == 0: session.begin_transaction() else: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -137,8 +134,8 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): valuea = "aaaaa" * 100 # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) self.large_updates(self.uri, valuea, ds, self.nrows, 20) self.check(valuea, self.uri, self.nrows, 20) @@ -151,10 +148,10 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): cursor1.set_key(ds.key(i)) cursor1.set_value(valuea) self.assertEqual(cursor1.insert(), 0) - session1.timestamp_transaction('commit_timestamp=' + timestamp_str(30)) + session1.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) # Set stable timestamp to 40 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) # Create a checkpoint thread done = threading.Event() @@ -191,8 +188,8 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): valueb = "bbbbb" * 100 # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) session1 = self.conn.open_session() session1.begin_transaction() @@ -208,10 +205,10 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): cursor2.set_key(ds.key(i)) cursor2.set_value(valuea) self.assertEqual(cursor2.insert(), 0) - session1.timestamp_transaction('commit_timestamp=' + timestamp_str(30)) + session1.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) # Set stable timestamp to 40 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) # Create a checkpoint thread done = threading.Event() diff --git a/src/third_party/wiredtiger/test/suite/test_compress02.py b/src/third_party/wiredtiger/test/suite/test_compress02.py index a88b8f54304..e2ce0dd3a33 100644 --- a/src/third_party/wiredtiger/test/suite/test_compress02.py +++ b/src/third_party/wiredtiger/test/suite/test_compress02.py @@ -39,11 +39,7 @@ from wiredtiger import stat # we are using zstd as the block compressor. Tables created before reconfiguration # will still use the previous compression level. # - -def timestamp_str(t): - return '%x' % t class test_compress02(wttest.WiredTigerTestCase): - # Create a table. uri = "table:test_compress02" nrows = 1000 diff --git a/src/third_party/wiredtiger/test/suite/test_config07.py b/src/third_party/wiredtiger/test/suite/test_config07.py index 6539178cb7e..33789d518a0 100644 --- a/src/third_party/wiredtiger/test/suite/test_config07.py +++ b/src/third_party/wiredtiger/test/suite/test_config07.py @@ -83,7 +83,7 @@ class test_config07(wttest.WiredTigerTestCase): msg = '/invalid log extend length/' config = 'log=(enabled,file_max=1M),file_extend=' + self.log_extend_len - configarg = 'create,statistics=(fast)' + ',' + config + configarg = 'create,' + config # Expect an error when an invalid log extend size is provided. if self.expected_log_size is None: diff --git a/src/third_party/wiredtiger/test/suite/test_cursor15.py b/src/third_party/wiredtiger/test/suite/test_cursor15.py index ab136333682..1a950b7fc51 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor15.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor15.py @@ -37,7 +37,7 @@ class test_cursor15(wttest.WiredTigerTestCase): tablename = 'test_read_once' uri = 'table:' + tablename - conn_config = 'cache_size=1M,statistics=(all)' + conn_config = 'cache_size=1M' def test_cursor15(self): # This test is configured to use 1MB of cache. It will insert 20 @@ -51,7 +51,7 @@ class test_cursor15(wttest.WiredTigerTestCase): cursor[key] = '1' * (100 * 1024) cursor.close() - # Restart the database to clear the cache and reset statistics. + # Restart the database to clear the cache. self.reopen_conn() # We don't restart the database between runs to exercise that read_once diff --git a/src/third_party/wiredtiger/test/suite/test_cursor17.py b/src/third_party/wiredtiger/test/suite/test_cursor17.py index f068279f8f1..60cad85a5e2 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor17.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor17.py @@ -30,9 +30,6 @@ import wiredtiger, wttest from wtscenario import make_scenarios from wiredtiger import stat, WT_NOTFOUND -def timestamp_str(t): - return '%x' % t - # test_cursor17.py # Test the cursor traversal optimization for delete heavy workloads. This optimization enables # cursor traversal mechanism to skip pages where all records on the page are deleted with a @@ -60,8 +57,8 @@ class test_cursor17(wttest.WiredTigerTestCase): total_keys = 40000 # Keep the oldest and the stable timestamp pinned. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2)) cursor = self.session.open_cursor(uri) commit_timestamp = 3 @@ -70,7 +67,7 @@ class test_cursor17(wttest.WiredTigerTestCase): for key in range(total_keys): self.session.begin_transaction() cursor[key] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) commit_timestamp += 1 # Delete everything on the table except for the first and the last KV pair. @@ -78,13 +75,13 @@ class test_cursor17(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(key) self.assertEqual(cursor.remove(),0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) commit_timestamp += 1 # Take a checkpoint to reconcile the pages. self.session.checkpoint() - self.session.begin_transaction('read_timestamp=' + timestamp_str(commit_timestamp)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(commit_timestamp)) # Position the cursor on the first record. cursor.set_key(0) self.assertEqual(cursor.search(), 0) @@ -107,11 +104,11 @@ class test_cursor17(wttest.WiredTigerTestCase): # Update a key in the middle of the table. self.session.begin_transaction() cursor[total_keys // 2] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) commit_timestamp += 1 # Make sure we can reach a the record we updated in the middle of the table. - self.session.begin_transaction('read_timestamp=' + timestamp_str(commit_timestamp)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(commit_timestamp)) # Position the cursor on the first record. cursor.set_key(0) self.assertEqual(cursor.search(), 0) diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode05.py b/src/third_party/wiredtiger/test/suite/test_debug_mode05.py index f322dd98f7a..bb480138997 100644 --- a/src/third_party/wiredtiger/test/suite/test_debug_mode05.py +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode05.py @@ -28,9 +28,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' %t - # test_debug_mode05.py # As per WT-5046, the debug table logging settings prevent rollback to # stable in the presence of prepared transactions. @@ -44,7 +41,7 @@ class test_debug_mode05(wttest.WiredTigerTestCase): def test_table_logging_rollback_to_stable(self): self.session.create(self.uri, 'key_format=i,value_format=u,log=(enabled=false)') - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() # Try doing a normal prepared txn and then rollback to stable. @@ -53,11 +50,11 @@ class test_debug_mode05(wttest.WiredTigerTestCase): for i in range(1, 50): cursor[i] = b'a' * 100 self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(150)) + 'prepare_timestamp=' + self.timestamp_str(150)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(200)) + 'commit_timestamp=' + self.timestamp_str(200)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(250)) + 'durable_timestamp=' + self.timestamp_str(250)) self.session.commit_transaction() cursor.close() @@ -70,11 +67,11 @@ class test_debug_mode05(wttest.WiredTigerTestCase): # Therefore, we're specifically not doing any operations here. self.session.begin_transaction() self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(300)) + 'prepare_timestamp=' + self.timestamp_str(300)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(350)) + 'commit_timestamp=' + self.timestamp_str(350)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(400)) + 'durable_timestamp=' + self.timestamp_str(400)) self.session.commit_transaction() # The aforementioned bug resulted in a failure in rollback to stable. @@ -88,7 +85,7 @@ class test_debug_mode05(wttest.WiredTigerTestCase): for i in range(1, 50): cursor[i] = b'b' * 100 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(450)) + 'commit_timestamp=' + self.timestamp_str(450)) cursor.close() self.conn.rollback_to_stable() diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode07.py b/src/third_party/wiredtiger/test/suite/test_debug_mode07.py index c957fe57799..6c1432937fe 100644 --- a/src/third_party/wiredtiger/test/suite/test_debug_mode07.py +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode07.py @@ -31,7 +31,7 @@ import wttest, wiredtiger # test_debug_mode07.py # Test the debug mode settings. Test realloc_exact use (from WT-4919). class test_debug_mode07(wttest.WiredTigerTestCase): - conn_config = 'log=(enabled=true),debug_mode=(realloc_exact=true),statistics=(all)' + conn_config = 'log=(enabled=true),debug_mode=(realloc_exact=true)' uri = 'file:test_debug_mode07' # Insert some data to ensure setting/unsetting the flag does not diff --git a/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py b/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py index aa5ed0aae65..dfad0d8869d 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py @@ -32,9 +32,6 @@ from suite_subprocess import suite_subprocess from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_durable_rollback_to_stable.py # Checking visibility and durability of updates with durable_timestamp and # with rollback to stable. @@ -44,8 +41,7 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces keyfmt = [ ('row-string', dict(keyfmt='S')), ('row-int', dict(keyfmt='i')), - # The commented columnar tests needs to be enabled once rollback to stable for columnar is fixed in (WT-5548). - # ('column-store', dict(keyfmt='r')), + ('column-store', dict(keyfmt='r')), ] types = [ ('file', dict(uri='file', ds=SimpleDataSet)), @@ -78,7 +74,7 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces cursor = session.open_cursor(uri, None) # Set stable timestamp to checkpoint initial data set. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() # Update all values with value 111 i.e. first update value. @@ -89,15 +85,15 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(220)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(220)) session.commit_transaction() # Check the values read are correct with different timestamps. # Read the initial dataset. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(150)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(150)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(i)) @@ -106,7 +102,7 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces # Read the first update value with timestamp. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(200)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(200)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) @@ -123,7 +119,7 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces session.commit_transaction() # Set a stable timestamp so that first update value is durable. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) # Update all values with value 222 i.e. second update value. self.assertEquals(cursor.reset(), 0) @@ -134,12 +130,12 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(200)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(200)) # Commit timestamp is earlier to stable timestamp but durable timestamp # is later than stable timestamp. Hence second update value is not durable. - session.timestamp_transaction('commit_timestamp=' + timestamp_str(240)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(300)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(240)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(300)) session.commit_transaction() # Checkpoint so that first update value will be visible and durable, @@ -160,8 +156,8 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces self.conn.rollback_to_stable() session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(250)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts01.py b/src/third_party/wiredtiger/test/suite/test_durable_ts01.py index 7195aa85569..cd20fbf6b35 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts01.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts01.py @@ -31,9 +31,6 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_durable_ts01.py # Checking visibility and durability of updates with durable_timestamp and # with restart. @@ -43,8 +40,7 @@ class test_durable_ts01(wttest.WiredTigerTestCase): keyfmt = [ ('row-string', dict(keyfmt='S')), ('row-int', dict(keyfmt='i')), - # The commented columnar tests needs to be enabled once rollback to stable for columnar is fixed in (WT-5548). - # ('column-store', dict(keyfmt='r')), + ('column-store', dict(keyfmt='r')), ] types = [ ('file', dict(uri='file', ds=SimpleDataSet)), @@ -77,7 +73,7 @@ class test_durable_ts01(wttest.WiredTigerTestCase): cursor = session.open_cursor(uri, None) # Set stable timestamp to checkpoint initial data set. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() # Update all values with value 111 i.e. first update value. @@ -88,15 +84,15 @@ class test_durable_ts01(wttest.WiredTigerTestCase): self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(220)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(220)) session.commit_transaction() # Check the values read are correct with different timestamps. # Read the initial dataset. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(150)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(150)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(i)) @@ -105,7 +101,7 @@ class test_durable_ts01(wttest.WiredTigerTestCase): # Read the first update value with timestamp. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(200)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(200)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) @@ -122,7 +118,7 @@ class test_durable_ts01(wttest.WiredTigerTestCase): session.commit_transaction() # Set a stable timestamp so that first update value is durable. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) # Update all values with value 222 i.e. second update value. self.assertEquals(cursor.reset(), 0) @@ -133,12 +129,12 @@ class test_durable_ts01(wttest.WiredTigerTestCase): self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(200)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(200)) # Commit timestamp is earlier to stable timestamp but durable timestamp # is later than stable timestamp. Hence second update value is not durable. - session.timestamp_transaction('commit_timestamp=' + timestamp_str(240)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(300)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(240)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(300)) session.commit_transaction() # Checkpoint so that first update value will be visible and durable, @@ -159,8 +155,8 @@ class test_durable_ts01(wttest.WiredTigerTestCase): self.reopen_conn() session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(250)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts02.py b/src/third_party/wiredtiger/test/suite/test_durable_ts02.py index b71743c232c..88dea1291a2 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts02.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts02.py @@ -31,9 +31,6 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_durable_ts03.py # Checking visibility and durability of updates with durable_timestamp class test_durable_ts03(wttest.WiredTigerTestCase): @@ -75,7 +72,7 @@ class test_durable_ts03(wttest.WiredTigerTestCase): cursor = session.open_cursor(uri, None) # Set stable timestamp to checkpoint initial data set. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() ''' @@ -90,15 +87,15 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) msg = "/is less than the commit timestamp/" # Check for error when commit timestamp > durable timestamp. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: session.commit_transaction('commit_timestamp=' +\ - timestamp_str(200) + ',durable_timestamp=' + timestamp_str(180)), msg) + self.timestamp_str(200) + ',durable_timestamp=' + self.timestamp_str(180)), msg) # Set a stable timestamp so that first update value is durable. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) # Scenario: 2 # Check to see durable timestamp < stable timestamp, returns error. @@ -111,13 +108,13 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) msg = "/is less than the stable timestamp/" # Check that error is returned when durable timestamp < stable timestamp. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: session.commit_transaction('commit_timestamp=' +\ - timestamp_str(200) + ',durable_timestamp=' + timestamp_str(240)), msg) + self.timestamp_str(200) + ',durable_timestamp=' + self.timestamp_str(240)), msg) ''' if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py index 298c607904d..deea2f3dfc5 100755 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py @@ -29,9 +29,6 @@ from helper import copy_wiredtiger_home import wiredtiger, wttest -def timestamp_str(t): - return '%x' %t - # test_durable_ts03.py # Check that the checkpoint honors the durable timestamp of updates. class test_durable_ts03(wttest.WiredTigerTestCase): @@ -48,8 +45,8 @@ class test_durable_ts03(wttest.WiredTigerTestCase): valueC = b"ccccc" * 100 # Start with setting a stable and oldest timestamp. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1) + \ - ',oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1) + \ + ',oldest_timestamp=' + self.timestamp_str(1)) # Load the data into the table. session = self.conn.open_session(self.session_config) @@ -57,12 +54,12 @@ class test_durable_ts03(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[i] = valueA - session.commit_transaction('commit_timestamp=' + timestamp_str(50)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) cursor.close() # Set the stable and the oldest timestamp to checkpoint initial data. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100) + \ - ',oldest_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100) + \ + ',oldest_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() # Update all the values within transaction. Commit the transaction with @@ -71,9 +68,9 @@ class test_durable_ts03(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[i] = valueB - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(220)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(220)) session.commit_transaction() # Check the checkpoint wrote only the durable updates. @@ -83,14 +80,14 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.assertEqual(value, valueA) self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(150)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(150)) for key, value in cursor: self.assertEqual(value, valueA) session.commit_transaction() # Read the updated data to confirm that it is visible. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(210)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(210)) for key, value in cursor: self.assertEqual(value, valueB) session.commit_transaction() @@ -102,8 +99,8 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.reopen_conn() session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(210) + \ - ',oldest_timestamp=' + timestamp_str(210)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(210) + \ + ',oldest_timestamp=' + self.timestamp_str(210)) for key, value in cursor: self.assertEqual(value, valueA) @@ -111,12 +108,12 @@ class test_durable_ts03(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[i] = valueC - session.prepare_transaction('prepare_timestamp=' + timestamp_str(220)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(230)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(240)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(220)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(230)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(240)) session.commit_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) self.session.checkpoint() cursor.close() session.close() @@ -124,8 +121,8 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.reopen_conn() session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250) + \ - ',oldest_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250) + \ + ',oldest_timestamp=' + self.timestamp_str(250)) for key, value in cursor: self.assertEqual(value, valueC) diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt08.py b/src/third_party/wiredtiger/test/suite/test_encrypt08.py index e15dd851908..5e4e065d3f3 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt08.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt08.py @@ -84,11 +84,10 @@ class test_encrypt08(wttest.WiredTigerTestCase): def test_encrypt(self): sysconfig = 'encryption=(name=sodium,{0}),'.format(self.sys_encrypt) - with self.expectedStdoutPattern('Failed wiredtiger_open'): - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: - self.reopen_conn(config = sysconfig), - self.msg) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: + self.reopen_conn(config = sysconfig), + self.msg) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_gc01.py b/src/third_party/wiredtiger/test/suite/test_gc01.py index 950c1e4ce54..8cec5461881 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc01.py +++ b/src/third_party/wiredtiger/test/suite/test_gc01.py @@ -36,11 +36,7 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_gc01.py - # Shared base class used by gc tests. class test_gc_base(wttest.WiredTigerTestCase): @@ -51,7 +47,7 @@ class test_gc_base(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def large_modifies(self, uri, value, ds, location, nbytes, nrows, commit_ts): @@ -63,12 +59,12 @@ class test_gc_base(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify(value, location, nbytes)] self.assertEqual(cursor.modify(mods), 0) - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, nrows, read_ts): session = self.session - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -100,8 +96,8 @@ class test_gc01(test_gc_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 @@ -119,8 +115,8 @@ class test_gc01(test_gc_base): self.check(bigvalue, uri, nrows, 10) # Pin oldest and stable to timestamp 100. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100) + - ',stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100) + + ',stable_timestamp=' + self.timestamp_str(100)) # Checkpoint to ensure that the history store is cleaned. self.session.checkpoint() @@ -152,8 +148,8 @@ class test_gc01(test_gc_base): self.check(bigvalue2, uri, nrows, 100) # Pin oldest and stable to timestamp 200. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200) + - ',stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200) + + ',stable_timestamp=' + self.timestamp_str(200)) # Checkpoint to ensure that the history store is cleaned. self.session.checkpoint() @@ -185,8 +181,8 @@ class test_gc01(test_gc_base): self.check(bigvalue, uri, nrows, 200) # Pin oldest and stable to timestamp 300. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(300) + - ',stable_timestamp=' + timestamp_str(300)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(300) + + ',stable_timestamp=' + self.timestamp_str(300)) # Checkpoint to ensure that the history store is cleaned. self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_gc02.py b/src/third_party/wiredtiger/test/suite/test_gc02.py index b42d3e23a47..7af18d02186 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc02.py +++ b/src/third_party/wiredtiger/test/suite/test_gc02.py @@ -30,9 +30,6 @@ from test_gc01 import test_gc_base from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_gc02.py # Test that checkpoint cleans the obsolete history store internal pages. class test_gc02(test_gc_base): @@ -49,8 +46,8 @@ class test_gc02(test_gc_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 @@ -76,8 +73,8 @@ class test_gc02(test_gc_base): c.close() # Pin oldest and stable to timestamp 100. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100) + - ',stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100) + + ',stable_timestamp=' + self.timestamp_str(100)) # Check that the new updates are only seen after the update timestamp. self.check(bigvalue2, uri, nrows, 100) @@ -114,8 +111,8 @@ class test_gc02(test_gc_base): self.check(bigvalue, uri, nrows, 200) # Pin oldest and stable to timestamp 200. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200) + - ',stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200) + + ',stable_timestamp=' + self.timestamp_str(200)) # Checkpoint to ensure that the history store is cleaned. self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_gc03.py b/src/third_party/wiredtiger/test/suite/test_gc03.py index c605a07963e..386a7bea801 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc03.py +++ b/src/third_party/wiredtiger/test/suite/test_gc03.py @@ -30,9 +30,6 @@ from test_gc01 import test_gc_base from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_gc03.py # Test that checkpoint cleans the obsolete history store pages that are in-memory. class test_gc03(test_gc_base): @@ -61,8 +58,8 @@ class test_gc03(test_gc_base): ds_extra.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 @@ -84,8 +81,8 @@ class test_gc03(test_gc_base): self.assertGreater(self.get_stat(stat.conn.cc_pages_visited), 0) # Pin oldest and stable to timestamp 100. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100) + - ',stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100) + + ',stable_timestamp=' + self.timestamp_str(100)) # Check that the new updates are only seen after the update timestamp. self.check(bigvalue2, uri, nrows, 100) @@ -113,8 +110,8 @@ class test_gc03(test_gc_base): self.check(bigvalue2, uri, nrows, 100) # Pin oldest and stable to timestamp 200. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200) + - ',stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200) + + ',stable_timestamp=' + self.timestamp_str(200)) # Update on extra table. self.large_updates(uri_extra, bigvalue, ds_extra, 100, 210) @@ -129,8 +126,8 @@ class test_gc03(test_gc_base): self.check(bigvalue, uri, nrows, 200) # Pin oldest and stable to timestamp 300. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(300) + - ',stable_timestamp=' + timestamp_str(300)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(300) + + ',stable_timestamp=' + self.timestamp_str(300)) self.large_updates(uri_extra, bigvalue, ds_extra, 100, 310) self.large_updates(uri_extra, bigvalue2, ds_extra, 100, 320) diff --git a/src/third_party/wiredtiger/test/suite/test_gc04.py b/src/third_party/wiredtiger/test/suite/test_gc04.py index 1db0cceb24f..4504bd3f75e 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc04.py +++ b/src/third_party/wiredtiger/test/suite/test_gc04.py @@ -30,9 +30,6 @@ from test_gc01 import test_gc_base from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_gc04.py # Test that checkpoint must not clean the pages that are not obsolete. class test_gc04(test_gc_base): @@ -55,8 +52,8 @@ class test_gc04(test_gc_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_gc05.py b/src/third_party/wiredtiger/test/suite/test_gc05.py index 1d34ce34dc6..75bf3d253e8 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc05.py +++ b/src/third_party/wiredtiger/test/suite/test_gc05.py @@ -29,9 +29,6 @@ from test_gc01 import test_gc_base from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_gc05.py # Verify a locked checkpoint is not removed during garbage collection. class test_gc05(test_gc_base): @@ -52,8 +49,8 @@ class test_gc05(test_gc_base): ds.populate() # Set the oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) # Insert values with varying timestamps. self.large_updates(uri, value_x, ds, nrows, 20) @@ -70,8 +67,8 @@ class test_gc05(test_gc_base): ckpt_cursor = self.session.open_cursor(uri, None, "checkpoint=checkpoint_one") # Move the oldest and stable timestamps to 40. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(40) + - ',stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(40) + + ',stable_timestamp=' + self.timestamp_str(40)) # Insert values with varying timestamps. self.large_updates(uri, value_z, ds, nrows, 50) @@ -79,8 +76,8 @@ class test_gc05(test_gc_base): self.large_updates(uri, value_x, ds, nrows, 70) # Move the oldest and stable timestamps to 70. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(70) + - ',stable_timestamp=' + timestamp_str(70)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(70) + + ',stable_timestamp=' + self.timestamp_str(70)) # Perform a checkpoint. self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_hs01.py b/src/third_party/wiredtiger/test/suite/test_hs01.py index 90a7be1cd0b..52c68cb699d 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs01.py +++ b/src/third_party/wiredtiger/test/suite/test_hs01.py @@ -28,72 +28,70 @@ from helper import copy_wiredtiger_home import wiredtiger, wttest +from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs01.py -# Smoke tests to ensure history store tables are working. +# Test that update and modify operations are durable across crash and recovery. +# Additionally test that checkpoint inserts content into the history store. class test_hs01(wttest.WiredTigerTestCase): - # Force a small cache. - conn_config = 'cache_size=50MB' + conn_config = 'cache_size=200MB,statistics=(all)' session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')) + ('row_integer', dict(key_format='i')), + ('row_string', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) + def get_stat(self, stat): + stat_cursor = self.session.open_cursor('statistics:') + val = stat_cursor[stat][2] + stat_cursor.close() + return val + def large_updates(self, session, uri, value, ds, nrows, timestamp=False): - # Update a large number of records, we'll hang if the history store table - # isn't doing its thing. cursor = session.open_cursor(uri) - for i in range(1, 10000): + for i in range(1, nrows): if timestamp == True: session.begin_transaction() - cursor.set_key(ds.key(nrows + i)) + cursor.set_key(ds.key(i)) cursor.set_value(value) self.assertEqual(cursor.update(), 0) if timestamp == True: - session.commit_transaction('commit_timestamp=' + timestamp_str(i + 1)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(i + 1)) cursor.close() def large_modifies(self, session, uri, offset, ds, nrows, timestamp=False): - # Modify a large number of records, we'll hang if the history store table - # isn't doing its thing. cursor = session.open_cursor(uri) - for i in range(1, 10000): - if timestamp == True: - session.begin_transaction() - cursor.set_key(ds.key(nrows + i)) + for i in range(1, nrows): + # Unlike inserts and updates, modify operations do not implicitly start/commit a transaction. + # Hence, we begin/commit transaction manually. + session.begin_transaction() + cursor.set_key(ds.key(i)) mods = [] mod = wiredtiger.Modify('A', offset, 1) mods.append(mod) - self.assertEqual(cursor.modify(mods), 0) + if timestamp == True: - session.commit_transaction('commit_timestamp=' + timestamp_str(i + 1)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(i + 1)) + else: + session.commit_transaction() cursor.close() - def durable_check(self, check_value, uri, ds, nrows): - # Checkpoint and backup so as to simulate recovery - self.session.checkpoint() + def durable_check(self, check_value, uri, ds): + # Simulating recovery. newdir = "BACKUP" copy_wiredtiger_home(self, '.', newdir, True) - conn = self.setUpConnectionOpen(newdir) session = self.setUpSessionOpen(conn) cursor = session.open_cursor(uri, None) - # Skip the initial rows, which were not updated - for i in range(0, nrows+1): - self.assertEqual(cursor.next(), 0) - if check_value != cursor.get_value(): - session.breakpoint() + + cursor.next() self.assertTrue(check_value == cursor.get_value(), - "for key " + str(i) + ", expected " + str(check_value) + + "for key " + str(1) + ", expected " + str(check_value) + ", got " + str(cursor.get_value())) cursor.close() session.close() @@ -102,15 +100,15 @@ class test_hs01(wttest.WiredTigerTestCase): def test_hs(self): # Create a small table. uri = "table:test_hs01" - nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format='u') + ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format='u') ds.populate() - bigvalue = b"aaaaa" * 100 - # Initially load huge data. + # Initially insert a lot of data. + nrows = 10000 + bigvalue = b"aaaaa" * 100 cursor = self.session.open_cursor(uri) - for i in range(1, 10000): - cursor.set_key(ds.key(nrows + i)) + for i in range(1, nrows): + cursor.set_key(ds.key(i)) cursor.set_value(bigvalue) self.assertEqual(cursor.insert(), 0) cursor.close() @@ -119,11 +117,19 @@ class test_hs01(wttest.WiredTigerTestCase): # Scenario: 1 # Check to see if the history store is working with the old reader. bigvalue2 = b"ccccc" * 100 + # Open session 2. session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') + # Large updates with session 1. self.large_updates(self.session, uri, bigvalue2, ds, nrows) - # Check to see the value after recovery. - self.durable_check(bigvalue2, uri, ds, nrows) + + # Checkpoint and then assert that the (nrows-1) insertions were moved to history store from data store. + self.session.checkpoint() + hs_writes = self.get_stat(stat.conn.cache_hs_insert) + self.assertEqual(hs_writes, nrows-1) + + # Check to see the latest updated value after recovery. + self.durable_check(bigvalue2, uri, ds) session2.rollback_transaction() session2.close() @@ -131,34 +137,48 @@ class test_hs01(wttest.WiredTigerTestCase): # Check to see the history store working with modify operations. bigvalue3 = b"ccccc" * 100 bigvalue3 = b'AA' + bigvalue3[2:] + # Open session 2. session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') - # Apply two modify operations - replacing the first two items with 'A'. - self.session.begin_transaction() + # Apply two modify operations (session1)- replacing the first two items with 'A'. self.large_modifies(self.session, uri, 0, ds, nrows) self.large_modifies(self.session, uri, 1, ds, nrows) - self.session.commit_transaction() - # Check to see the value after recovery. - self.durable_check(bigvalue3, uri, ds, nrows) + + # Checkpoint and then assert if updates (nrows-1) and first large modifies (nrows-1) were moved to history store. + self.session.checkpoint() + hs_writes = self.get_stat(stat.conn.cache_hs_insert) + # The updates in data store: nrows-1 + # The first modifies in cache: nrows-1 + # The stats was already set at: nrows-1 (previous hs stats) + # Total: (nrows-1)*3 + self.assertEqual(hs_writes, (nrows-1) * 3) + + # Check to see the modified value after recovery. + self.durable_check(bigvalue3, uri, ds) session2.rollback_transaction() session2.close() - # FIXME-WT-7120: Rollback to stable support for column store is not implemented, and it - # fails only when it is used with timestamps. - if self.key_format == 'r': - return - # Scenario: 3 # Check to see if the history store is working with the old timestamp. bigvalue4 = b"ddddd" * 100 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) self.large_updates(self.session, uri, bigvalue4, ds, nrows, timestamp=True) - # Check to see data can be see only till the stable_timestamp - self.durable_check(bigvalue3, uri, ds, nrows) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(i + 1)) + self.session.checkpoint() + # Check if the (nrows-1) modifications were moved to history store from data store. + # The stats was already set at: (nrows-1)*3 (previous hs stats) + # Total: (nrows-1)*4 + hs_writes = self.get_stat(stat.conn.cache_hs_insert) + self.assertEqual(hs_writes, (nrows-1) * 4) + + # Check to see data can be see only till the stable_timestamp. + self.durable_check(bigvalue3, uri, ds) + + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(i + 1)) + # No need to check history store stats here as nothing will be moved. + self.session.checkpoint() # Check that the latest data can be seen. - self.durable_check(bigvalue4, uri, ds, nrows) + self.durable_check(bigvalue4, uri, ds) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hs02.py b/src/third_party/wiredtiger/test/suite/test_hs02.py index 6f92f422161..42cd3b327ce 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs02.py +++ b/src/third_party/wiredtiger/test/suite/test_hs02.py @@ -31,9 +31,6 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs02.py # Test that truncate with history store entries and timestamps gives expected results. class test_hs02(wttest.WiredTigerTestCase): @@ -42,9 +39,8 @@ class test_hs02(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ - ('string', dict(key_format='S')), - # FIXME-WT-7120: Uncomment the column store scenario when rollback to stable support is complete. - # ('column', dict(key_format='r')) + ('string-row', dict(key_format='S')), + ('column', dict(key_format='r')) ] scenarios = make_scenarios(key_format_values) @@ -55,12 +51,12 @@ class test_hs02(wttest.WiredTigerTestCase): for i in range(1, nrows + 1): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, nrows, read_ts): session = self.session - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -84,8 +80,8 @@ class test_hs02(wttest.WiredTigerTestCase): ds2.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 self.large_updates(uri, bigvalue, ds, nrows // 3, 1) @@ -110,7 +106,7 @@ class test_hs02(wttest.WiredTigerTestCase): end.set_key(ds.key(nrows // 2)) self.session.truncate(None, None, end) end.close() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(200)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(200)) # Check that the truncate is visible after commit self.check(bigvalue2, uri, nrows // 2, 200) diff --git a/src/third_party/wiredtiger/test/suite/test_hs03.py b/src/third_party/wiredtiger/test/suite/test_hs03.py index 6980859ced6..54a54bf1f0f 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs03.py +++ b/src/third_party/wiredtiger/test/suite/test_hs03.py @@ -32,9 +32,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs03.py # Ensure checkpoints don't read too unnecessary history store entries. class test_hs03(wttest.WiredTigerTestCase): @@ -43,8 +40,8 @@ class test_hs03(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')) + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -61,7 +58,7 @@ class test_hs03(wttest.WiredTigerTestCase): for i in range(nrows + 1, nrows + nops + 1): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(i)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(i)) cursor.close() def test_checkpoint_hs_reads(self): @@ -81,7 +78,7 @@ class test_hs03(wttest.WiredTigerTestCase): # Check to see the history store working with old timestamp. bigvalue2 = b"ddddd" * 100 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) hs_writes_start = self.get_stat(stat.conn.cache_write_hs) self.large_updates(self.session, uri, bigvalue2, ds, nrows, 10000) @@ -91,7 +88,7 @@ class test_hs03(wttest.WiredTigerTestCase): self.assertGreaterEqual(hs_writes, 0) for ts in range(2, 4): - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(ts)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(ts)) # Now just update one record and checkpoint again. self.large_updates(self.session, uri, bigvalue2, ds, nrows, 1) diff --git a/src/third_party/wiredtiger/test/suite/test_hs05.py b/src/third_party/wiredtiger/test/suite/test_hs05.py index 5a66d04f546..6f869e83aaa 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs05.py +++ b/src/third_party/wiredtiger/test/suite/test_hs05.py @@ -32,9 +32,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs05.py # Verify hs_score reflects cache pressure due to history # even if we're not yet actively pushing into the history store file. @@ -47,8 +44,8 @@ class test_hs05(wttest.WiredTigerTestCase): stable = 1 key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')) + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -66,7 +63,7 @@ class test_hs05(wttest.WiredTigerTestCase): for i in range(nrows + 1, nrows + nops + 1): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(self.stable + i)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(self.stable + i)) cursor.close() score_end = self.get_stat(stat.conn.cache_hs_score) score_diff = score_end - score_start @@ -93,7 +90,7 @@ class test_hs05(wttest.WiredTigerTestCase): self.session.checkpoint() # Pin the oldest timestamp so that all history has to stay. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) # Loop a couple times, partly filling the cache but not # overfilling it to see the history store score value change # even if the history store is not yet in use. @@ -103,7 +100,7 @@ class test_hs05(wttest.WiredTigerTestCase): loop_start = self.get_stat(stat.conn.cache_hs_score) for i in range(1, 9): bigvalue2 = valstr[i].encode() * 50 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(self.stable)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.stable)) entries_start = self.get_stat(stat.conn.cache_hs_insert) score_start = self.get_stat(stat.conn.cache_hs_score) self.pr("Update iteration: " + str(i) + " Value: " + str(bigvalue2)) @@ -127,14 +124,14 @@ class test_hs05(wttest.WiredTigerTestCase): # By moving the oldest after updating we should see the score drop # to zero. score_start = loop_end - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(self.stable)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(self.stable)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.stable)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(self.stable)) for i in range(9, 11): bigvalue2 = valstr[i].encode() * 50 self.pr("Update iteration with oldest: " + str(i) + " Value: " + str(bigvalue2)) self.large_updates(self.session, uri, bigvalue2, ds, nrows, nrows) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(self.stable)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(self.stable)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.stable)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(self.stable)) self.stable += nrows score_end = self.get_stat(stat.conn.cache_hs_score) self.assertLess(score_end, score_start) diff --git a/src/third_party/wiredtiger/test/suite/test_hs06.py b/src/third_party/wiredtiger/test/suite/test_hs06.py index a7747e281e4..1c2f18f09f2 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs06.py +++ b/src/third_party/wiredtiger/test/suite/test_hs06.py @@ -30,9 +30,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs06.py # Verify that triggering history store usage does not cause a spike in memory usage # to form an update chain from the history store contents. @@ -48,10 +45,11 @@ class test_hs06(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')) + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) + nrows = 2000 def get_stat(self, stat): stat_cursor = self.session.open_cursor('statistics:') @@ -77,21 +75,21 @@ class test_hs06(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Load 1Mb of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 1Mb of data with a later timestamp. self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Write a version of the data to disk. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2)) self.session.checkpoint() # Check the checkpoint wrote the expected values. @@ -102,7 +100,7 @@ class test_hs06(wttest.WiredTigerTestCase): # # cursor2 = self.session.open_cursor(uri, None, 'checkpoint=WiredTigerCheckpoint') cursor2 = self.session.open_cursor(uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) for key, value in cursor2: self.assertEqual(value, value1) self.session.commit_transaction() @@ -113,8 +111,8 @@ class test_hs06(wttest.WiredTigerTestCase): # Whenever we request something out of cache of timestamp 2, we should # be reading it straight from the history store without initialising a full # update chain of every version of the data. - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) - for i in range(1, 2000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) + for i in range(1, self.nrows): self.assertEqual(cursor[self.create_key(i)], value1) self.session.rollback_transaction() @@ -143,34 +141,34 @@ class test_hs06(wttest.WiredTigerTestCase): value2 = 'd' * 500 # Load 1Mb of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load a slight modification with a later timestamp. self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor.set_key(self.create_key(i)) mods = [wiredtiger.Modify('B', 100, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # And another. self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor.set_key(self.create_key(i)) mods = [wiredtiger.Modify('C', 200, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Now write something completely different. self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Now the latest version will get written to the data file. self.session.checkpoint() @@ -187,8 +185,8 @@ class test_hs06(wttest.WiredTigerTestCase): # t4: full update in las # t3: (reverse delta in las) <= We're querying for t4 so we begin here. # t2: value2 (full update in las) - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) - for i in range(1, 2000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) + for i in range(1, self.nrows): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() @@ -203,17 +201,12 @@ class test_hs06(wttest.WiredTigerTestCase): # t4: full update in las <= We're querying for t4 and we return. # t3: (reverse delta in las) # t2: value2 (full update in las) - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) - for i in range(1, 2000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) + for i in range(1, self.nrows): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() def test_hs_prepare_reads(self): - # FIXME-WT-6061: Prepare reads currently not supported with columnar store. - # Remove this once prepare reads is supported. - if self.key_format == 'r': - return - # Create a small table. uri = "table:test_hs06" create_params = 'key_format={},value_format=S'.format(self.key_format) @@ -222,12 +215,12 @@ class test_hs06(wttest.WiredTigerTestCase): value1 = 'a' * 500 value2 = 'b' * 500 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load prepared data and leave it in a prepared state. prepare_session = self.conn.open_session(self.session_config) @@ -236,20 +229,20 @@ class test_hs06(wttest.WiredTigerTestCase): for i in range(1, 11): prepare_cursor[self.create_key(i)] = value2 prepare_session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(3)) + 'prepare_timestamp=' + self.timestamp_str(3)) # Write some more to cause eviction of the prepared data. - for i in range(11, 2000): + for i in range(11, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.checkpoint() # Try to read every key of the prepared data again. # Ensure that we read the history store to find the prepared update and # return a prepare conflict as appropriate. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): cursor.set_key(self.create_key(i)) self.assertRaisesException( @@ -259,9 +252,9 @@ class test_hs06(wttest.WiredTigerTestCase): self.session.rollback_transaction() prepare_session.commit_transaction( - 'commit_timestamp=' + timestamp_str(5) + ',durable_timestamp=' + timestamp_str(6)) + 'commit_timestamp=' + self.timestamp_str(5) + ',durable_timestamp=' + self.timestamp_str(6)) - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) for i in range(1, 11): self.assertEquals(value2, cursor[self.create_key(i)]) self.session.rollback_transaction() @@ -278,12 +271,12 @@ class test_hs06(wttest.WiredTigerTestCase): value4 = 'd' * 500 # Load 1Mb of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Do two different updates to the same key with the same timestamp. # We want to make sure that the second value is the one that is visible even after eviction. @@ -291,16 +284,16 @@ class test_hs06(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor[self.create_key(i)] = value2 cursor[self.create_key(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Write a newer value on top. - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value4 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Ensure that we see the last of the two updates that got applied. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.assertEquals(cursor[self.create_key(i)], value3) self.session.rollback_transaction() @@ -315,12 +308,12 @@ class test_hs06(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Load 1Mb of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply three sets of modifies. # They specifically need to be in separate modify calls. @@ -330,7 +323,7 @@ class test_hs06(wttest.WiredTigerTestCase): self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) expected = list(value1) expected[100] = 'B' @@ -339,13 +332,13 @@ class test_hs06(wttest.WiredTigerTestCase): expected = str().join(expected) # Write a newer value on top. - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Go back and read. We should get the initial value with the 3 modifies applied on top. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() @@ -361,25 +354,25 @@ class test_hs06(wttest.WiredTigerTestCase): # Load 5Mb of data. self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) for i in range(1, 10000): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply three sets of modifies. for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Since the stable timestamp is still at 1, there will be no birthmark record. # History store instantiation should choose this update since it is the most recent. @@ -388,7 +381,7 @@ class test_hs06(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Make a bunch of updates to another table to flush everything out of cache. uri2 = 'table:test_hs06_extra' @@ -397,7 +390,7 @@ class test_hs06(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor2[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(6)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(6)) expected = list(value1) expected[100] = 'B' @@ -406,7 +399,7 @@ class test_hs06(wttest.WiredTigerTestCase): expected = str().join(expected) # Go back and read. We should get the initial value with the 3 modifies applied on top. - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) for i in range(1, 11): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() @@ -422,7 +415,7 @@ class test_hs06(wttest.WiredTigerTestCase): # Load 5Mb of data. self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) # The base update is at timestamp 1. # When we history store evict these pages, the base update is the only thing behind @@ -431,26 +424,26 @@ class test_hs06(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) # Apply three sets of modifies. for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Make a bunch of updates to another table to flush everything out of cache. uri2 = 'table:test_hs06_extra' @@ -459,7 +452,7 @@ class test_hs06(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor2[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) expected = list(value1) expected[100] = 'B' @@ -468,7 +461,7 @@ class test_hs06(wttest.WiredTigerTestCase): expected = str().join(expected) # Go back and read. - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) for i in range(1, 11): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() @@ -483,44 +476,44 @@ class test_hs06(wttest.WiredTigerTestCase): value2 = 'b' * 500 self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) # Base update. for i in range(1, 10000): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply three sets of modifies. for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # This is the one we want to be selected by the checkpoint. for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Apply another update and evict the pages with the modifies out of cache. for i in range(1, 10000): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(6)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(6)) # Checkpoint such that the modifies will be selected. When we grab it from the history # store, we'll need to unflatten it before using it for reconciliation. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) self.session.checkpoint() expected = list(value1) @@ -530,7 +523,7 @@ class test_hs06(wttest.WiredTigerTestCase): expected = str().join(expected) # Check that the correct value is visible after checkpoint. - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) for i in range(1, 11): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs07.py b/src/third_party/wiredtiger/test/suite/test_hs07.py index 58d44374e42..7e2562e5420 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs07.py +++ b/src/third_party/wiredtiger/test/suite/test_hs07.py @@ -32,9 +32,6 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs07.py # Test that the history store sweep cleans the obsolete history store entries and gives expected results. class test_hs07(wttest.WiredTigerTestCase): @@ -45,7 +42,7 @@ class test_hs07(wttest.WiredTigerTestCase): key_format_values = ( ('column', dict(key_format='r')), - ('int', dict(key_format='i')) + ('integer-row', dict(key_format='i')) ) scenarios = make_scenarios(key_format_values) @@ -56,12 +53,12 @@ class test_hs07(wttest.WiredTigerTestCase): for i in range(1, nrows + 1): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, nrows, read_ts): session = self.session - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -85,8 +82,8 @@ class test_hs07(wttest.WiredTigerTestCase): ds2.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 @@ -102,8 +99,8 @@ class test_hs07(wttest.WiredTigerTestCase): self.check(bigvalue, uri, nrows, 100) # Pin oldest and stable to timestamp 100. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100) + - ',stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100) + + ',stable_timestamp=' + self.timestamp_str(100)) # Sleep here to let that sweep server to trigger cleanup of obsolete entries. time.sleep(10) @@ -118,7 +115,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('A', 10, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(110)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(110)) # Load a slight modification with a later timestamp. self.session.begin_transaction() @@ -126,7 +123,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('B', 20, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(120)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(120)) # Load a slight modification with a later timestamp. self.session.begin_transaction() @@ -134,7 +131,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('C', 30, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(130)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(130)) cursor.close() # Second set of update operations with increased timestamp @@ -147,8 +144,8 @@ class test_hs07(wttest.WiredTigerTestCase): self.check(bigvalue2, uri, nrows, 200) # Pin oldest and stable to timestamp 300. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200) + - ',stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200) + + ',stable_timestamp=' + self.timestamp_str(200)) # Sleep here to let that sweep server to trigger cleanup of obsolete entries. time.sleep(10) @@ -163,7 +160,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('A', 10, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(210)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(210)) # Load a slight modification with a later timestamp. self.session.begin_transaction() @@ -171,7 +168,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('B', 20, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(220)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(220)) # Load a slight modification with a later timestamp. self.session.begin_transaction() @@ -179,7 +176,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('C', 30, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(230)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(230)) cursor.close() # Third set of update operations with increased timestamp @@ -192,8 +189,8 @@ class test_hs07(wttest.WiredTigerTestCase): self.check(bigvalue, uri, nrows, 300) # Pin oldest and stable to timestamp 400. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(300) + - ',stable_timestamp=' + timestamp_str(300)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(300) + + ',stable_timestamp=' + self.timestamp_str(300)) # Sleep here to let that sweep server to trigger cleanup of obsolete entries. time.sleep(10) diff --git a/src/third_party/wiredtiger/test/suite/test_hs08.py b/src/third_party/wiredtiger/test/suite/test_hs08.py index 0cb26149687..13769f2f19f 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs08.py +++ b/src/third_party/wiredtiger/test/suite/test_hs08.py @@ -30,9 +30,6 @@ import wiredtiger, wttest, time from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs08.py # Verify modify insert into history store logic. class test_hs08(wttest.WiredTigerTestCase): @@ -40,7 +37,7 @@ class test_hs08(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), + ('integer-row', dict(key_format='i')), ] scenarios = make_scenarios(key_format_values) @@ -57,27 +54,27 @@ class test_hs08(wttest.WiredTigerTestCase): self.session.create(uri, create_params) # Insert a full value. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) self.session.begin_transaction() cursor[1] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Insert 3 modifies in separate transactions. self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('A', 1000, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 1001, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 1002, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Call checkpoint. self.session.checkpoint('use_timestamp=true') @@ -89,15 +86,15 @@ class test_hs08(wttest.WiredTigerTestCase): self.assertEqual(squashed_write, 0) # Validate that we see the correct value at each of the timestamps. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) self.assertEqual(cursor[1], value1 + 'A') self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) self.assertEqual(cursor[1], value1 + 'AB') self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) self.assertEqual(cursor[1], value1 + 'ABC') self.session.commit_transaction() @@ -107,12 +104,12 @@ class test_hs08(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 1000, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(7)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(7)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('E', 1001, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(8)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(8)) # Call checkpoint again. self.session.checkpoint('use_timestamp=true') @@ -126,11 +123,11 @@ class test_hs08(wttest.WiredTigerTestCase): # Validate that we see the expected value on the modifies, this # scenario tests the logic that will retrieve a full value for # a modify previously inserted into the history store. - self.session.begin_transaction('read_timestamp=' + timestamp_str(7)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(7)) self.assertEqual(cursor[1], value1 + 'DBC') self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(8)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(8)) self.assertEqual(cursor[1], value1 + 'DEC') self.session.commit_transaction() @@ -140,7 +137,7 @@ class test_hs08(wttest.WiredTigerTestCase): self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('H', 1004, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(9)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(9)) # Call checkpoint again. self.session.checkpoint('use_timestamp=true') @@ -157,13 +154,13 @@ class test_hs08(wttest.WiredTigerTestCase): cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(11)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(11)) # Call checkpoint again. self.session.checkpoint('use_timestamp=true') @@ -179,17 +176,17 @@ class test_hs08(wttest.WiredTigerTestCase): # modify to guarantee we squash zero modifies. self.session.begin_transaction() cursor.set_key(1) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(12)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(12)) self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(13)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(13)) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) self.session.commit_transaction() self.session.begin_transaction() cursor.set_key(1) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(14)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(14)) self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(15)) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs09.py b/src/third_party/wiredtiger/test/suite/test_hs09.py index 4217da2b02d..1b7993239b2 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs09.py +++ b/src/third_party/wiredtiger/test/suite/test_hs09.py @@ -34,23 +34,21 @@ import wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs09.py # Verify that we write the newest committed version to data store and the # second newest committed version to history store. class test_hs09(wttest.WiredTigerTestCase): # Force a small cache. - conn_config = 'cache_size=20MB,statistics=(fast)' + conn_config = 'cache_size=20MB' session_config = 'isolation=snapshot' uri = "table:test_hs09" key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')), + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')), ] scenarios = make_scenarios(key_format_values) + nrows = 1000 def create_key(self, i): if self.key_format == 'S': @@ -101,18 +99,18 @@ class test_hs09(wttest.WiredTigerTestCase): value3 = 'c' * 500 # Load 500KB of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 500KB of data with a later timestamp. self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Uncommitted changes. self.session.begin_transaction() @@ -122,11 +120,6 @@ class test_hs09(wttest.WiredTigerTestCase): self.check_ckpt_hs(value2, value1, 2, 3) def test_prepared_updates_not_written_to_hs(self): - # FIXME-WT-6061: Prepare reads currently not supported with columnar store. - # Remove this once prepare reads is supported. - if self.key_format == 'r': - return - # Create a small table. create_params = 'key_format={},value_format=S'.format(self.key_format) self.session.create(self.uri, create_params) @@ -136,30 +129,30 @@ class test_hs09(wttest.WiredTigerTestCase): value3 = 'c' * 500 # Load 1MB of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() for i in range(1, 2000): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 1MB of data with a later timestamp. self.session.begin_transaction() for i in range(1, 2000): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Prepare some updates. self.session.begin_transaction() for i in range(1, 11): cursor[self.create_key(i)] = value3 - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(4)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(4)) # We can expect prepared values to show up in data store if the eviction runs between now # and the time when we open a cursor on the user table. self.check_ckpt_hs(value2, value1, 2, 3, True) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5) + - ',durable_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5) + + ',durable_timestamp=' + self.timestamp_str(5)) def test_write_newest_version_to_data_store(self): # Create a small table. @@ -170,18 +163,18 @@ class test_hs09(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Load 500KB of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 500KB of data with a later timestamp. self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.check_ckpt_hs(value2, value1, 2, 3) @@ -194,26 +187,26 @@ class test_hs09(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Load 500KB of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 500KB of data with a later timestamp. self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Delete records. self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor = self.session.open_cursor(self.uri) cursor.set_key(self.create_key(i)) self.assertEqual(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.check_ckpt_hs(value2, value1, 2, 3) diff --git a/src/third_party/wiredtiger/test/suite/test_hs10.py b/src/third_party/wiredtiger/test/suite/test_hs10.py index 9e3cb4f2699..256ecff4f7b 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs10.py +++ b/src/third_party/wiredtiger/test/suite/test_hs10.py @@ -30,9 +30,6 @@ import wiredtiger, wttest, time from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs10.py # Verify modify read after eviction. class test_hs10(wttest.WiredTigerTestCase): @@ -40,7 +37,7 @@ class test_hs10(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = ( ('column', dict(key_format='r')), - ('int', dict(key_format='i')) + ('integer-row', dict(key_format='i')) ) scenarios = make_scenarios(key_format_values) @@ -61,28 +58,28 @@ class test_hs10(wttest.WiredTigerTestCase): session2.create(uri2, create_params) cursor2 = session2.open_cursor(uri2) # Insert a full value. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) self.session.begin_transaction() cursor[1] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Insert 3 modifies in separate transactions. self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('A', 1000, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 1001, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 1002, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) self.session.checkpoint() # Insert a whole bunch of data into the other table to force wiredtiger to evict data @@ -91,20 +88,20 @@ class test_hs10(wttest.WiredTigerTestCase): cursor2[i] = value2 # Validate that we see the correct value at each of the timestamps. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) cursor.set_key(1) cursor.search() self.assertEqual(cursor[1], value1 + 'A') self.session.commit_transaction() cursor2 = self.session.open_cursor(uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) cursor2.set_key(1) cursor2.search() self.assertEqual(cursor2.get_value(), value1 + 'AB') self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) self.assertEqual(cursor[1], value1 + 'ABC') self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs11.py b/src/third_party/wiredtiger/test/suite/test_hs11.py index 7f0bbfe141a..fd2afc0d977 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs11.py +++ b/src/third_party/wiredtiger/test/suite/test_hs11.py @@ -30,23 +30,22 @@ import wiredtiger, wttest from wtscenario import make_scenarios from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_hs11.py # Ensure that updates without timestamps clear the history store records. class test_hs11(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,statistics=(all)' session_config = 'isolation=snapshot' - key_format_values = ( - ('int', dict(key_format='i')), - ('string', dict(key_format='S')), - ('column', dict(key_format='r')) - ) - scenarios = make_scenarios([ + key_format_values = [ + ('column', dict(key_format='r')), + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')) + ] + update_type_values = [ ('deletion', dict(update_type='deletion')), - ('update', dict(update_type='update')), - ], key_format_values) + ('update', dict(update_type='update')) + ] + scenarios = make_scenarios(key_format_values, update_type_values) + nrows = 10000 def create_key(self, i): if self.key_format == 'S': @@ -68,19 +67,19 @@ class test_hs11(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Apply a series of updates from timestamps 1-4. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) for ts in range(1, 5): - for i in range(1, 10000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) # Reconcile and flush versions 1-3 to the history store. self.session.checkpoint() # Apply an update without timestamp. - for i in range(1, 10000): + for i in range(1, self.nrows): if i % 2 == 0: if self.update_type == 'deletion': cursor.set_key(self.create_key(i)) @@ -92,20 +91,15 @@ class test_hs11(wttest.WiredTigerTestCase): self.session.checkpoint() # Now apply an update at timestamp 10. - for i in range(1, 10000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) - - # FIXME-WT-7120: Remove for column store until rollback to stable is implemented for column - # store. - if self.key_format == 'r': - return + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Ensure that we blew away history store content. for ts in range(1, 5): - self.session.begin_transaction('read_timestamp=' + timestamp_str(ts)) - for i in range(1, 10000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts)) + for i in range(1, self.nrows): if i % 2 == 0: if self.update_type == 'deletion': cursor.set_key(self.create_key(i)) @@ -129,38 +123,38 @@ class test_hs11(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Apply a series of updates from timestamps 1-4. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) for ts in range(1, 5): - for i in range(1, 10000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) # Reconcile and flush versions 1-3 to the history store. self.session.checkpoint() # Remove the key with timestamp 10. - for i in range(1, 10000): + for i in range(1, self.nrows): if i % 2 == 0: self.session.begin_transaction() cursor.set_key(self.create_key(i)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Reconcile and remove the obsolete entries. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) self.session.checkpoint() # Now apply an update at timestamp 20. - for i in range(1, 10000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Ensure that we didn't select old history store content even if it is not blew away. - self.session.begin_transaction('read_timestamp=' + timestamp_str(10)) - for i in range(1, 10000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(10)) + for i in range(1, self.nrows): if i % 2 == 0: cursor.set_key(self.create_key(i)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) diff --git a/src/third_party/wiredtiger/test/suite/test_hs12.py b/src/third_party/wiredtiger/test/suite/test_hs12.py index d3be5f40874..e92e338737c 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs12.py +++ b/src/third_party/wiredtiger/test/suite/test_hs12.py @@ -29,19 +29,14 @@ import wiredtiger, wttest, time from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs12.py # Verify we can correctly append modifies to the end of string values class test_hs12(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all),eviction=(threads_max=1)' + conn_config = 'cache_size=2MB,eviction=(threads_max=1)' session_config = 'isolation=snapshot' key_format_values = [ - # FIXME-WT-7120: The commented columnar tests needs to be enabled once columnar - # modify type update is fixed. - # ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), + ('column', dict(key_format='r')), + ('integer-row', dict(key_format='i')), ] scenarios = make_scenarios(key_format_values) @@ -94,12 +89,12 @@ class test_hs12(wttest.WiredTigerTestCase): cursor[1] = value2 self.session.commit_transaction() - # Insert a whole bunch of data into the table to force wiredtiger to evict data - # from the previous table. - self.session.begin_transaction() - for i in range(2, 10000): - cursor[i] = valuebig - self.session.commit_transaction() + # Configure debug behavior on a cursor to evict the positioned page on cursor reset + # and evict the page. + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + evict_cursor.set_key(1) + self.assertEquals(evict_cursor.search(), 0) + evict_cursor.reset() # Try to find the value we saw earlier cursor2.set_key(1) diff --git a/src/third_party/wiredtiger/test/suite/test_hs13.py b/src/third_party/wiredtiger/test/suite/test_hs13.py index ad0992a1656..3781ab200ad 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs13.py +++ b/src/third_party/wiredtiger/test/suite/test_hs13.py @@ -29,19 +29,14 @@ import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs13.py # Verify reverse modify traversal after eviction. class test_hs13(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all),eviction=(threads_max=1)' + conn_config = 'cache_size=2MB,eviction=(threads_max=1)' session_config = 'isolation=snapshot' key_format_values = [ - # FIXME-WT-5550: The commented columnar tests needs to be enabled once columnar modify type - # update is fixed. - # ('column', dict(key_format='r')), - ('integer', dict(key_format='i')) + ('column', dict(key_format='r')), + ('integer-row', dict(key_format='i')) ] scenarios = make_scenarios(key_format_values) @@ -94,12 +89,12 @@ class test_hs13(wttest.WiredTigerTestCase): cursor[1] = value2 self.session.commit_transaction() - # Insert a whole bunch of data into the table to force wiredtiger to evict data - # from the previous table. - self.session.begin_transaction() - for i in range(2, 10000): - cursor[i] = value3 - self.session.commit_transaction() + # Configure debug behavior on a cursor to evict the positioned page on cursor reset + # and evict the page. + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + evict_cursor.set_key(1) + self.assertEquals(evict_cursor.search(), 0) + evict_cursor.reset() # Try to find the value we saw earlier. cursor2.set_key(1) diff --git a/src/third_party/wiredtiger/test/suite/test_hs14.py b/src/third_party/wiredtiger/test/suite/test_hs14.py index 1351dd23e37..7c9d0241d82 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs14.py +++ b/src/third_party/wiredtiger/test/suite/test_hs14.py @@ -29,9 +29,6 @@ import time, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs14.py # Ensure that point in time reads with few visible history store records don't # damage performance. @@ -40,7 +37,7 @@ class test_hs14(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -52,35 +49,37 @@ class test_hs14(wttest.WiredTigerTestCase): def test_hs14(self): uri = 'table:test_hs14' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) + nrows = 10000 + value1 = 'a' * 500 value2 = 'b' * 500 value3 = 'c' * 500 value4 = 'd' * 500 value5 = 'e' * 500 - for i in range(1, 10000): + for i in range(1, nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() cursor[self.create_key(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.session.begin_transaction() cursor[self.create_key(i)] = value4 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # A checkpoint will ensure that older values are written to the history store. self.session.checkpoint() start = time.time() - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) - for i in range(1, 10000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) + for i in range(1, nrows): self.assertEqual(cursor[self.create_key(i)], value3) self.session.rollback_transaction() end = time.time() @@ -88,21 +87,21 @@ class test_hs14(wttest.WiredTigerTestCase): # The time spent when all history store keys are visible to us. visible_hs_latency = (end - start) - for i in range(1, 10000): + for i in range(1, nrows): self.session.begin_transaction() cursor.set_key(self.create_key(i)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) self.session.begin_transaction() cursor[self.create_key(i)] = value5 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # A checkpoint will ensure that older values are written to the history store. self.session.checkpoint() start = time.time() - self.session.begin_transaction('read_timestamp=' + timestamp_str(9)) - for i in range(1, 10000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(9)) + for i in range(1, nrows): cursor.set_key(self.create_key(i)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs15.py b/src/third_party/wiredtiger/test/suite/test_hs15.py index 7d27d166a70..9dded0b2d7d 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs15.py +++ b/src/third_party/wiredtiger/test/suite/test_hs15.py @@ -34,9 +34,6 @@ import time, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs15.py # Ensure eviction doesn't clear the history store again after checkpoint has done so because of the same update without timestamp. class test_hs15(wttest.WiredTigerTestCase): @@ -44,7 +41,7 @@ class test_hs15(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -71,46 +68,46 @@ class test_hs15(wttest.WiredTigerTestCase): for i in range(2, 1000): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Do a modify and an update with timestamps self.session.begin_transaction() cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('B', 100, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) self.session.begin_transaction() cursor[self.create_key(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Make the modify with timestamp and the update without timestamp obsolete - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) # Do a checkpoint self.session.checkpoint() self.session.begin_transaction() cursor[self.create_key(1)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Insert a bunch of other contents to trigger eviction for i in range(2, 1000): self.session.begin_transaction() cursor[self.create_key(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) expected = list(value1) expected[100] = 'B' expected = str().join(expected) - self.session.begin_transaction('read_timestamp=' + timestamp_str(1)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1)) self.assertEqual(cursor[self.create_key(1)], expected) self.session.rollback_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) self.assertEqual(cursor[self.create_key(1)], value2) self.session.rollback_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) self.assertEqual(cursor[self.create_key(1)], value3) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs16.py b/src/third_party/wiredtiger/test/suite/test_hs16.py index 11ca79456d0..1a5ed51128c 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs16.py +++ b/src/third_party/wiredtiger/test/suite/test_hs16.py @@ -29,9 +29,6 @@ import time, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs16.py # Ensure that we don't panic when inserting an update without timestamp to the history store. class test_hs16(wttest.WiredTigerTestCase): @@ -39,7 +36,7 @@ class test_hs16(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = ( ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ) scenarios = make_scenarios(key_format_values) @@ -62,7 +59,7 @@ class test_hs16(wttest.WiredTigerTestCase): # Update an update at timestamp 1 self.session.begin_transaction() cursor[self.create_key(1)] = 'b' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) # Open anther session to make the next update without timestamp non-globally visible session2 = self.setUpSessionOpen(self.conn) @@ -78,7 +75,7 @@ class test_hs16(wttest.WiredTigerTestCase): # Update an update at timestamp 2 self.session.begin_transaction() cursor[self.create_key(1)] = 'd' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Do a checkpoint, it should not panic self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_hs18.py b/src/third_party/wiredtiger/test/suite/test_hs18.py index 889d2dceccc..bcef53e4d17 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs18.py +++ b/src/third_party/wiredtiger/test/suite/test_hs18.py @@ -29,9 +29,6 @@ import time, wiredtiger, wttest, unittest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs18.py # Test various older reader scenarios class test_hs18(wttest.WiredTigerTestCase): @@ -39,7 +36,7 @@ class test_hs18(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -76,7 +73,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = value0 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a long running transaction which could see update 0. session2.begin_transaction() @@ -85,12 +82,12 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Insert a bunch of contents to fill the cache for i in range(2000, 10000): @@ -106,7 +103,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 15 self.session.begin_transaction() cursor[self.create_key(1)] = value5 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Check our value is still correct. self.check_value(cursor2, value0) @@ -122,11 +119,6 @@ class test_hs18(wttest.WiredTigerTestCase): # Test that we don't get the wrong value if we read with a timestamp originally. def test_read_timestamp_weirdness(self): - # FIXME-WT-7136: Mixed mode transactions not supported for column store scenario. - # Re-enable once complete. - if self.key_format == 'r': - return - uri = 'table:test_hs18' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) @@ -144,20 +136,20 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a long running transaction which could see update 0. - session2.begin_transaction('read_timestamp=' + timestamp_str(5)) + session2.begin_transaction('read_timestamp=' + self.timestamp_str(5)) # Check our value is still correct. self.check_value(cursor2, value1) # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a long running transaction which could see update 0. - session3.begin_transaction('read_timestamp=' + timestamp_str(5)) + session3.begin_transaction('read_timestamp=' + self.timestamp_str(5)) # Check our value is still correct. self.check_value(cursor3, value1) @@ -175,7 +167,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 15 self.session.begin_transaction() cursor[self.create_key(1)] = value5 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Check our value is still correct. self.check_value(cursor2, value1) @@ -219,12 +211,12 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Insert a bunch of other contents to trigger eviction for i in range(2, 10000): @@ -251,11 +243,6 @@ class test_hs18(wttest.WiredTigerTestCase): # Test older readers for each of the updates moved to the history store. def test_multiple_older_readers(self): - # FIXME-WT-7136: Mixed mode transactions not supported for column store scenario. - # Re-enable once complete. - if self.key_format == 'r': - return - uri = 'table:test_multiple_older_readers' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) @@ -274,7 +261,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = values[0] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a transaction that will see update 0. self.start_txn(sessions, cursors, values, 0) @@ -282,7 +269,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = values[1] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Start a transaction that will see update 1. self.start_txn(sessions, cursors, values, 1) @@ -290,7 +277,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = values[2] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a transaction that will see update 2. self.start_txn(sessions, cursors, values, 2) @@ -312,7 +299,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 15 self.session.begin_transaction() cursor[self.create_key(1)] = values[4] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Insert a bunch of other contents to trigger eviction for i in range(10001, 20000): @@ -328,11 +315,6 @@ class test_hs18(wttest.WiredTigerTestCase): cursors[i].reset() def test_multiple_older_readers_with_multiple_mixed_mode(self): - # FIXME-WT-7136: Mixed mode transactions not supported for column store scenario. - # Re-enable once complete. - if self.key_format == 'r': - return - uri = 'table:test_multiple_older_readers' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) @@ -351,7 +333,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = values[0] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a transaction that will see update 0. self.start_txn(sessions, cursors, values, 0) @@ -359,7 +341,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = values[1] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Start a transaction that will see update 1. self.start_txn(sessions, cursors, values, 1) @@ -367,7 +349,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = values[2] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a transaction that will see update 2. self.start_txn(sessions, cursors, values, 2) @@ -389,7 +371,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = values[4] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Start a transaction that will see update 4. self.start_txn(sessions, cursors, values, 4) @@ -397,7 +379,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = values[5] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a transaction that will see update 5. self.start_txn(sessions, cursors, values, 5) @@ -405,7 +387,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 15 self.session.begin_transaction() cursor[self.create_key(1)] = values[6] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Start a transaction that will see update 6. self.start_txn(sessions, cursors, values, 6) @@ -434,7 +416,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = values[8] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Insert a bunch of other contents to trigger eviction for i in range(10001, 20000): @@ -450,17 +432,14 @@ class test_hs18(wttest.WiredTigerTestCase): cursors[i].reset() def test_modifies(self): - # FIXME-WT-7136: Modify not supported for column store scenario. - # Re-enable once complete. - if self.key_format == 'r': - return - uri = 'table:test_modifies' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) session_ts_reader = self.setUpSessionOpen(self.conn) cursor_ts_reader = session_ts_reader.open_cursor(uri) + self.skipTest('Skip this part of test_hs18 until WT-7931 is resolved') + # The ID of the session corresponds the value it should see. sessions = [] cursors = [] @@ -480,7 +459,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = values[0] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a long running transaction which could see update 0. self.start_txn(sessions, cursors, values, 0) @@ -489,7 +468,7 @@ class test_hs18(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(self.create_key(1)) cursor.modify([wiredtiger.Modify('a', 0, 0)]) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) session_ts_reader.begin_transaction('read_timestamp=3') self.check_value(cursor_ts_reader, values[0]) @@ -501,7 +480,7 @@ class test_hs18(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(self.create_key(1)) cursor.modify([wiredtiger.Modify('b', 0, 0)]) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a long running transaction which could see modify 1. self.start_txn(sessions, cursors, values, 2) @@ -524,7 +503,7 @@ class test_hs18(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(self.create_key(1)) cursor.modify([wiredtiger.Modify('e', 0, 0)]) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Start a long running transaction which could see modify 2. sessions[4].begin_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs19.py b/src/third_party/wiredtiger/test/suite/test_hs19.py index e0549077275..2e3452ae0ca 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs19.py +++ b/src/third_party/wiredtiger/test/suite/test_hs19.py @@ -29,9 +29,6 @@ import time, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs19.py # Ensure eviction doesn't clear the history store again after checkpoint has done so because of the same update without timestamp. class test_hs19(wttest.WiredTigerTestCase): @@ -39,7 +36,7 @@ class test_hs19(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -57,7 +54,7 @@ class test_hs19(wttest.WiredTigerTestCase): cursor2 = session2.open_cursor(junk_uri) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) value1 = 'a' * 500 value2 = 'b' * 500 @@ -73,13 +70,13 @@ class test_hs19(wttest.WiredTigerTestCase): cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('B', 100, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) self.session.begin_transaction() cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('C', 101, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Start a transaction to pin back the reconciliation last running value. session2.begin_transaction() @@ -93,14 +90,14 @@ class test_hs19(wttest.WiredTigerTestCase): cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('AAAAAAAAAA', 102, 0)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Insert a modify to get written as the on disk value by checkpoint. self.session.begin_transaction() cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('D', 102, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Checkpoint such that all modifies get written out to the history store and the latest # modify gets written to the on disk value. @@ -112,7 +109,7 @@ class test_hs19(wttest.WiredTigerTestCase): cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('E', 103, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # First deposition the first cursor, so the page can be evicted. cursor.reset() @@ -129,7 +126,7 @@ class test_hs19(wttest.WiredTigerTestCase): expected = str().join(expected) # Retrieve the value at timestamp 1. - self.session.begin_transaction('read_timestamp=' + timestamp_str(1)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1)) cursor.set_key(self.create_key(1)) cursor.search() @@ -143,7 +140,7 @@ class test_hs19(wttest.WiredTigerTestCase): expected = str().join(expected) # Retrieve the value at timestamp 1. - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) cursor.set_key(self.create_key(1)) cursor.search() @@ -159,7 +156,7 @@ class test_hs19(wttest.WiredTigerTestCase): expected = str().join(expected) # Retrieve the value at timestamp 1. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) cursor.set_key(self.create_key(1)) cursor.search() # Assert that it matches our expected value. diff --git a/src/third_party/wiredtiger/test/suite/test_hs20.py b/src/third_party/wiredtiger/test/suite/test_hs20.py index 53e2f141726..8a8a89ffe4f 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs20.py +++ b/src/third_party/wiredtiger/test/suite/test_hs20.py @@ -27,23 +27,36 @@ # OTHER DEALINGS IN THE SOFTWARE. import time, wiredtiger, wttest +from wtscenario import make_scenarios # test_hs20.py # Ensure we never reconstruct a reverse modify update in the history store based on the onpage overflow value -def timestamp_str(t): - return '%x' % t - class test_hs20(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,eviction=(threads_max=1)' session_config = 'isolation=snapshot' + # Return the k'th (0-based) key. + def make_column_key(k): + return k + 1 + def make_string_key(k): + return str(k) + + key_format_values = [ + ('column', dict(key_format='r', make_key=make_column_key)), + ('string-row', dict(key_format='S', make_key=make_string_key)), + ] + + scenarios = make_scenarios(key_format_values) + def test_hs20(self): uri = 'table:test_hs20' + key_format = 'key_format=' + self.key_format + # Set a very small maximum leaf value to trigger writing overflow values - self.session.create(uri, 'key_format=S,value_format=S,leaf_value_max=10B') + self.session.create(uri, '{},value_format=S,leaf_value_max=10B'.format(key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) value1 = 'a' * 500 value2 = 'b' * 50 @@ -51,41 +64,41 @@ class test_hs20(wttest.WiredTigerTestCase): # Insert a value that is larger than the maximum leaf value. for i in range(0, 10): self.session.begin_transaction() - cursor[str(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + cursor[self.make_key(i)] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Do 2 modifies. for i in range(0, 10): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.make_key(i)) mods = [wiredtiger.Modify('B', 500, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(0, 10): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.make_key(i)) mods = [wiredtiger.Modify('C', 501, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Insert more data to trigger eviction. for i in range(10, 100000): self.session.begin_transaction() - cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + cursor[self.make_key(i)] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Update the overflow values. for i in range(0, 10): self.session.begin_transaction() - cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + cursor[self.make_key(i)] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Do a checkpoint to move the overflow values to the history store but keep the current in memory disk image. self.session.checkpoint() # Search the first modifies. for i in range(0, 10): - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) - self.assertEqual(cursor[str(i)], value1 + "B") + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) + self.assertEqual(cursor[self.make_key(i)], value1 + "B") self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs21.py b/src/third_party/wiredtiger/test/suite/test_hs21.py index 920dc41951b..7e8755430ca 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs21.py +++ b/src/third_party/wiredtiger/test/suite/test_hs21.py @@ -30,9 +30,7 @@ import time, re import wiredtiger, wttest from wtdataset import SimpleDataSet from wiredtiger import stat - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs21.py # Test we don't lose any data when idle files with an active history are closed/sweeped. @@ -49,6 +47,13 @@ class test_hs21(wttest.WiredTigerTestCase): numfiles = 10 nrows = 1000 + key_format_values = [ + ('column', dict(key_format='r', key1=1, key2=2)), + ('string-row', dict(key_format='S', key1=str(0), key2=str(1))), + ] + + scenarios = make_scenarios(key_format_values) + def large_updates(self, uri, value, ds, nrows, commit_ts): # Update a large number of records, we'll hang if the history store table isn't working. session = self.session @@ -56,13 +61,13 @@ class test_hs21(wttest.WiredTigerTestCase): session.begin_transaction() for i in range(1, nrows + 1): cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, session, check_value, uri, nrows, read_ts=-1): # Validate we read an expected value (optionally at a given read timestamp). if read_ts != -1: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -70,6 +75,12 @@ class test_hs21(wttest.WiredTigerTestCase): count += 1 if read_ts != -1: session.rollback_transaction() + if count != nrows: + self.prout("Oops") + self.prout("value: " + str(check_value)) + self.prout("count: " + str(count)) + self.prout("nrows: " + str(nrows)) + self.prout("read_ts: " + str(read_ts)) self.assertEqual(count, nrows) cursor.close() @@ -101,7 +112,7 @@ class test_hs21(wttest.WiredTigerTestCase): file_uri = 'file:%s.%d.wt' % (self.file_name, f) # Create a small table. ds = SimpleDataSet( - self, table_uri, 0, key_format='S', value_format='S', config='log=(enabled=false)') + self, table_uri, 0, key_format=self.key_format, value_format='S', config='log=(enabled=false)') ds.populate() # Checkpoint to ensure we write the files metadata checkpoint value. self.session.checkpoint() @@ -110,8 +121,8 @@ class test_hs21(wttest.WiredTigerTestCase): active_files.append((base_write_gen, ds)) # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) # Perform a series of updates over our files at timestamp 2. This being data we can later assert # to ensure the history store is working as intended. @@ -122,7 +133,7 @@ class test_hs21(wttest.WiredTigerTestCase): # We want to create a long running read transaction in a seperate session which we will persist over the closing and # re-opening of handles. We want to ensure the correct data gets read throughout this time period. session_read = self.conn.open_session() - session_read.begin_transaction('read_timestamp=' + timestamp_str(2)) + session_read.begin_transaction('read_timestamp=' + self.timestamp_str(2)) # Check our inital set of updates are seen at the read timestamp. for (_, ds) in active_files: # Check that all updates at timestamp 2 are seen. diff --git a/src/third_party/wiredtiger/test/suite/test_hs22.py b/src/third_party/wiredtiger/test/suite/test_hs22.py index cf30767b8bc..97a65ace10a 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs22.py +++ b/src/third_party/wiredtiger/test/suite/test_hs22.py @@ -27,9 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs22.py # Test the case that out of order timestamp @@ -38,43 +36,53 @@ class test_hs22(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', key1=1, key2=2)), + ('string-row', dict(key_format='S', key1=str(0), key2=str(1))), + ] + + scenarios = make_scenarios(key_format_values) + def test_onpage_out_of_order_timestamp_update(self): uri = 'table:test_hs22' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) + + key1 = self.key1 + key2 = self.key2 value1 = 'a' value2 = 'b' # Insert a key. self.session.begin_transaction() - cursor[str(0)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + cursor[key1] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Remove the key. self.session.begin_transaction() - cursor.set_key(str(0)) + cursor.set_key(key1) self.assertEqual(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Do an out of order timestamp # update and write it to the data # store later. self.session.begin_transaction() - cursor[str(0)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + cursor[key1] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Insert another key. self.session.begin_transaction() - cursor[str(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + cursor[key2] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Update the key. self.session.begin_transaction() - cursor[str(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + cursor[key2] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) # Do a checkpoint to trigger # history store reconciliation. @@ -83,58 +91,61 @@ class test_hs22(wttest.WiredTigerTestCase): evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") # Search the key to evict it. - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(evict_cursor[str(0)], value2) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(evict_cursor[key1], value2) self.assertEqual(evict_cursor.reset(), 0) self.session.rollback_transaction() # Search the key again to verify the data is still as expected. - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(cursor[str(0)], value2) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(cursor[key1], value2) self.session.rollback_transaction() def test_out_of_order_timestamp_update_newer_than_tombstone(self): uri = 'table:test_hs22' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) + + key1 = self.key1 + key2 = self.key2 value1 = 'a' value2 = 'b' # Insert a key. self.session.begin_transaction() - cursor[str(0)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + cursor[key1] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Remove a key. self.session.begin_transaction() - cursor.set_key(str(0)) + cursor.set_key(key1) self.assertEqual(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Do an out of order timestamp # update and write it to the # history store later. self.session.begin_transaction() - cursor[str(0)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + cursor[key1] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Add another update. self.session.begin_transaction() - cursor[str(0)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + cursor[key1] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Insert another key. self.session.begin_transaction() - cursor[str(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + cursor[key2] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Update the key. self.session.begin_transaction() - cursor[str(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + cursor[key2] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) # Do a checkpoint to trigger # history store reconciliation. @@ -143,12 +154,12 @@ class test_hs22(wttest.WiredTigerTestCase): evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") # Search the key to evict it. - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(evict_cursor[str(0)], value2) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(evict_cursor[key1], value2) self.assertEqual(evict_cursor.reset(), 0) self.session.rollback_transaction() # Search the key again to verify the data is still as expected. - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(cursor[str(0)], value2) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(cursor[key1], value2) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs23.py b/src/third_party/wiredtiger/test/suite/test_hs23.py index cfc7ad70623..d6bc1ec3d70 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs23.py +++ b/src/third_party/wiredtiger/test/suite/test_hs23.py @@ -27,9 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs23.py # Test the case that we have update, out of order timestamp @@ -38,12 +36,21 @@ class test_hs23(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', key=1)), + ('string-row', dict(key_format='S', key=str(0))), + ] + + scenarios = make_scenarios(key_format_values) + def test(self): uri = 'table:test_hs23' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) + + key = self.key value1 = 'a' value2 = 'b' @@ -53,34 +60,34 @@ class test_hs23(wttest.WiredTigerTestCase): # Insert a key. self.session.begin_transaction() - cursor[str(0)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + cursor[key] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Update at 10, update at 20, update at 15 (out of order), and # update at 20 in the same transaction self.session.begin_transaction() - cursor.set_key(str(0)) + cursor.set_key(key) cursor.set_value(value2) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(10)) + 'commit_timestamp=' + self.timestamp_str(10)) self.assertEquals(cursor.update(), 0) - cursor.set_key(str(0)) + cursor.set_key(key) cursor.set_value(value3) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(20)) + 'commit_timestamp=' + self.timestamp_str(20)) self.assertEquals(cursor.update(), 0) - cursor.set_key(str(0)) + cursor.set_key(key) cursor.set_value(value4) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(15)) + 'commit_timestamp=' + self.timestamp_str(15)) self.assertEquals(cursor.update(), 0) - cursor.set_key(str(0)) + cursor.set_key(key) cursor.set_value(value5) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(20)) + 'commit_timestamp=' + self.timestamp_str(20)) self.assertEquals(cursor.update(), 0) self.session.commit_transaction() @@ -92,16 +99,16 @@ class test_hs23(wttest.WiredTigerTestCase): # Search the key to evict it. self.session.begin_transaction() - self.assertEqual(evict_cursor[str(0)], value5) + self.assertEqual(evict_cursor[key], value5) self.assertEqual(evict_cursor.reset(), 0) self.session.rollback_transaction() # Search the latest update - self.session.begin_transaction("read_timestamp=" + timestamp_str(20)) - self.assertEqual(cursor[str(0)], value5) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(20)) + self.assertEqual(cursor[key], value5) self.session.rollback_transaction() - # Serarch the out of order timestamp update - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(cursor[str(0)], value4) + # Search the out of order timestamp update + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(cursor[key], value4) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs24.py b/src/third_party/wiredtiger/test/suite/test_hs24.py index 4c4ee4a4b94..08d6f303bc5 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs24.py +++ b/src/third_party/wiredtiger/test/suite/test_hs24.py @@ -28,9 +28,7 @@ import wttest, threading, wiredtiger from helper import simulate_crash_restart - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs24.py # Test that out of order timestamp fix racing with checkpointing the history store doesn't create inconsistent checkpoint. @@ -38,24 +36,32 @@ class test_hs24(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,timing_stress_for_test=(history_store_checkpoint_delay)' session_config = 'isolation=snapshot' uri = 'table:test_hs24' + numrows = 2000 + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) value1 = 'a' * 500 value2 = 'b' * 500 value3 = 'c' * 500 value4 = 'd' * 500 def test_zero_ts(self): - self.session.create(self.uri, 'key_format=i,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.session.create(self.uri, 'key_format={},value_format=S'. format(self.key_format)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): self.session.begin_transaction() cursor[i] = self.value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor[i] = self.value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) cursor.close() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) thread = threading.Thread(target=self.zero_ts_deletes) thread.start() self.session.checkpoint() @@ -64,13 +70,13 @@ class test_hs24(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri) session2 = self.conn.open_session(None) cursor2 = session2.open_cursor(self.uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) - session2.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) + session2.begin_transaction('read_timestamp=' + self.timestamp_str(4)) # Check the data store and the history store content is consistent. # If we have a value in the data store, we should see the older # version in the history store as well. newer_data_visible = False - for i in range(0, 2000): + for i in range(1, self.numrows + 1): cursor.set_key(i) cursor2.set_key(i) ret = cursor.search() @@ -88,7 +94,7 @@ class test_hs24(wttest.WiredTigerTestCase): def zero_ts_deletes(self): session = self.setUpSessionOpen(self.conn) cursor = session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): session.begin_transaction() cursor.set_key(i) cursor.remove() @@ -97,29 +103,29 @@ class test_hs24(wttest.WiredTigerTestCase): session.close() def test_zero_commit(self): - self.session.create(self.uri, 'key_format=i,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): self.session.begin_transaction() cursor[i] = self.value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor[i] = self.value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) cursor.close() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(4)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(4)) thread = threading.Thread(target=self.zero_ts_commits) thread.start() self.session.checkpoint() thread.join() simulate_crash_restart(self, '.', "RESTART") cursor = self.session.open_cursor(self.uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) # Check we can only see the version committed by the zero timestamp # commit thread before the checkpoint starts or value1. newer_data_visible = False - for i in range(0, 2000): + for i in range(1, self.numrows + 1): value = cursor[i] if not newer_data_visible: newer_data_visible = value != self.value3 @@ -132,7 +138,7 @@ class test_hs24(wttest.WiredTigerTestCase): def zero_ts_commits(self): session = self.setUpSessionOpen(self.conn) cursor = session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): session.begin_transaction() cursor[i] = self.value3 session.commit_transaction() @@ -140,21 +146,21 @@ class test_hs24(wttest.WiredTigerTestCase): session.close() def test_out_of_order_ts(self): - self.session.create(self.uri, 'key_format=i,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): self.session.begin_transaction() cursor[i] = self.value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor[i] = self.value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(4)) - for i in range(0, 2000): + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(4)) + for i in range(1, self.numrows + 1): self.session.begin_transaction() cursor[i] = self.value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(6)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(6)) cursor.close() thread = threading.Thread(target=self.out_of_order_ts_commits) thread.start() @@ -162,12 +168,12 @@ class test_hs24(wttest.WiredTigerTestCase): thread.join() simulate_crash_restart(self, '.', "RESTART") cursor = self.session.open_cursor(self.uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) # Check we can only see the version at timestamp 4, it's either # committed by the out of order timestamp commit thread before the # checkpoint starts or value1. newer_data_visible = False - for i in range(0, 2000): + for i in range(1, self.numrows + 1): value = cursor[i] if not newer_data_visible: newer_data_visible = value != self.value4 @@ -180,9 +186,9 @@ class test_hs24(wttest.WiredTigerTestCase): def out_of_order_ts_commits(self): session = self.setUpSessionOpen(self.conn) cursor = session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): session.begin_transaction() cursor[i] = self.value4 - session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) cursor.close() session.close() diff --git a/src/third_party/wiredtiger/test/suite/test_hs25.py b/src/third_party/wiredtiger/test/suite/test_hs25.py index e353ce076fe..a1a18928864 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs25.py +++ b/src/third_party/wiredtiger/test/suite/test_hs25.py @@ -27,9 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs25.py # Ensure updates structure is correct when processing each key. @@ -38,31 +36,38 @@ class test_hs25(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' uri = 'table:test_hs25' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_insert_updates_hs(self): - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) - self.session.create(self.uri, 'key_format=i,value_format=S') + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) s = self.conn.open_session() # Update the first key. cursor1 = self.session.open_cursor(self.uri) self.session.begin_transaction() cursor1[1] = 'a' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Update the second key. self.session.begin_transaction() cursor1[2] = 'a' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() cursor1[2] = 'b' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Prepared update on the first key. self.session.begin_transaction() cursor1[1] = 'b' cursor1[1] = 'c' - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(4)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(4)) # Run eviction cursor. s.begin_transaction('ignore_prepare=true') diff --git a/src/third_party/wiredtiger/test/suite/test_hs26.py b/src/third_party/wiredtiger/test/suite/test_hs26.py new file mode 100644 index 00000000000..9aab2322c43 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs26.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_hs26.py +# Test that changes overlapping variable-length column store RLE groups don't lose or corrupt data. +# (The concern doesn't exist for row stores, so while this test could be run for row stores there's +# little benefit to doing so; thus, no row-store scenarios are generated.) +# +# This works by writing batches of duplicate values, then overwriting them with new batches, +# all with relatively prime counts so the ranges overlap. It also tests the cases where not +# all the old or new value batches exist. +# +# It checks that the immediately written values can be read back, but the interesting part is +# whether they read back correctly after forcing eviction, which will RLE-encode the duplicates +# and read them back. There are many opportunities for the interaction betwee RLE groups and +# history store accesses to go off the rails. +class test_hs26(wttest.WiredTigerTestCase): + conn_config = '' + session_config = 'isolation=snapshot' + + # We control the duplication of values by appending a number computed from the key. + # Because the keys are 1..N (not 0..N-1), to get aligned RLE groups the suffix is + # computed as (key - 1) // value_modulus. This way, if value_modulus is e.g. 7, we + # get the first 7 keys (1-7) using the same suffix. + # + # It would not be _wrong_ to use key // value modulus, but it makes it much more + # difficult to reason about which RLE groups are overlapping where if the first RLE + # group is shorter than the others by 1. So we don't do that. + # + # The cases where the RLE group is mismatched by exactly 1 are particularly likely to + # have issues, so we want to be sure to exercise those cases. Any pair of relatively + # prime integers will generate these cases as long as there are enough keys (e.g. 7 + # and 13 generate one at 14 and the other at 78) but we also want to encounter these + # cases relative to the number of keys written, both the shorter and longer numbers. + # + # Consequently I've picked 103 for one key count (between multiples of 17 and 13) and + # 209 for the other (between multiples of 13 and 7). The ratio of the two key counts + # doesn't signify much but it's close to 2x on general principles. + # + # Other cases of overlapping the key count are still interesting so we still generate + # the full product of the scenarios. + + nrows_values = [ + ('more', dict(nrows_1=103, nrows_2=211)), + ('same', dict(nrows_1=211, nrows_2=211)), + ('less', dict(nrows_1=211, nrows_2=103)), + ] + value_modulus_1_values = [ + ('7', dict(value_modulus_1=7)), + ('13', dict(value_modulus_1=13)), + ('17', dict(value_modulus_1=17)), + ] + value_modulus_2_values = [ + ('7', dict(value_modulus_2=7)), + ('13', dict(value_modulus_2=13)), + ('17', dict(value_modulus_2=17)), + ] + + scenarios = make_scenarios(nrows_values, value_modulus_1_values, value_modulus_2_values) + + value_1 = 'a' * 500 + value_2 = 'd' * 500 + + timestamp_1 = 2 + timestamp_2 = 100 + + # Generate the value for a key. + def make_value(self, key, base_value, value_modulus): + return base_value + str((key - 1) // value_modulus) + + # Write nrows records, using value as the base value string. + def make_updates(self, uri, ds, value, value_modulus, nrows, commit_ts): + session = self.session + cursor = session.open_cursor(uri) + session.begin_transaction() + for i in range(1, nrows + 1): + cursor[ds.key(i)] = self.make_value(i, value, value_modulus) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + cursor.close() + + # Figure the expected value for a key, based on the read time. + # - If the read time is timestamp_1, we should see value_1. + # - If the read time is timestamp_2, we should see value_2, except if nrows_2 < nrows_1, + # in which case we should see value_1 for keys past nrows_1. + def expected_value(self, key, readtime): + if readtime == self.timestamp_1: + return self.make_value(key, self.value_1, self.value_modulus_1) + elif readtime == self.timestamp_2: + if self.nrows_2 < self.nrows_1 and key > self.nrows_2: + return self.make_value(key, self.value_1, self.value_modulus_1) + else: + return self.make_value(key, self.value_2, self.value_modulus_2) + else: + self.prout("expected_value: Unexpected readtime {}".format(readtime)) + self.assertTrue(False) + return None + + # Return the number of keys we expect, based on the read time. + # - If the read time is timestamp_1, we should see nrows_1. + # - If the read time is timestamp_2, we should see max(nrows_1, nrows_2). + def expected_numvalues(self, readtime): + if readtime == self.timestamp_1: + return self.nrows_1 + elif readtime == self.timestamp_2: + return max(self.nrows_1, self.nrows_2) + else: + self.prout("expected_numvalues: Unexpected readtime {}".format(readtime)) + self.assertTrue(False) + return None + + # Check that we got the values we expected. In particular, also make sure that + # we get the expected number of values back. Expect the values that should be + # there at readtime; if explicit_read_ts is set, open a transaction at that + # timestamp. + def check(self, session, uri, readtime, explicit_read_ts=-1): + if explicit_read_ts != -1: + session.begin_transaction('read_timestamp=' + self.timestamp_str(explicit_read_ts)) + cursor = session.open_cursor(uri) + count = 0 + for k, v in cursor: + # Count is key - 1, so pass count + 1 as key. + self.assertEqual(v, self.expected_value(count + 1, readtime)) + count += 1 + if explicit_read_ts != -1: + session.rollback_transaction() + self.assertEqual(count, self.expected_numvalues(readtime)) + cursor.close() + + def test_hs(self): + + # Create a file that contains active history (content newer than the oldest timestamp). + table_uri = 'table:hs26' + ds = SimpleDataSet( + self, table_uri, 0, key_format='r', value_format='S', config='log=(enabled=false)') + ds.populate() + self.session.checkpoint() + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) + + # Write the first set of values at timestamp_1. + self.make_updates(ds.uri, ds, self.value_1, self.value_modulus_1, self.nrows_1, self.timestamp_1) + + # Create a long running read transaction in a separate session. + # (Is it necessary for it to be separate? Not sure.) + session_read = self.conn.open_session() + session_read.begin_transaction('read_timestamp=' + self.timestamp_str(self.timestamp_1)) + + # Check that the initial writes (at timestamp_1) are seen. + self.check(session_read, ds.uri, self.timestamp_1) + + # Write different values at a later timestamp. + self.make_updates(ds.uri, ds, self.value_2, self.value_modulus_2, self.nrows_2, self.timestamp_2) + + # Check that the new updates are only seen after the update timestamp. + self.check(self.session, ds.uri, self.timestamp_1, self.timestamp_1) + self.check(self.session, ds.uri, self.timestamp_2, self.timestamp_2) + + self.session.breakpoint() + + # Now forcibly evict, so that all the pages are RLE-encoded and then read back in. + # There doesn't seem to be any way to just forcibly evict an entire table, so what + # I'm going to do is assume that each page can hold at least 41 values, and evict + # every 41st key. If this evicts pages repeatedly it won't really hurt anything, + # just waste some time. + + evict_cursor = self.session.open_cursor(ds.uri, None, "debug=(release_evict)") + self.session.begin_transaction() + for k in range(1, max(self.nrows_1, self.nrows_2) + 1, 41): + # Search the key to evict it. + v = evict_cursor[ds.key(k)] + xv = self.expected_value(k, self.timestamp_2) + self.assertEqual(v, xv) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + + # Using the long running read transaction, check that the correct data can still be read. + # It should see all the updates at timestamp_1. + self.check(session_read, ds.uri, self.timestamp_1) + session_read.rollback_transaction() + + # Also check that the most recent transaction has the later data. + self.check(self.session, ds.uri, self.timestamp_2, self.timestamp_2) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hs27.py b/src/third_party/wiredtiger/test/suite/test_hs27.py new file mode 100644 index 00000000000..e685503b8a2 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs27.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_hs27.py +# Test that variable-length column store doesn't RLE-compact adjacent data with heterogeneous +# timestamps. +# +# (The concern doesn't exist for row stores, so while this test could be run for row stores there's +# little benefit to doing so; thus, no row-store scenarios are generated.) +# +# This works by writing the same value to adjacent keys at different times, evicting them, and +# making sure they read back correctly. (The eviction is necessary to go through the RLE code.) +class test_hs27(wttest.WiredTigerTestCase): + conn_config = '' + session_config = 'isolation=snapshot' + + nrows = 100 + value_1 = 'a' * 119 + value_2 = 'd' * 119 + + # Configure the number of different timestamps to write at. + ntimes_values = [ + ('two', dict(ntimes=2)), + ('three', dict(ntimes=3)), + ('ten', dict(ntimes=10)), + ] + + # Configure the number of keys to write at each timestamp. + nkeys_values = [ + ('one', dict(nkeys=1)), + ('two', dict(nkeys=2)), + ('three', dict(nkeys=3)), + ] + + # Configure whether the keys should be initialized first. + doinit_values = [ + ('init', dict(doinit=True)), + ('noinit', dict(doinit=False)), + ] + + # Configure whether the timestamp groups of keys are written forwards or backwards. + group_forward_values = [ + ('forward', dict(group_forward=True)), + ('backward', dict(group_forward=False)), + ] + + # Configure whether the keys within a timestamp group are written forwards or backwards. + keys_forward_values = [ + ('forward', dict(keys_forward=True)), + ('backward', dict(keys_forward=False)), + ] + + scenarios = make_scenarios(ntimes_values, nkeys_values, doinit_values, + group_forward_values, keys_forward_values) + + # Get the m'th writer timestamp. + def get_writetime(self, m): + return 40 + 2 * m + + # Get the m'th reader timestamp. + def get_readtime(self, m): + return 41 + 2 * m + + # Get the k'th key for the m'th timestamp. + def get_key(self, k, m): + if self.group_forward: + ts_offset = m * self.nkeys + else: + ts_offset = (self.ntimes - m - 1) * self.nkeys + if self.keys_forward: + key_offset = k + else: + key_offset = self.nkeys - k - 1 + return 71 + ts_offset + key_offset + + # Return the key number k and timestamp number m for a key (inverse of get_key). + def invert_key(self, key): + if key < 71 or key >= 71 + self.ntimes * self.nkeys: + return None + key -= 71 + if self.group_forward: + m = key // self.nkeys + else: + m = self.ntimes - (key // self.nkeys) - 1 + if self.keys_forward: + k = key % self.nkeys + else: + k = self.nkeys - (key % self.nkeys) - 1 + return (k, m) + + # Return the timestamp number k for a timestamp (inverse of get_read/writetime). + # Return -1 or self.ntimes for other timestamps as appropriate, for comparison. + def invert_timestamp(self, ts, beforeval, afterval): + if ts < 40: + return beforeval + if ts >= 40 + self.ntimes * 2: + return afterval + ts -= 40 + m = ts // 2 + return m + + # Figure if we should see value_2 for a given key and read time. + def expect_value_2(self, key, readtime): + # Get the key number and timestamp number for the key. + km = self.invert_key(key) + if km is None: + return False + (k, m) = km + + # Get the timestamp number associated with readtime. + m2 = self.invert_timestamp(readtime, -1, self.ntimes) + + # We should see value_2 if we are at or after the time it was written. + return m2 >= m + + # Check each value explicitly to make sure it's what we meant to see. + # Don't bother checking the background values. + def check1(self, session, uri, ds, readtime, make_own_txn): + if make_own_txn: + session.begin_transaction('read_timestamp=' + self.timestamp_str(readtime)) + cursor = session.open_cursor(uri) + for k in range(0, self.nkeys): + for m in range(0, self.ntimes): + key = self.get_key(k, m) + if self.expect_value_2(key, readtime): + self.assertEqual(cursor[ds.key(key)], self.value_2) + elif self.doinit: + self.assertEqual(cursor[ds.key(key)], self.value_1) + else: + self.assertRaisesException(KeyError, lambda: cursor[ds.key(key)]) + if make_own_txn: + session.rollback_transaction() + cursor.close() + + # Scan through the whole table and make sure it's what we expect. + def check2(self, session, uri, readtime, make_own_txn): + if make_own_txn: + session.begin_transaction('read_timestamp=' + self.timestamp_str(readtime)) + cursor = session.open_cursor(uri) + count = 0 + for (key, valueseen) in cursor: + val = self.value_2 if self.expect_value_2(key, readtime) else self.value_1 + self.assertEqual(valueseen, val) + count += 1 + if self.doinit: + # The table was initialized; should always see all rows. + self.assertEqual(count, self.nrows) + else: + # The table was not initialized; should see only what we wrote by this time. + # (If m is 0 we should see one batch of keys.) + m = self.invert_timestamp(readtime, -1, self.ntimes - 1) + self.assertEqual(count, self.nkeys * (m + 1)) + if make_own_txn: + session.rollback_transaction() + cursor.close() + + # Scan through the whole table backward too, since cursor_prev isn't adequately tested. + def check3(self, session, uri, readtime, make_own_txn): + if make_own_txn: + session.begin_transaction('read_timestamp=' + self.timestamp_str(readtime)) + cursor = session.open_cursor(uri) + count = 0 + while cursor.prev() == 0: + key = cursor.get_key() + valueseen = cursor[key] + val = self.value_2 if self.expect_value_2(key, readtime) else self.value_1 + self.assertEqual(valueseen, val) + count += 1 + if self.doinit: + # The table was initialized; should always see all rows. + self.assertEqual(count, self.nrows) + else: + # The table was not initialized; should see only what we wrote by this time. + # (If m is 0 we should see one batch of keys.) + m = self.invert_timestamp(readtime, -1, self.ntimes - 1) + self.assertEqual(count, self.nkeys * (m + 1)) + if make_own_txn: + session.rollback_transaction() + cursor.close() + + # Do all three checks. + def check(self, session, uri, ds, readtime, make_own_txn=True): + self.check1(session, uri, ds, readtime, make_own_txn) + self.check2(session, uri, readtime, make_own_txn) + self.check3(session, uri, readtime, make_own_txn) + + # Scan through the whole table at each relevant timestamp and make sure it's what we expect. + def checkall(self, session, uri, ds): + self.check(session, uri, ds, 2) + for m in range(0, self.ntimes): + self.check(session, uri, ds, self.get_readtime(m)) + self.check(session, uri, ds, 100) + + # Write self.nrows records at timestamp 1, using self.value_1 as the value. + def initialize(self, uri, ds): + session = self.session + cursor = session.open_cursor(uri) + session.begin_transaction() + for i in range(1, self.nrows + 1): + cursor[ds.key(i)] = self.value_1 + session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) + cursor.close() + + # Do the m'th update. + def update(self, uri, ds, m): + writetime = self.get_writetime(m) + + session = self.session + cursor = session.open_cursor(uri) + session.begin_transaction() + if self.keys_forward: + for k in range(0, self.nkeys): + key = self.get_key(k, m) + cursor[ds.key(key)] = self.value_2 + else: + for k in range(self.nkeys - 1, -1, -1): + key = self.get_key(k, m) + cursor[ds.key(key)] = self.value_2 + session.commit_transaction('commit_timestamp=' + self.timestamp_str(writetime)) + cursor.close() + + # Do all the updates. + def updateall(self, uri, ds): + if self.group_forward: + for m in range(0, self.ntimes): + self.update(uri, ds, m) + else: + for m in range(self.ntimes - 1, -1, -1): + self.update(uri, ds, m) + + def test_hs(self): + + # Create a file that contains active history (content newer than the oldest timestamp). + table_uri = 'table:hs27' + ds = SimpleDataSet( + self, table_uri, 0, key_format='r', value_format='S', config='log=(enabled=false)') + ds.populate() + self.session.checkpoint() + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) + + # Write the initial values, if requested. + if self.doinit: + self.initialize(ds.uri, ds) + + # Create a long running read transaction in a separate session. + # (Is it necessary for it to be separate? Not sure.) + session_read = self.conn.open_session() + session_read.begin_transaction('read_timestamp=' + self.timestamp_str(2)) + + # Check that the initial writes (at timestamp 1) are seen (at timestamp 2). + self.check(session_read, ds.uri, ds, 2, make_own_txn=False) + + # Write more values at assorted timestamps. + self.updateall(ds.uri, ds) + + # Check that the new updates are appropriately visible. + self.checkall(self.session, ds.uri, ds) + + self.session.breakpoint() + + # Now forcibly evict, so that all the pages are RLE-encoded and then read back in. + # There doesn't seem to be any way to just forcibly evict an entire table, so what + # I'm going to do is assume that what we care about is evicting the updates (the + # initial values are not so interesting) and they are on a maximum of two pages, + # so we can evict the first and last key. If this evicts the same page twice, it + # won't really hurt anything. (This also avoids having to worry about whether we + # wrote initial values or not.) + + evict_cursor = self.session.open_cursor(ds.uri, None, "debug=(release_evict)") + self.session.begin_transaction() + firstkey = self.get_key(0, 0) + lastkey = self.get_key(self.nkeys - 1, self.ntimes - 1) + for k in [firstkey, lastkey]: + # Search the key to evict it. + v = evict_cursor[ds.key(k)] + self.assertEqual(v, self.value_2) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + + # Check that the long-running read transaction still reads the correct data. + self.check(session_read, ds.uri, ds, 2, make_own_txn=False) + + # Check that our main session reads the correct data. + self.checkall(self.session, ds.uri, ds) + + # Drop the long running read transaction. + session_read.rollback_transaction() + + # Check that our main session can still read the latest data. + self.check(self.session, ds.uri, ds, 100) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_import01.py b/src/third_party/wiredtiger/test/suite/test_import01.py index 9376b805a23..daaae5f95e3 100644 --- a/src/third_party/wiredtiger/test/suite/test_import01.py +++ b/src/third_party/wiredtiger/test/suite/test_import01.py @@ -103,14 +103,10 @@ class test_import_base(wttest.WiredTigerTestCase): "Tmplog" not in file_name and "Preplog" not in file_name: shutil.copy(src_path, dest_dir) - # Convert a WiredTiger timestamp to a string. - def timestamp_str(self, t): - return '%x' % t - # test_import01 class test_import01(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' original_db_file = 'original_db_file' diff --git a/src/third_party/wiredtiger/test/suite/test_import02.py b/src/third_party/wiredtiger/test/suite/test_import02.py index 57f80b769d0..9dbb7b6f66a 100644 --- a/src/third_party/wiredtiger/test/suite/test_import02.py +++ b/src/third_party/wiredtiger/test/suite/test_import02.py @@ -34,7 +34,7 @@ import wiredtiger, wttest from test_import01 import test_import_base class test_import02(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' original_db_file = 'original_db_file' diff --git a/src/third_party/wiredtiger/test/suite/test_import03.py b/src/third_party/wiredtiger/test/suite/test_import03.py index a62486f1bb9..278daafa98e 100644 --- a/src/third_party/wiredtiger/test/suite/test_import03.py +++ b/src/third_party/wiredtiger/test/suite/test_import03.py @@ -34,7 +34,7 @@ from wtscenario import make_scenarios from test_import01 import test_import_base class test_import03(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' ntables = 10 diff --git a/src/third_party/wiredtiger/test/suite/test_import04.py b/src/third_party/wiredtiger/test/suite/test_import04.py index 230fad578e2..f64aefdee70 100644 --- a/src/third_party/wiredtiger/test/suite/test_import04.py +++ b/src/third_party/wiredtiger/test/suite/test_import04.py @@ -54,7 +54,7 @@ from wtscenario import make_scenarios from test_import01 import test_import_base class test_import04(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' ntables = 10 diff --git a/src/third_party/wiredtiger/test/suite/test_import05.py b/src/third_party/wiredtiger/test/suite/test_import05.py index b78a4b81116..0307c6586cb 100644 --- a/src/third_party/wiredtiger/test/suite/test_import05.py +++ b/src/third_party/wiredtiger/test/suite/test_import05.py @@ -35,7 +35,7 @@ from wtscenario import make_scenarios from test_import01 import test_import_base class test_import05(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' ntables = 10 diff --git a/src/third_party/wiredtiger/test/suite/test_import06.py b/src/third_party/wiredtiger/test/suite/test_import06.py index 083fc860a67..7d1360bacba 100644 --- a/src/third_party/wiredtiger/test/suite/test_import06.py +++ b/src/third_party/wiredtiger/test/suite/test_import06.py @@ -87,7 +87,7 @@ class test_import06(test_import_base): extlist.extension('encryptors', self.encryptor) def conn_config(self): - return 'cache_size=50MB,log=(enabled),statistics=(all),encryption=(name={})'.format( + return 'cache_size=50MB,log=(enabled),encryption=(name={})'.format( self.encryptor + self.encryptor_args) def test_import_repair(self): diff --git a/src/third_party/wiredtiger/test/suite/test_import08.py b/src/third_party/wiredtiger/test/suite/test_import08.py index bac4dc8f431..ed5578d02d2 100644 --- a/src/third_party/wiredtiger/test/suite/test_import08.py +++ b/src/third_party/wiredtiger/test/suite/test_import08.py @@ -34,7 +34,7 @@ from test_import01 import test_import_base from wtscenario import make_scenarios class test_import08(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' original_db_file = 'original_db_file' diff --git a/src/third_party/wiredtiger/test/suite/test_import09.py b/src/third_party/wiredtiger/test/suite/test_import09.py index a41aff32571..cc13688d7a1 100644 --- a/src/third_party/wiredtiger/test/suite/test_import09.py +++ b/src/third_party/wiredtiger/test/suite/test_import09.py @@ -100,7 +100,7 @@ class test_import09(test_import_base): extlist.extension('encryptors', self.encryptor) def conn_config(self): - return 'cache_size=50MB,log=(enabled),statistics=(all),encryption=(name={})'.format( + return 'cache_size=50MB,log=(enabled),encryption=(name={})'.format( self.encryptor + self.encryptor_args) def test_import_table_repair(self): diff --git a/src/third_party/wiredtiger/test/suite/test_prepare02.py b/src/third_party/wiredtiger/test/suite/test_prepare02.py index 60e61376c14..9f796d6a50e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare02.py @@ -33,9 +33,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_prepare02(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' @@ -91,9 +88,6 @@ class test_prepare02(wttest.WiredTigerTestCase, suite_subprocess): self.assertTimestampsEqual(self.session.query_timestamp('get=prepare'), '2a') self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:self.session.checkpoint(), msg) - # WT_SESSION.transaction_pinned_range permitted, not supported in the Python API. - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda:self.session.transaction_sync(), msg) self.session.breakpoint() # Commit the transaction. Test that no "not permitted in a prepared transaction" error has diff --git a/src/third_party/wiredtiger/test/suite/test_prepare05.py b/src/third_party/wiredtiger/test/suite/test_prepare05.py index c5971da32fa..c92a079bf6c 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare05.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare05.py @@ -32,33 +32,38 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_prepare05' uri = 'table:' + tablename session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_timestamp_api(self): - self.session.create(self.uri, 'key_format=i,value_format=i') + self.session.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) c = self.session.open_cursor(self.uri) # It is illegal to set a prepare timestamp older than oldest timestamp. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(1)), + 'prepare_timestamp=' + self.timestamp_str(1)), "/older than the oldest timestamp/") self.session.rollback_transaction() # Check setting the prepare timestamp same as oldest timestamp is valid. self.session.begin_transaction() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(2)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(3)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(3)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(2)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(3)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(3)) self.session.commit_transaction() # In a single transaction it is illegal to set a commit timestamp @@ -67,10 +72,10 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): # prepare itself is illegal. self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(3)) + 'commit_timestamp=' + self.timestamp_str(3)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(2)), + 'prepare_timestamp=' + self.timestamp_str(2)), "/should not have been set before/") self.session.rollback_transaction() @@ -79,11 +84,11 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): # Start a new reader to have an active read timestamp. if wiredtiger.diagnostic_build(): s_reader = self.conn.open_session() - s_reader.begin_transaction('read_timestamp=' + timestamp_str(4)) + s_reader.begin_transaction('read_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(4)), + 'prepare_timestamp=' + self.timestamp_str(4)), "/must be greater than the latest active read timestamp/") self.session.rollback_transaction() @@ -92,7 +97,7 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[1] = 1 self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(5)) + 'prepare_timestamp=' + self.timestamp_str(5)) # Resolve the reader transaction started earlier. s_reader.rollback_transaction() self.session.rollback_transaction() @@ -103,10 +108,10 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): # It is illegal to set a commit timestamp older than prepare timestamp of a transaction. self.session.begin_transaction() c[1] = 1 - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(5)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(5)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(4)), + 'commit_timestamp=' + self.timestamp_str(4)), "/less than the prepare timestamp/") ''' @@ -115,9 +120,9 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[1] = 1 self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(5)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(5)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(5)) + 'prepare_timestamp=' + self.timestamp_str(5)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(5)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(5)) self.session.commit_transaction() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_prepare06.py b/src/third_party/wiredtiger/test/suite/test_prepare06.py index 58361bb39a2..b4e65f183b0 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare06.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare06.py @@ -32,36 +32,41 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_prepare06' uri = 'table:' + tablename session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_timestamp_api(self): - self.session.create(self.uri, 'key_format=i,value_format=i') + self.session.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) c = self.session.open_cursor(self.uri) # It is illegal to set the prepare timestamp older than the oldest # timestamp. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(20)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(10)), + 'prepare_timestamp=' + self.timestamp_str(10)), "/older than the oldest timestamp/") self.session.rollback_transaction() # Check setting a prepared transaction timestamps earlier than the # oldest timestamp is valid with roundup_timestamps settings. self.session.begin_transaction('roundup_timestamps=(prepared=true)') - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(15)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(35)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(15)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(35)) self.session.commit_transaction() ''' @@ -71,11 +76,11 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): # oldest timestamp is invalid, if durable timestamp is less than the # stable timestamp. self.session.begin_transaction('roundup_timestamps=(prepared=true)') - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(15)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(25)), + 'durable_timestamp=' + self.timestamp_str(25)), "/is less than the stable timestamp/") self.session.rollback_transaction() ''' @@ -83,7 +88,7 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): # Check the cases with an active reader. # Start a new reader to have an active read timestamp. s_reader = self.conn.open_session() - s_reader.begin_transaction('read_timestamp=' + timestamp_str(40)) + s_reader.begin_transaction('read_timestamp=' + self.timestamp_str(40)) # It is illegal to set the prepare timestamp as earlier than an active # read timestamp even with roundup_timestamps settings. This is only @@ -92,7 +97,7 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction('roundup_timestamps=(prepared=true)') self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(10)), + 'prepare_timestamp=' + self.timestamp_str(10)), "/must be greater than the latest active read timestamp/") self.session.rollback_transaction() @@ -101,7 +106,7 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction('roundup_timestamps=(prepared=true)') self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(40)), + 'prepare_timestamp=' + self.timestamp_str(40)), "/must be greater than the latest active read timestamp/") self.session.rollback_transaction() @@ -112,10 +117,10 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): # timestamp of a transaction. self.session.begin_transaction() c[1] = 1 - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(45)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(45)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(30)), + 'commit_timestamp=' + self.timestamp_str(30)), "/less than the prepare timestamp/") ''' @@ -127,12 +132,12 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction('roundup_timestamps=(prepared=true)') c[1] = 1 self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(45)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(30)) - #self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(30)) + 'prepare_timestamp=' + self.timestamp_str(45)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) + #self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(30)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(30)), + 'durable_timestamp=' + self.timestamp_str(30)), "/is less than the commit timestamp/") self.session.rollback_transaction() ''' diff --git a/src/third_party/wiredtiger/test/suite/test_prepare07.py b/src/third_party/wiredtiger/test/suite/test_prepare07.py index c53f6aaa262..49d3544cf40 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare07.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare07.py @@ -30,9 +30,7 @@ import fnmatch, os, shutil, time from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare07.py # Test to ensure prepared updates older than oldest timestamp are not visible. @@ -43,10 +41,17 @@ class test_prepare07(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=50MB' + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) + def older_prepare_updates(self, uri, ds, nrows, value_a): # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) # Commit some updates. value_b = b"bbbbb" * 100 @@ -55,12 +60,12 @@ class test_prepare07(wttest.WiredTigerTestCase): cursor.set_key(ds.key(nrows + 1)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(110)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(110)) self.session.begin_transaction('isolation=snapshot') cursor.set_key(ds.key(nrows + 2)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(120)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(120)) # Prepare a transaction and keep it open. session_p = self.conn.open_session() @@ -69,36 +74,36 @@ class test_prepare07(wttest.WiredTigerTestCase): cursor_p.set_key(ds.key(nrows + 3)) cursor_p.set_value(value_b) self.assertEquals(cursor_p.update(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(130)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(130)) # Commit some more updates. self.session.begin_transaction('isolation=snapshot') cursor.set_key(ds.key(nrows + 4)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(140)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(140)) self.session.begin_transaction('isolation=snapshot') cursor.set_key(ds.key(nrows + 5)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(150)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(150)) # Move the oldest and the stable timestamp to the latest. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(155)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(155)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(155)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(155)) # Commit an update newer than the stable timestamp. self.session.begin_transaction('isolation=snapshot') cursor.set_key(ds.key(nrows + 6)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(160)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(160)) # Take a checkpoint here, so that prepared transaction will not be durable.. self.session.checkpoint() # Commit the prepared transaction. - session_p.commit_transaction('commit_timestamp=' + timestamp_str(140) + ',durable_timestamp=' + timestamp_str(160)) + session_p.commit_transaction('commit_timestamp=' + self.timestamp_str(140) + ',durable_timestamp=' + self.timestamp_str(160)) # Take a backup and check the values. self.backup_dir = os.path.join(self.home, "WT_BACKUP") @@ -143,7 +148,7 @@ class test_prepare07(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test" nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') + ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format='u') ds.populate() value_a = b"aaaaa" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_prepare08.py b/src/third_party/wiredtiger/test/suite/test_prepare08.py index 76a831b0a05..05ad8ab287e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare08.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare08.py @@ -30,9 +30,7 @@ import fnmatch, os, shutil, time from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare08.py # Test to ensure prepared tombstones are properly aborted/committed even when they are written @@ -41,6 +39,13 @@ class test_prepare08(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=10MB,eviction_dirty_trigger=80,eviction_updates_trigger=80' + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) + def updates(self, ds, uri, nrows, value, ts): cursor = self.session.open_cursor(uri) self.session.begin_transaction('isolation=snapshot') @@ -48,7 +53,7 @@ class test_prepare08(wttest.WiredTigerTestCase): cursor.set_key(ds.key(i)) cursor.set_value(value) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def removes(self, ds, uri, nrows, ts): @@ -57,7 +62,7 @@ class test_prepare08(wttest.WiredTigerTestCase): for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def check(self, ds, uri, nrows, value, ts, release_evict): @@ -65,7 +70,7 @@ class test_prepare08(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") else: cursor = self.session.open_cursor(uri) - self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + timestamp_str(ts)) + self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts)) for i in range(1, nrows): cursor.set_key(ds.key(i)) if value == None: @@ -82,11 +87,11 @@ class test_prepare08(wttest.WiredTigerTestCase): # Create a small table. uri_1 = "table:test_prepare08_1" - ds_1 = SimpleDataSet(self, uri_1, 0, key_format="S", value_format='u') + ds_1 = SimpleDataSet(self, uri_1, 0, key_format=self.key_format, value_format='u') ds_1.populate() uri_2 = "table:test_prepare08_2" - ds_2 = SimpleDataSet(self, uri_2, 0, key_format="S", value_format='u') + ds_2 = SimpleDataSet(self, uri_2, 0, key_format=self.key_format, value_format='u') ds_2.populate() value_a = b"aaaaa" * 100 @@ -96,8 +101,8 @@ class test_prepare08(wttest.WiredTigerTestCase): value_e = b"eeeee" * 100 # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Initially load huge data self.updates(ds_1, uri_1, nrows, value_a, 20) @@ -124,7 +129,7 @@ class test_prepare08(wttest.WiredTigerTestCase): for i in range(1, nrows): cursor_p.set_key(ds_1.key(i)) self.assertEquals(cursor_p.remove(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Adding more updates to other table should trigger eviction on uri_1 self.updates(ds_2, uri_2, nrows, value_c, 40) @@ -150,12 +155,12 @@ class test_prepare08(wttest.WiredTigerTestCase): # Create a small table. uri_1 = "table:test_prepare10_1" - ds_1 = SimpleDataSet(self, uri_1, 0, key_format="S", value_format='u') + ds_1 = SimpleDataSet(self, uri_1, 0, key_format=self.key_format, value_format='u') ds_1.populate() # Create another small table. uri_2 = "table:test_prepare10_2" - ds_2 = SimpleDataSet(self, uri_2, 0, key_format="S", value_format='u') + ds_2 = SimpleDataSet(self, uri_2, 0, key_format=self.key_format, value_format='u') ds_2.populate() value_a = b"aaaaa" * 100 @@ -165,8 +170,8 @@ class test_prepare08(wttest.WiredTigerTestCase): value_e = b"eeeee" * 100 # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Initially load huge data self.updates(ds_1, uri_1, nrows, value_a, 20) @@ -196,7 +201,7 @@ class test_prepare08(wttest.WiredTigerTestCase): self.assertEquals(cursor_p.update(), 0) cursor_p.set_key(ds_1.key(i)) self.assertEquals(cursor_p.remove(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Adding more updates to other table should trigger eviction on uri_1 self.updates(ds_2, uri_2, nrows, value_c, 40) @@ -208,7 +213,7 @@ class test_prepare08(wttest.WiredTigerTestCase): self.check(ds_1, uri_1, nrows, value_b, 50, True) # Commit the prepared session - session_p.commit_transaction('commit_timestamp=' + timestamp_str(50) + ',durable_timestamp=' + timestamp_str(60)) + session_p.commit_transaction('commit_timestamp=' + self.timestamp_str(50) + ',durable_timestamp=' + self.timestamp_str(60)) self.check(ds_1, uri_1, nrows, value_a, 20, False) self.check(ds_1, uri_1, nrows, value_b, 30, False) @@ -224,12 +229,12 @@ class test_prepare08(wttest.WiredTigerTestCase): # Create a small table. uri_1 = "table:test_prepare10_1" - ds_1 = SimpleDataSet(self, uri_1, 0, key_format="S", value_format='u') + ds_1 = SimpleDataSet(self, uri_1, 0, key_format=self.key_format, value_format='u') ds_1.populate() # Create another small table. uri_2 = "table:test_prepare10_2" - ds_2 = SimpleDataSet(self, uri_2, 0, key_format="S", value_format='u') + ds_2 = SimpleDataSet(self, uri_2, 0, key_format=self.key_format, value_format='u') ds_2.populate() value_a = b"aaaaa" * 100 @@ -239,8 +244,8 @@ class test_prepare08(wttest.WiredTigerTestCase): value_e = b"eeeee" * 100 # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Initially load huge data self.updates(ds_1, uri_1, nrows, value_a, 20) @@ -266,7 +271,7 @@ class test_prepare08(wttest.WiredTigerTestCase): self.assertEquals(cursor_p.update(), 0) cursor_p.set_key(ds_1.key(i)) self.assertEquals(cursor_p.remove(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Adding more updates to other table should trigger eviction on uri_1 self.updates(ds_2, uri_2, nrows, value_c, 40) @@ -278,7 +283,7 @@ class test_prepare08(wttest.WiredTigerTestCase): self.check(ds_1, uri_1, 0, None, 50, True) # Commit the prepared session - session_p.commit_transaction('commit_timestamp=' + timestamp_str(50) + ',durable_timestamp=' + timestamp_str(60)) + session_p.commit_transaction('commit_timestamp=' + self.timestamp_str(50) + ',durable_timestamp=' + self.timestamp_str(60)) self.check(ds_1, uri_1, nrows, value_a, 20, False) self.check(ds_1, uri_1, 0, None, 30, False) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare09.py b/src/third_party/wiredtiger/test/suite/test_prepare09.py index 3c0aa546d08..8aaacf1c3fb 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare09.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare09.py @@ -31,24 +31,30 @@ # [END_TAGS] import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare09.py # Validate scenarios involving inserting tombstones when rolling back prepares class test_prepare09(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all)' + conn_config = 'cache_size=2MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_prepared_update_is_aborted_correctly_with_on_disk_value(self): uri = "table:test_prepare09" - create_params = 'value_format=S,key_format=i' + create_params = 'value_format=S,key_format={}'.format(self.key_format) value1 = 'a' * 10000 value2 = 'b' * 10000 value3 = 'c' * 10000 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) self.session.create(uri, create_params) cursor = self.session.open_cursor(uri) @@ -58,7 +64,7 @@ class test_prepare09(wttest.WiredTigerTestCase): # Insert a new value. self.session.begin_transaction() cursor[1] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Get the previous update onto disk. for i in range(2, 10000): @@ -68,7 +74,7 @@ class test_prepare09(wttest.WiredTigerTestCase): # Prepare a full value and roll it back. self.session.begin_transaction() cursor[1] = value3 - self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(3)), 0) + self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(3)), 0) self.session.rollback_transaction() # Get the previous update onto disk. @@ -89,8 +95,8 @@ class test_prepare09(wttest.WiredTigerTestCase): value2 = 'b' * 10000 value3 = 'e' * 10000 value4 = 'd' * 10000 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) self.session.create(uri, create_params) cursor = self.session.open_cursor(uri) @@ -102,7 +108,7 @@ class test_prepare09(wttest.WiredTigerTestCase): cursor[1] = value1 cursor[2] = value2 cursor[3] = value3 - self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(3)), 0) + self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(3)), 0) # Get the prepare onto disk. for i in range(4, 10000): diff --git a/src/third_party/wiredtiger/test/suite/test_prepare10.py b/src/third_party/wiredtiger/test/suite/test_prepare10.py index a9ae73848bd..a4eec9bfcc4 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare10.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare10.py @@ -30,9 +30,7 @@ import fnmatch, os, shutil, time from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare10.py # Test to ensure prepared tombstones are properly aborted even when they are written @@ -42,6 +40,13 @@ class test_prepare10(wttest.WiredTigerTestCase): conn_config = 'cache_size=10MB,eviction_dirty_trigger=80,eviction_updates_trigger=80' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) + def updates(self, ds, uri, nrows, value, ts): cursor = self.session.open_cursor(uri) self.session.begin_transaction() @@ -49,7 +54,7 @@ class test_prepare10(wttest.WiredTigerTestCase): cursor.set_key(ds.key(i)) cursor.set_value(value) self.assertEquals(cursor.insert(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def removes(self, ds, uri, nrows, ts): @@ -58,12 +63,12 @@ class test_prepare10(wttest.WiredTigerTestCase): for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def check(self, ds, uri, nrows, value, ts): cursor = self.session.open_cursor(uri) - self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + timestamp_str(ts)) + self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts)) for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.search(), 0) @@ -73,7 +78,7 @@ class test_prepare10(wttest.WiredTigerTestCase): def check_not_found(self, ds, uri, nrows, ts): cursor = self.session.open_cursor(uri) - self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + timestamp_str(ts)) + self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts)) for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) @@ -84,7 +89,7 @@ class test_prepare10(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_prepare10" nrows = 1000 - ds = SimpleDataSet(self, uri, 0, key_format="S", value_format='u') + ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format='u') ds.populate() value_a = b"aaaaa" * 100 @@ -92,8 +97,8 @@ class test_prepare10(wttest.WiredTigerTestCase): value_c = b"ccccc" * 100 # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Initially load huge data self.updates(ds, uri, nrows, value_a, 20) @@ -141,7 +146,7 @@ class test_prepare10(wttest.WiredTigerTestCase): cursor_p.set_key(ds.key(i)) cursor_p.set_value(value_c) self.assertEquals(cursor_p.insert(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(50)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(50)) self.check(ds, uri, nrows, value_a, 20) self.check(ds, uri, nrows, value_b, 35) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare11.py b/src/third_party/wiredtiger/test/suite/test_prepare11.py index 4b35a173576..5770d9ad2f2 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare11.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare11.py @@ -29,38 +29,40 @@ import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_prepare11.py # Test prepare rollback with a reserved update between updates. class test_prepare11(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all)' + conn_config = 'cache_size=2MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', key1=17)), + ('string-row', dict(key_format='S', key1='key1')), + ] + commit_values = [ ('commit', dict(commit=True)), ('rollback', dict(commit=False)), ] - scenarios = make_scenarios(commit_values) + scenarios = make_scenarios(key_format_values, commit_values) def test_prepare_update_rollback(self): uri = "table:test_prepare11" - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) self.session.begin_transaction("isolation=snapshot") # In the scenario where we have a reserved update in between two updates, the key repeated # flag won't get set and we'll call resolve prepared op on both prepared updates. c = self.session.open_cursor(uri, None) - c['key1'] = 'xxxx' - c.set_key('key1') + c[self.key1] = 'xxxx' + c.set_key(self.key1) c.reserve() - c['key1'] = 'yyyy' + c[self.key1] = 'yyyy' self.session.prepare_transaction('prepare_timestamp=10') if self.commit: - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(20)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(30)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(20)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(30)) self.session.commit_transaction() else: self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare12.py b/src/third_party/wiredtiger/test/suite/test_prepare12.py index 55d6c69c2d7..f559bde9c1b 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare12.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare12.py @@ -31,42 +31,48 @@ # [END_TAGS] import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare12.py # Test update restore of a page with prepared update. class test_prepare12(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all)' + conn_config = 'cache_size=2MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_prepare_update_restore(self): uri = "table:test_prepare12" - self.session.create(uri, 'key_format=i,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) # Prepare a transaction cursor = self.session.open_cursor(uri, None) self.session.begin_transaction() - cursor[0] = 'a' - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(1)) + cursor[1] = 'a' + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(1)) # Insert an uncommitted key session2 = self.conn.open_session(None) cursor2 = session2.open_cursor(uri, None) session2.begin_transaction() - cursor2[1] = 'b' + cursor2[2] = 'b' # Insert a bunch of other content to fill the database to trigger eviction. session3 = self.conn.open_session(None) cursor3 = session3.open_cursor(uri, None) - for i in range(2, 100): + for i in range(3, 101): session3.begin_transaction() cursor3[i] = 'a' * 500 session3.commit_transaction() # Commit the prepared update - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1) + ',durable_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1) + ',durable_timestamp=' + self.timestamp_str(2)) # Read the prepared update - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) - self.assertEqual(cursor[0], 'a') + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) + self.assertEqual(cursor[1], 'a') diff --git a/src/third_party/wiredtiger/test/suite/test_prepare13.py b/src/third_party/wiredtiger/test/suite/test_prepare13.py index 8542a182328..fbb735ce4f1 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare13.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare13.py @@ -34,22 +34,29 @@ # Fast-truncate fails when a page contains prepared updates. import wiredtiger, wttest from wtdataset import simple_key, simple_value +from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t class test_prepare13(wttest.WiredTigerTestCase): # Force a small cache. - conn_config = 'cache_size=10MB,statistics=(all),statistics_log=(json,on_close,wait=1)' + conn_config = 'cache_size=10MB' + + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) def test_prepare(self): nrows = 20000 # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) # Create a large table with lots of pages. uri = "table:test_prepare13" - config = 'allocation_size=512,leaf_page_max=512,key_format=S,value_format=S' + key_format_str = "key_format=" + self.key_format + config = 'allocation_size=512,leaf_page_max=512,{},value_format=S'.format(key_format_str) self.session.create(uri, config) cursor = self.session.open_cursor(uri) for i in range(1, nrows): @@ -61,12 +68,12 @@ class test_prepare13(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri) cursor[simple_key(cursor, 1000)] = "replacement_value" cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) try: # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) # Open a separate session and cursor and perform updates to let prepared update to evict. s = self.conn.open_session() @@ -74,7 +81,7 @@ class test_prepare13(wttest.WiredTigerTestCase): for i in range(2000, nrows): s.begin_transaction() cursor[simple_key(cursor, i)] = simple_value(cursor, i) - s.commit_transaction('commit_timestamp=' + timestamp_str(20)) + s.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) cursor.close() # Truncate the middle chunk and expect a conflict. @@ -90,8 +97,8 @@ class test_prepare13(wttest.WiredTigerTestCase): s.rollback_transaction() finally: - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(50)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(50)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(50)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(50)) self.session.commit_transaction() s.close() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare14.py b/src/third_party/wiredtiger/test/suite/test_prepare14.py index fb32aefc713..71737907526 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare14.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare14.py @@ -28,11 +28,9 @@ import wttest from wiredtiger import WT_NOTFOUND +from wtdataset import simple_key from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_prepare14.py # Test that the transaction visibility of an on-disk update # that has both the start and the stop time points from the @@ -61,37 +59,34 @@ class test_prepare14(wttest.WiredTigerTestCase): return config def test_prepare14(self): - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:prepare14" - create_config = 'allocation_size=512,key_format=S,value_format=S' + create_config = 'allocation_size=512,key_format={},value_format=S'.format(self.key_format) self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value = 'a' - # Perform several updates and removes. + # Perform an update and a remove. s = self.conn.open_session() cursor = s.open_cursor(uri) s.begin_transaction() - cursor[str(0)] = value - cursor.set_key(str(0)) + key = simple_key(cursor, 1) + cursor[key] = value + cursor.set_key(key) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction("ignore_prepare = true") - evict_cursor.set_key(str(0)) + evict_cursor.set_key(key) self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() evict_cursor.close() @@ -99,6 +94,6 @@ class test_prepare14(wttest.WiredTigerTestCase): self.session.begin_transaction("ignore_prepare = true") cursor2 = self.session.open_cursor(uri) - cursor2.set_key(str(0)) + cursor2.set_key(key) self.assertEquals(cursor2.search(), WT_NOTFOUND) self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare15.py b/src/third_party/wiredtiger/test/suite/test_prepare15.py index 4c4ba49a182..b4db822caf6 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare15.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare15.py @@ -30,9 +30,6 @@ import wttest from wiredtiger import WT_NOTFOUND from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_prepare15.py # Test that the prepare transaction rollback removes the on-disk key # or replace it with history store and commit retains the changes when @@ -64,18 +61,14 @@ class test_prepare15(wttest.WiredTigerTestCase): return config def test_prepare_restore_hs_update(self): - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:prepare15" create_config = 'allocation_size=512,key_format=S,value_format=S' self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' valueb = 'a' @@ -84,12 +77,12 @@ class test_prepare15(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri) self.session.begin_transaction() cursor[str(0)] = valuea - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) self.session.begin_transaction() cursor.set_key(str(0)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) cursor.close() # Perform an update and remove. @@ -100,7 +93,7 @@ class test_prepare15(wttest.WiredTigerTestCase): cursor.set_key(str(0)) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") @@ -115,8 +108,8 @@ class test_prepare15(wttest.WiredTigerTestCase): if self.commit: # Commit the prepared transaction - s.timestamp_transaction('commit_timestamp=' + timestamp_str(50)) - s.timestamp_transaction('durable_timestamp=' + timestamp_str(60)) + s.timestamp_transaction('commit_timestamp=' + self.timestamp_str(50)) + s.timestamp_transaction('durable_timestamp=' + self.timestamp_str(60)) s.commit_transaction() else: # Rollback the prepared transaction @@ -133,7 +126,7 @@ class test_prepare15(wttest.WiredTigerTestCase): evict_cursor.close() self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(20)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) cursor2 = self.session.open_cursor(uri) cursor2.set_key(str(0)) self.assertEquals(cursor2.search(), 0) @@ -141,18 +134,14 @@ class test_prepare15(wttest.WiredTigerTestCase): self.session.commit_transaction() def test_prepare_not_found(self): - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:prepare15" create_config = 'allocation_size=512,key_format=S,value_format=S' self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value = 'a' @@ -164,7 +153,7 @@ class test_prepare15(wttest.WiredTigerTestCase): cursor.set_key(str(0)) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") @@ -179,8 +168,8 @@ class test_prepare15(wttest.WiredTigerTestCase): if self.commit: # Commit the prepared transaction - s.timestamp_transaction('commit_timestamp=' + timestamp_str(30)) - s.timestamp_transaction('durable_timestamp=' + timestamp_str(40)) + s.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) + s.timestamp_transaction('durable_timestamp=' + self.timestamp_str(40)) s.commit_transaction() else: # Rollback the prepared transaction diff --git a/src/third_party/wiredtiger/test/suite/test_prepare16.py b/src/third_party/wiredtiger/test/suite/test_prepare16.py index 23eb85bf444..f6e05aadfa9 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare16.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare16.py @@ -30,9 +30,6 @@ import wttest from wiredtiger import WT_NOTFOUND from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_prepare16.py # Test that the prepare transaction rollback/commit multiple keys # and each key can occupy a leaf page. @@ -65,18 +62,14 @@ class test_prepare16(wttest.WiredTigerTestCase): def test_prepare(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:prepare16" create_config = 'allocation_size=512,key_format=S,value_format=S,leaf_page_max=512,leaf_value_max=64MB' self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' * 400 @@ -87,7 +80,7 @@ class test_prepare16(wttest.WiredTigerTestCase): cursor.reset() cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) s = self.conn.open_session() s.begin_transaction('ignore_prepare = true') @@ -100,17 +93,17 @@ class test_prepare16(wttest.WiredTigerTestCase): evict_cursor.reset() if self.commit: - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(20)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(30)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(20)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(30)) self.session.commit_transaction() else: self.session.rollback_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) if not self.in_memory: self.session.checkpoint() - self.session.begin_transaction('read_timestamp=' + timestamp_str(20)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) cursor = self.session.open_cursor(uri) for i in range(1, nrows + 1): cursor.set_key(str(i)) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py b/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py index 3f298e63b9b..d62ee17b2a2 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py @@ -32,15 +32,21 @@ import wiredtiger, wttest from wtdataset import simple_key, simple_value - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_prepare_conflict(wttest.WiredTigerTestCase): + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_prepare(self): # Create a large table with lots of pages. uri = "table:test_prepare_conflict" - config = 'allocation_size=512,leaf_page_max=512,key_format=S,value_format=S' + key_format = 'key_format=' + self.key_format + config = 'allocation_size=512,leaf_page_max=512,{},value_format=S'.format(key_format) self.session.create(uri, config) cursor = self.session.open_cursor(uri) for i in range(1, 80000): @@ -68,9 +74,9 @@ class test_prepare_conflict(wttest.WiredTigerTestCase): cursor.close() # Prepare and commit the transaction. - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(20)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(20)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(20)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(20)) self.session.commit_transaction() # WT-6325 reports WT_PREPARE_CONFLICT while iterating the cursor. diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py b/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py index aa2e45bec32..579927bd38e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py @@ -35,9 +35,6 @@ from wtdataset import SimpleDataSet, SimpleIndexDataSet from wtdataset import SimpleLSMDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_prepare_cursor01.py # WT_CURSOR navigation (next/prev) tests with prepared transactions class test_prepare_cursor01(wttest.WiredTigerTestCase): @@ -103,18 +100,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_cursor.set_key(ds.key(51)) prep_cursor.set_value(ds.value(51)) prep_cursor.insert() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(100)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(100)) # Point all cursors to key 50. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(50)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(50)) before_ts_c.set_key(ds.key(50)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(150)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(150)) between_ts_c.set_key(ds.key(50)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(250)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(250)) after_ts_c.set_key(ds.key(50)) self.assertEquals(after_ts_c.search(), 0) @@ -138,8 +135,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(200)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(200)) prep_session.commit_transaction() before_ts_s.commit_transaction() @@ -160,18 +157,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_cursor.set_key(ds.key(1)) prep_cursor.set_value(ds.value(1)) prep_cursor.insert() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(100)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(100)) # Point all cursors to key 2. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(50)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(50)) before_ts_c.set_key(ds.key(2)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(150)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(150)) between_ts_c.set_key(ds.key(2)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(250)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(250)) after_ts_c.set_key(ds.key(2)) self.assertEquals(after_ts_c.search(), 0) @@ -196,8 +193,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(200)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(200)) prep_session.commit_transaction() # As read is between(i.e before commit), prev is not found. @@ -223,18 +220,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_cursor.set_key(ds.key(51)) prep_cursor.set_value(ds.value(151)) prep_cursor.update() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(300)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(300)) # Point all cursors to key 51. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(250)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(250)) before_ts_c.set_key(ds.key(50)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(350)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(350)) between_ts_c.set_key(ds.key(50)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(450)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(450)) after_ts_c.set_key(ds.key(50)) self.assertEquals(after_ts_c.search(), 0) @@ -256,8 +253,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(400)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(400)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(400)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(400)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -287,18 +284,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_cursor.set_key(ds.key(1)) prep_cursor.set_value(ds.value(111)) prep_cursor.update() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(300)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(300)) # Point all cursors to key 2. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(250)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(250)) before_ts_c.set_key(ds.key(2)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(350)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(350)) between_ts_c.set_key(ds.key(2)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(450)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(450)) after_ts_c.set_key(ds.key(2)) self.assertEquals(after_ts_c.search(), 0) @@ -320,8 +317,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(400)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(400)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(400)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(400)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -355,18 +352,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_session.begin_transaction() prep_cursor.set_key(ds.key(51)) prep_cursor.remove() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(500)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(500)) # Point all cursors to key 51. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(450)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(450)) before_ts_c.set_key(ds.key(50)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(550)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(550)) between_ts_c.set_key(ds.key(50)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(650)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(650)) after_ts_c.set_key(ds.key(50)) self.assertEquals(after_ts_c.search(), 0) @@ -387,8 +384,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(600)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(600)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(600)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(600)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -411,18 +408,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_session.begin_transaction() prep_cursor.set_key(ds.key(1)) prep_cursor.remove() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(500)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(500)) # Point all cursors to key 2. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(450)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(450)) before_ts_c.set_key(ds.key(2)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(550)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(550)) between_ts_c.set_key(ds.key(2)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(650)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(650)) after_ts_c.set_key(ds.key(2)) self.assertEquals(after_ts_c.search(), 0) @@ -443,8 +440,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(600)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(600)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(600)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(600)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -474,18 +471,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_session.begin_transaction() prep_cursor.set_key(ds.key(49)) prep_cursor.remove() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(700)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(700)) # Point all cursors to key 48. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(650)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(650)) before_ts_c.set_key(ds.key(48)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(750)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(750)) between_ts_c.set_key(ds.key(48)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(850)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(850)) after_ts_c.set_key(ds.key(48)) self.assertEquals(after_ts_c.search(), 0) @@ -506,8 +503,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(800)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(800)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(800)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(800)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -532,18 +529,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_session.begin_transaction() prep_cursor.set_key(ds.key(3)) prep_cursor.remove() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(700)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(700)) # Point all cursors to key 4. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(650)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(650)) before_ts_c.set_key(ds.key(4)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(750)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(750)) between_ts_c.set_key(ds.key(4)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(850)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(850)) after_ts_c.set_key(ds.key(4)) self.assertEquals(after_ts_c.search(), 0) @@ -564,8 +561,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(800)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(800)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(800)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(800)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py b/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py index 65df05fd239..4353ffabf70 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py @@ -31,9 +31,6 @@ from wtdataset import SimpleDataSet, SimpleIndexDataSet from wtdataset import SimpleLSMDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_prepare_cursor02.py # WT_CURSOR navigation (next/prev) tests with prepared transactions class test_prepare_cursor02(wttest.WiredTigerTestCase): @@ -69,7 +66,7 @@ class test_prepare_cursor02(wttest.WiredTigerTestCase): cursor.set_key(ds.key(1)) cursor.set_value(ds.value(1)) cursor.insert() - session.prepare_transaction('prepare_timestamp=' + timestamp_str(100)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(100)) prep_session = self.conn.open_session(self.session_config) prep_cursor = prep_session.open_cursor(uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py index c7810e0f719..909f534982e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py @@ -29,9 +29,7 @@ from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare_hs01.py # test to ensure history store eviction is working for prepared transactions. @@ -39,9 +37,16 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=50MB,eviction_updates_trigger=95,eviction_updates_target=80' + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) + def check(self, uri, ds, nrows, nsessions, nkeys, read_ts, expected_value, not_expected_value): cursor = self.session.open_cursor(uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) for i in range(1, nsessions * nkeys): cursor.set_key(ds.key(nrows + i)) self.assertEquals(cursor.search(), 0) @@ -67,7 +72,7 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): # memory. Hence testing if we can read prepared updates from the history store. # Start with setting a stable timestamp to pin history in cache - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) # Commit some updates to get eviction and history store fired up bigvalue1 = b"bbbbb" * 100 @@ -77,7 +82,7 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): cursor.set_key(ds.key(nrows + i)) cursor.set_value(bigvalue1) self.assertEquals(cursor.insert(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) # Have prepared updates in multiple sessions. This should ensure writing # prepared updates to the history store @@ -95,7 +100,7 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): cursors[j].set_key(ds.key(nrows + i)) cursors[j].set_value(bigvalue2) self.assertEquals(cursors[j].insert(), 0) - sessions[j].prepare_transaction('prepare_timestamp=' + timestamp_str(2)) + sessions[j].prepare_transaction('prepare_timestamp=' + self.timestamp_str(2)) # Re-read the original versions of all the data. This ensures reading # original versions from the history store @@ -116,7 +121,7 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_prepare_hs01" nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') + ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format='u') ds.populate() bigvalue = b"aaaaa" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py index 5ba68e6a614..6fbd416e94b 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py @@ -36,9 +36,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_prepare_cursor' uri = 'table:' + tablename @@ -72,16 +69,16 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction(self.txn_config) c[1] = 1 # update the value with in this transaction - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(100)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(100)) if self.txn_commit == True: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(101) + ',durable_timestamp=' + timestamp_str(101)) + 'commit_timestamp=' + self.timestamp_str(101) + ',durable_timestamp=' + self.timestamp_str(101)) else: self.session.rollback_transaction() # Trigger a checkpoint, which could trigger reconciliation - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(150)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(150)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(150)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(150)) self.session.checkpoint() # Scenario: 2 @@ -94,16 +91,16 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): # update a uncommitted value, insert and update a key. c[2] = 1 c[2] = 2 - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(200)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(200)) if self.txn_commit == True: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(201) + ',durable_timestamp=' + timestamp_str(201)) + 'commit_timestamp=' + self.timestamp_str(201) + ',durable_timestamp=' + self.timestamp_str(201)) else: self.session.rollback_transaction() # Trigger a checkpoint, which could trigger reconciliation - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(250)) self.session.checkpoint() # Scenario: 3 @@ -121,10 +118,10 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): c[3] = 2 c.set_key(3) c.remove() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(300)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(300)) if self.txn_commit == True: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(301) + ',durable_timestamp=' + timestamp_str(301)) + 'commit_timestamp=' + self.timestamp_str(301) + ',durable_timestamp=' + self.timestamp_str(301)) else: self.session.rollback_transaction() @@ -133,11 +130,11 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): c[1] = 1 c[2] = 1 c[3] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(302)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(302)) # Trigger a checkpoint, which could trigger reconciliation - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(350)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(350)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(350)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(350)) self.session.checkpoint() #Scenario: 4 @@ -145,8 +142,8 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): # creating the modify update_chain for key instead of insert update # chain. self.reopen_conn() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(350)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(350)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(350)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(350)) self.session.create(self.uri, self.s_config) cur = self.session.open_cursor(self.uri) @@ -159,16 +156,16 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): # Remove a updated key cur.set_key(3) cur.remove() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(400)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(400)) if self.txn_commit == True: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(401) + ',durable_timestamp=' + timestamp_str(401)) + 'commit_timestamp=' + self.timestamp_str(401) + ',durable_timestamp=' + self.timestamp_str(401)) else: self.session.rollback_transaction() # Trigger a checkpoint, which could trigger reconciliation - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(450)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(450)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(450)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(450)) self.session.checkpoint() cur.close() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py index 502a7ba9c8e..c2d6ca66d4e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py @@ -38,9 +38,6 @@ import os, shutil from wtscenario import make_scenarios from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_prepare_hs03.py # test to ensure salvage, verify & simulating crash are working for prepared transactions. class test_prepare_hs03(wttest.WiredTigerTestCase): @@ -51,10 +48,17 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_prepare_hs03" - scenarios = make_scenarios([ + corrupt_values = [ ('corrupt_table', dict(corrupt=True)), ('dont_corrupt_table', dict(corrupt=False)) - ]) + ] + + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(corrupt_values, key_format_values) def corrupt_table(self): tablename="test_prepare_hs03.wt" @@ -96,12 +100,12 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): cursor.set_key(ds.key(nrows + i)) cursor.set_value(commit_value) self.assertEquals(cursor.insert(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) cursor.close() # Set the stable/oldest timstamps. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) # Corrupt the table, call salvage to recover data from the corrupted table and call verify self.corrupt_salvage_verify() @@ -124,7 +128,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): cursors[j].set_key(ds.key(nrows + i)) cursors[j].set_value(prepare_value) self.assertEquals(cursors[j].insert(), 0) - sessions[j].prepare_transaction('prepare_timestamp=' + timestamp_str(4)) + sessions[j].prepare_transaction('prepare_timestamp=' + self.timestamp_str(4)) hs_writes = self.get_stat(stat.conn.cache_write_hs) - hs_writes_start @@ -133,7 +137,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): # Test if we can read prepared updates from the history store. cursor = self.session.open_cursor(self.uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) for i in range(1, nsessions * nkeys): cursor.set_key(ds.key(nrows + i)) # The search should pass. @@ -156,7 +160,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): # Finally, search for the keys inserted with commit timestamp cursor = self.session.open_cursor(self.uri) self.pr('Read Keys') - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) for i in range(1, nkeys): cursor.set_key(ds.key(nrows + i)) # The search should pass @@ -177,7 +181,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri) # Search the keys inserted with commit timestamp after crash - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) for i in range(1, nkeys): cursor.set_key(ds.key(nrows + i)) # The search should pass @@ -195,7 +199,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): def test_prepare_hs(self): nrows = 100 - ds = SimpleDataSet(self, self.uri, nrows, key_format="S", value_format='u') + ds = SimpleDataSet(self, self.uri, nrows, key_format=self.key_format, value_format='u') ds.populate() bigvalue = b"aaaaa" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py index 40a80ed0b0e..fa74584ab0c 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py @@ -32,9 +32,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_prepare_hs04.py # Read prepared updates from on-disk with ignore_prepare. # Committing or aborting a prepared update when there exists a tombstone for that key already. @@ -50,10 +47,18 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): nkeys = 40 nrows = 100 - scenarios = make_scenarios([ + commit_values = [ ('commit_transaction', dict(commit=True)), ('rollback_transaction', dict(commit=False)) - ]) + ] + + key_format_values = [ + # Note: commit_key must exceed nrows to give behavior comparable to the row case. + ('column', dict(key_format='r', commit_key=1000)), + ('string-row', dict(key_format='S', commit_key='C')), + ] + + scenarios = make_scenarios(commit_values, key_format_values) def get_stat(self, stat): stat_cursor = self.session.open_cursor('statistics:') @@ -64,7 +69,7 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): def search_keys_timestamp_and_ignore(self, ds, txn_config, expected_value, conflict=False): cursor = self.session.open_cursor(self.uri) - commit_key = "C" + commit_key = self.commit_key self.session.begin_transaction(txn_config) for i in range(1, self.nsessions * self.nkeys): key = commit_key + ds.key(self.nrows + i) @@ -82,12 +87,12 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): def prepare_updates(self, ds): # Set oldest and stable timestamp for the database. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) # Commit some updates to get eviction and history store fired up. # Insert a key at timestamp 1. - commit_key = "C" + commit_key = self.commit_key commit_value = b"bbbbb" * 100 cursor = self.session.open_cursor(self.uri) for i in range(1, self.nsessions * self.nkeys): @@ -96,7 +101,7 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): cursor.set_key(key) cursor.set_value(commit_value) self.assertEquals(cursor.insert(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) cursor.close() # Call checkpoint. @@ -108,11 +113,11 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): key = commit_key + ds.key(self.nrows + i) cursor.set_key(key) self.assertEquals(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) cursor.close() # Move the stable timestamp to match the timestamp for the last update. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) hs_writes_start = self.get_stat(stat.conn.cache_write_hs) # Have prepared updates in multiple sessions. This should ensure writing prepared updates to @@ -131,22 +136,22 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): cursors[j].set_key(commit_key + ds.key(self.nrows + i)) cursors[j].set_value(prepare_value) self.assertEquals(cursors[j].insert(), 0) - sessions[j].prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + sessions[j].prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) hs_writes = self.get_stat(stat.conn.cache_write_hs) - hs_writes_start # Assert if not writing anything to the history store. self.assertGreaterEqual(hs_writes, 0) - txn_config = 'read_timestamp=' + timestamp_str(5) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(5) + ',ignore_prepare=true' # Search keys with timestamp 5, ignore_prepare=true and expect the cursor search to return 0 (key found) self.search_keys_timestamp_and_ignore(ds, txn_config, commit_value) - txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(20) + ',ignore_prepare=true' # Search keys with timestamp 20, ignore_prepare=true, expect the cursor to return wiredtiger.WT_NOTFOUND self.search_keys_timestamp_and_ignore(ds, txn_config, None) prepare_conflict_msg = '/conflict with a prepared update/' - txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(20) + ',ignore_prepare=false' # Search keys with timestamp 20, ignore_prepare=false and expect the cursor the cursor search to return prepare conflict message self.search_keys_timestamp_and_ignore(ds, txn_config, prepare_conflict_msg, True) @@ -156,9 +161,9 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): # Commit the prepared_transactions with timestamp 30. for j in range (0, self.nsessions): sessions[j].commit_transaction( - 'commit_timestamp=' + timestamp_str(30) + ',durable_timestamp=' + timestamp_str(30)) + 'commit_timestamp=' + self.timestamp_str(30) + ',durable_timestamp=' + self.timestamp_str(30)) # Move the stable timestamp to match the durable timestamp for prepared updates. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) self.session.checkpoint() @@ -174,44 +179,44 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): # Search keys with timestamp 5, ignore_prepare=true and expect the cursor search to return # value committed before prepared update. - txn_config = 'read_timestamp=' + timestamp_str(5) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(5) + ',ignore_prepare=false' self.search_keys_timestamp_and_ignore(ds, txn_config, commit_value) # Search keys with timestamp 20, ignore_prepare=true and expect the cursor search to return # WT_NOTFOUND. - txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(20) + ',ignore_prepare=true' self.search_keys_timestamp_and_ignore(ds, txn_config, None) # Search keys with timestamp 20, ignore_prepare=false and expect the cursor search to return WT_NOTFOUND. - txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(20) + ',ignore_prepare=false' self.search_keys_timestamp_and_ignore(ds, txn_config, None) # If commit is true then the commit_tramsactions was called and we will expect prepare_value. if self.commit == True: - txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(30) + ',ignore_prepare=true' # Search keys with timestamp 30, ignore_prepare=true and expect the cursor value to be prepare_value. self.search_keys_timestamp_and_ignore(ds, txn_config, prepare_value) else: # Commit is false and we simulated a crash/restart which would have rolled-back the transactions, therefore we expect the # cursor search to return WT_NOTFOUND. - txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(30) + ',ignore_prepare=true' # Search keys with timestamp 30, ignore_prepare=true and expect the cursor value to return WT_NOTFOUND. self.search_keys_timestamp_and_ignore(ds, txn_config, None) if self.commit == True: - txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(30) + ',ignore_prepare=false' # Search keys with timestamp 30, ignore_prepare=false and expect the cursor value to be prepare_value. self.search_keys_timestamp_and_ignore(ds, txn_config, prepare_value) else: # Commit is false and we simulated a crash/restart which would have rolled-back the transactions, therefore we expect the # cursor search to return WT_NOTFOUND. - txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(30) + ',ignore_prepare=false' # Search keys with timestamp 30, ignore_prepare=false and expect the cursor value to return WT_NOTFOUND. self.search_keys_timestamp_and_ignore(ds, txn_config, None) def test_prepare_hs(self): - ds = SimpleDataSet(self, self.uri, self.nrows, key_format="S", value_format='u') + ds = SimpleDataSet(self, self.uri, self.nrows, key_format=self.key_format, value_format='u') ds.populate() bigvalue = b"aaaaa" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py index ae79bd247f5..cc6f4a2f05d 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py @@ -30,52 +30,56 @@ import wiredtiger, wttest from wtscenario import make_scenarios from wiredtiger import stat, WT_NOTFOUND -def timestamp_str(t): - return '%x' % t - # test_prepare_hs05.py # Test that after aborting prepare transaction, correct update from the history store is restored. class test_prepare_hs05(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', key=1)), + ('string-row', dict(key_format='S', key=str(1))), + ] + + scenarios = make_scenarios(key_format_values) + def test_check_prepare_abort_hs_restore(self): uri = 'table:test_prepare_hs05' - create_params = 'key_format=S,value_format=S' + create_params = 'key_format={},value_format=S'.format(self.key_format) self.session.create(uri, create_params) value1 = 'a' * 5 value2 = 'b' * 5 value3 = 'c' * 5 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - key = 1 + key = self.key self.session.begin_transaction() - cursor[str(key)] = value1 - cursor.set_key(str(key)) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + cursor[key] = value1 + cursor.set_key(key) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Commit update and remove operation in the same transaction. self.session.begin_transaction() - cursor[str(key)] = value2 - cursor.set_key(str(key)) + cursor[key] = value2 + cursor.set_key(key) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Add a prepared update for the key. self.session.begin_transaction() - cursor[str(key)] = value3 - self.session.prepare_transaction('prepare_timestamp='+ timestamp_str(4)) + cursor[key] = value3 + self.session.prepare_transaction('prepare_timestamp='+ self.timestamp_str(4)) # Try to evict the page with prepared update. This will ensure that prepared update is # written as the on-disk version and the older versions are moved to the history store. session2 = self.conn.open_session() session2.begin_transaction('ignore_prepare=true') cursor2 = session2.open_cursor(uri, None, "debug=(release_evict=true)") - cursor2.set_key(str(key)) + cursor2.set_key(key) self.assertEquals(cursor2.search(), WT_NOTFOUND) cursor2.reset() @@ -85,14 +89,14 @@ class test_prepare_hs05(wttest.WiredTigerTestCase): self.session.checkpoint() # We should be able to read the older version of the key from the history store. - self.session.begin_transaction('read_timestamp='+timestamp_str(2)) - cursor.set_key(str(key)) + self.session.begin_transaction('read_timestamp='+self.timestamp_str(2)) + cursor.set_key(key) self.assertEqual(cursor.search(), 0) self.assertEqual(cursor.get_value(), value1) self.session.rollback_transaction() # The latest version should be marked deleted. self.session.begin_transaction() - cursor.set_key(str(key)) + cursor.set_key(key) self.assertEqual(cursor.search(), WT_NOTFOUND) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_readonly02.py b/src/third_party/wiredtiger/test/suite/test_readonly02.py index 4adaf476a0b..62270a52302 100644 --- a/src/third_party/wiredtiger/test/suite/test_readonly02.py +++ b/src/third_party/wiredtiger/test/suite/test_readonly02.py @@ -42,15 +42,15 @@ class test_readonly02(wttest.WiredTigerTestCase, suite_subprocess): entries = 10 conn_params = \ - 'create,statistics=(fast),' + \ + 'create,' + \ 'log=(enabled,file_max=100K,zero_fill=true),' + \ 'operation_tracking=(enabled=false),' conn_params_rd = \ - 'create,readonly=true,statistics=(fast),' + \ + 'create,readonly=true,' + \ 'log=(enabled,zero_fill=false),' + \ 'operation_tracking=(enabled=false),' conn_params_rdcfg = \ - 'create,readonly=true,statistics=(fast),log=(enabled),' + \ + 'create,readonly=true,log=(enabled),' + \ 'operation_tracking=(enabled=false),' # diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py index 7613e169fb1..e85de366970 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py @@ -34,9 +34,6 @@ from wiredtiger import stat, wiredtiger_strerror, WiredTigerError, WT_ROLLBACK from wtscenario import make_scenarios from time import sleep -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable01.py # Shared base class used by rollback to stable tests. class test_rollback_to_stable_base(wttest.WiredTigerTestCase): @@ -77,12 +74,12 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): if commit_ts == 0: session.commit_transaction() elif prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() except WiredTigerError as e: rollback_str = wiredtiger_strerror(WT_ROLLBACK) @@ -104,12 +101,12 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): if commit_ts == 0: session.commit_transaction() elif prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() except WiredTigerError as e: rollback_str = wiredtiger_strerror(WT_ROLLBACK) @@ -129,12 +126,12 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): if commit_ts == 0: session.commit_transaction() elif prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() except WiredTigerError as e: rollback_str = wiredtiger_strerror(WT_ROLLBACK) @@ -147,7 +144,7 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): if read_ts == 0: session.begin_transaction() else: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -189,10 +186,6 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable01" ds = SimpleDataSet( @@ -200,8 +193,8 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) valuea = "aaaaa" * 100 self.large_updates(uri, valuea, ds, nrows, self.prepare, 10) @@ -215,9 +208,9 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): # Pin stable to timestamp 20 if prepare otherwise 10. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Checkpoint to ensure that all the updates are flushed to disk. if not self.in_memory: self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py index 20b5d7d6a41..8c27f5395e7 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py @@ -34,9 +34,6 @@ from wiredtiger import stat from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable02.py # Test that rollback to stable brings back the history value to replace on-disk value. class test_rollback_to_stable02(test_rollback_to_stable_base): @@ -70,10 +67,6 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable02" ds = SimpleDataSet( @@ -81,8 +74,8 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) valuea = "aaaaa" * 100 valueb = "bbbbb" * 100 @@ -106,9 +99,9 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) # Checkpoint to ensure that all the data is flushed. if not self.in_memory: self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py index a0e2d4337b2..c7260db1166 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py @@ -34,9 +34,6 @@ from wiredtiger import stat from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable03.py # Test that rollback to stable clears the history store updates from reconciled pages. class test_rollback_to_stable01(test_rollback_to_stable_base): @@ -70,10 +67,6 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable03" ds = SimpleDataSet( @@ -81,8 +74,8 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) valuea = "aaaaa" * 100 valueb = "bbbbb" * 100 @@ -101,9 +94,9 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) # Checkpoint to ensure that all the updates are flushed to disk. if not self.in_memory: self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py index f865a3975f7..2342163a327 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py @@ -31,9 +31,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - def mod_val(value, char, location, nbytes=1): return value[0:location] + char + value[location+nbytes:] @@ -71,10 +68,6 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable04" ds = SimpleDataSet( @@ -82,8 +75,8 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -131,9 +124,9 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): # Pin stable to timestamp 40 if prepare otherwise 30. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) # Checkpoint to ensure the data is flushed, then rollback to the stable timestamp. if not self.in_memory: diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py index 5604196241c..4a32b589b18 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py @@ -34,9 +34,6 @@ from wiredtiger import stat from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable05.py # Test that rollback to stable cleans history store for non-timestamp tables. class test_rollback_to_stable05(test_rollback_to_stable_base): @@ -70,10 +67,6 @@ class test_rollback_to_stable05(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create two tables without logging. uri_1 = "table:rollback_to_stable05_1" ds_1 = SimpleDataSet( diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py index cd2c35783ce..7b702ca7b5e 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py @@ -31,9 +31,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable06.py # Test that rollback to stable removes all keys when the stable timestamp is earlier than # all commit timestamps. @@ -68,10 +65,6 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable06" ds = SimpleDataSet( @@ -79,8 +72,8 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py index 9c9e39d16b9..f745d0a3592 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py @@ -33,9 +33,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable07.py # Test the rollback to stable operation performs as expected following a server crash and # recovery. Verify that @@ -63,10 +60,6 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable07" ds = SimpleDataSet( @@ -74,8 +67,8 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -96,9 +89,9 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) # Perform additional updates. self.large_updates(uri, value_b, ds, nrows, self.prepare, 60) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py index 05153034362..e5c3c5bc9ae 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py @@ -31,9 +31,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable08.py # Test that rollback to stable does not abort updates when the stable timestamp is # set to the latest commit. @@ -68,10 +65,6 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable08" ds = SimpleDataSet( @@ -79,8 +72,8 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -101,9 +94,9 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Checkpoint to ensure the data is flushed, then rollback to the stable timestamp. if not self.in_memory: diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py index d0a45f0522d..70664a3d776 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py @@ -32,15 +32,17 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable09.py # Test that rollback to stable does not abort schema operations that are done # as they don't have transaction support class test_rollback_to_stable09(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + colstore_values = [ + ('column', dict(use_columns=True)), + ('row', dict(use_columns=False)), + ] + in_memory_values = [ ('no_inmem', dict(in_memory=False)), ('inmem', dict(in_memory=True)) @@ -55,10 +57,10 @@ class test_rollback_to_stable09(test_rollback_to_stable_base): uri = "table:" + tablename index_uri = "index:test_rollback_stable09:country" - scenarios = make_scenarios(in_memory_values, prepare_values) + scenarios = make_scenarios(colstore_values, in_memory_values, prepare_values) def conn_config(self): - config = 'cache_size=250MB,statistics=(all)' + config = 'cache_size=250MB' if self.in_memory: config += ',in_memory=true' else: @@ -69,16 +71,20 @@ class test_rollback_to_stable09(test_rollback_to_stable_base): self.pr('create table') session = self.session session.begin_transaction() - session.create(self.uri, 'key_format=5s,value_format=HQ,columns=(country,year,population)') + if self.use_columns: + config = 'key_format=r,value_format=5sHQ,columns=(id,country,year,population)' + else: + config = 'key_format=5s,value_format=HQ,columns=(country,year,population)' + session.create(self.uri, config) if commit_ts == 0: session.commit_transaction() elif self.prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) def drop_table(self, commit_ts): self.pr('drop table') @@ -88,12 +94,12 @@ class test_rollback_to_stable09(test_rollback_to_stable_base): if commit_ts == 0: session.commit_transaction() elif self.prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) def create_index(self, commit_ts): session = self.session @@ -102,17 +108,17 @@ class test_rollback_to_stable09(test_rollback_to_stable_base): if commit_ts == 0: session.commit_transaction() elif self.prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) def test_rollback_to_stable(self): # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) # Create table and index at a later timestamp self.create_table(20) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py index cfd0c2f3c46..25aaae7ff67 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py @@ -34,9 +34,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from wtthread import checkpoint_thread, op_thread -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable10.py # Test the rollback to stable operation performs sweeping history store. class test_rollback_to_stable10(test_rollback_to_stable_base): @@ -61,10 +58,6 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate tables") uri_1 = "table:rollback_to_stable10_1" @@ -79,8 +72,8 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): ds_2.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -114,9 +107,9 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() @@ -180,10 +173,6 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): def test_rollback_to_stable_prepare(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate tables") uri_1 = "table:rollback_to_stable10_1" @@ -198,8 +187,8 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): ds_2.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -235,9 +224,9 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Here's the update operations we'll perform, encapsulated so we can easily retry # it if we get a rollback. Rollbacks may occur when checkpoint is running. @@ -265,7 +254,7 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): self.retry_rollback('update ds1', session_p1, lambda: prepare_range_updates( session_p1, cursor_p1, ds_1, value_e, nrows, - 'prepare_timestamp=' + timestamp_str(69))) + 'prepare_timestamp=' + self.timestamp_str(69))) # Perform several updates in parallel with checkpoint. session_p2 = self.conn.open_session() @@ -274,7 +263,7 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): self.retry_rollback('update ds2', session_p2, lambda: prepare_range_updates( session_p2, cursor_p2, ds_2, value_e, nrows, - 'prepare_timestamp=' + timestamp_str(69))) + 'prepare_timestamp=' + self.timestamp_str(69))) finally: done.set() ckpt.join() @@ -290,8 +279,8 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): copy_wiredtiger_home(self, ".", "RESTART") # Commit the prepared transaction. - session_p1.commit_transaction('commit_timestamp=' + timestamp_str(70) + ',durable_timestamp=' + timestamp_str(71)) - session_p2.commit_transaction('commit_timestamp=' + timestamp_str(70) + ',durable_timestamp=' + timestamp_str(71)) + session_p1.commit_transaction('commit_timestamp=' + self.timestamp_str(70) + ',durable_timestamp=' + self.timestamp_str(71)) + session_p2.commit_transaction('commit_timestamp=' + self.timestamp_str(70) + ',durable_timestamp=' + self.timestamp_str(71)) session_p1.close() session_p2.close() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py index 59014d5aac9..f88d1ec0082 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py @@ -33,9 +33,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable11.py # Test the rollback to stable is retrieving the proper history store update. class test_rollback_to_stable11(test_rollback_to_stable_base): @@ -60,10 +57,6 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable11" ds = SimpleDataSet( @@ -71,8 +64,8 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -90,9 +83,9 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) # Checkpoint to ensure that all the updates are flushed to disk. self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py index cd0211f2d12..0ab66b1d991 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py @@ -33,9 +33,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable12.py # Test the rollback to stable operation skipping subtrees in during tree walk. class test_rollback_to_stable12(test_rollback_to_stable_base): @@ -60,10 +57,6 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable12" ds = SimpleDataSet( @@ -71,8 +64,8 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -85,9 +78,9 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) # Load a single row modification to be removed. commit_ts = 30 @@ -95,12 +88,12 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): self.session.begin_transaction() cursor[ds.key(1)] = value_b if self.prepare: - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) self.session.commit_transaction() else: - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py index 73f23b1f615..dddbd02c260 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py @@ -31,9 +31,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable13.py # Test the rollback to stable should retain/restore the tombstone from # the update list or from the history store for on-disk database. @@ -59,10 +56,6 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( @@ -70,8 +63,8 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -92,9 +85,9 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) self.session.checkpoint() # Simulate a server crash and restart. @@ -113,10 +106,6 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): def test_rollback_to_stable_with_aborted_updates(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( @@ -124,8 +113,8 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -164,9 +153,9 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) self.session.checkpoint() # Simulate a server crash and restart. @@ -185,10 +174,6 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): def test_rollback_to_stable_with_history_tombstone(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( @@ -196,8 +181,8 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -216,14 +201,14 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): cursor[ds.key(i)] = value_b cursor.set_key(ds.key(i)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(40)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) cursor.close() # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) self.session.checkpoint() @@ -251,17 +236,14 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): def test_rollback_to_stable_with_stable_remove(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( self, uri, 0, key_format=self.key_format, value_format="S", config='split_pct=50,log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 value_c = "ccccc" * 100 @@ -273,9 +255,9 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): self.large_removes(uri, ds, nrows, self.prepare, 40) # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) # Perform several updates and checkpoint. self.large_updates(uri, value_c, ds, nrows, self.prepare, 60) self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py index e7a5229555d..25b625e1cfa 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py @@ -34,9 +34,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from wtthread import checkpoint_thread, op_thread -def timestamp_str(t): - return '%x' % t - def mod_val(value, char, location, nbytes=1): return value[0:location] + char + value[location+nbytes:] @@ -67,10 +64,6 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1500 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" @@ -79,8 +72,8 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 @@ -106,9 +99,9 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() @@ -172,10 +165,6 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): def test_rollback_to_stable_same_ts(self): nrows = 1500 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" @@ -184,8 +173,8 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 @@ -213,7 +202,7 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.check(value_modQ, uri, nrows, 30) self.check(value_modT, uri, nrows, 60) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() @@ -275,10 +264,6 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): def test_rollback_to_stable_same_ts_append(self): nrows = 1500 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" @@ -287,8 +272,8 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 @@ -316,7 +301,7 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.check(value_modQ, uri, nrows, 30) self.check(value_modT, uri, nrows, 60) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py index b91691b99cd..080fcddfe9a 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py @@ -32,9 +32,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable15.py # Test that roll back to stable handles updates present in the # update-list for both fixed length and variable length column store. @@ -43,7 +40,7 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), + ('integer-row', dict(key_format='i')), ] value_format_values = [ # Fixed length @@ -70,7 +67,7 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): if read_ts == 0: session.begin_transaction() else: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -89,8 +86,8 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri) # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) value20 = 0x20 value30 = 0x30 @@ -101,17 +98,17 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): for i in range(1, nrows): self.session.begin_transaction() cursor[i] = value20 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) #First Update to value 30 at timestamp 5 for i in range(1, nrows): self.session.begin_transaction() cursor[i] = value30 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) cursor.close() #Set stable timestamp to 2 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2)) self.conn.rollback_to_stable() # Check that only value20 is available self.check(value20, uri, nrows - 1, 2) @@ -121,17 +118,17 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): for i in range(1, nrows): self.session.begin_transaction() cursor[i] = value30 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(7)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(7)) #Third Update to value40 at timestamp 9 for i in range(1, nrows): self.session.begin_transaction() cursor[i] = value40 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(9)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(9)) cursor.close() #Set stable timestamp to 7 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(7)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(7)) self.conn.rollback_to_stable() #Check that only value30 is available self.check(value30, uri, nrows - 1, 7) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py index 5ca08c18baf..4c52e52f50c 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py @@ -37,9 +37,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable16.py # Test that rollback to stable removes updates present on disk for column store. class test_rollback_to_stable16(wttest.WiredTigerTestCase): @@ -81,7 +78,7 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): cursor[i] = value + str(i) else: cursor[i] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(timestamp)) cursor.close() def check(self, check_value, uri, nrows, start_row, read_ts): @@ -89,7 +86,7 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): if read_ts == 0: session.begin_transaction() else: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 @@ -128,14 +125,14 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): self.session.create(uri, create_params) # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) for i in range(len(values)): self.insert_update_data(uri, values[i], start_row, nrows, ts[i]) start_row += nrows - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) if not self.in_memory: # Checkpoint to ensure that all the updates are flushed to disk. diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py index 829e5d13680..1697cd50c8b 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py @@ -33,9 +33,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable17.py # Test that rollback to stable handles updates present on history store and data store for variable # length column store. @@ -67,12 +64,12 @@ class test_rollback_to_stable17(wttest.WiredTigerTestCase): for i in range(start_row, end_row): self.session.begin_transaction() cursor[i] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(timestamp)) cursor.close() def check(self, check_value, uri, nrows, read_ts): session = self.session - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 @@ -96,15 +93,15 @@ class test_rollback_to_stable17(wttest.WiredTigerTestCase): self.session.create(uri, create_params) # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) # Make a series of updates for the same keys with different values at different timestamps. for i in range(len(values)): self.insert_update_data(uri, values[i], start_row, nrows, ts[i]) # Set the stable timestamp to 5. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) if not self.in_memory: # Checkpoint to ensure that all the updates are flushed to disk. diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py index a9725841b8b..d34e4ffb08b 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py @@ -38,9 +38,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable18.py # Test the rollback to stable shouldn't skip any pages that don't have aggregated time window. class test_rollback_to_stable18(test_rollback_to_stable_base): @@ -65,10 +62,6 @@ class test_rollback_to_stable18(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable18" ds = SimpleDataSet( @@ -76,8 +69,8 @@ class test_rollback_to_stable18(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 @@ -102,11 +95,11 @@ class test_rollback_to_stable18(test_rollback_to_stable_base): # Pin stable and oldest to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(30) + - ',stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(30) + + ',stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(20) + - ',stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(20) + + ',stable_timestamp=' + self.timestamp_str(20)) # Perform rollback to stable. self.conn.rollback_to_stable() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py index dd98263ae41..3f10b186b47 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py @@ -33,9 +33,6 @@ from wiredtiger import stat, WT_NOTFOUND from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable19.py # Test that rollback to stable aborts both insert and remove updates from a single prepared transaction class test_rollback_to_stable19(test_rollback_to_stable_base): @@ -52,8 +49,8 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): ] restart_options = [ - ('shutdown', dict(crash='false')), - ('crash', dict(crash='true')), + ('shutdown', dict(crash=False)), + ('crash', dict(crash=True)), ] scenarios = make_scenarios(in_memory_values, key_format_values, restart_options) @@ -69,10 +66,6 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): def test_rollback_to_stable_no_history(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable19" ds = SimpleDataSet( @@ -80,8 +73,8 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = "aaaaa" * 100 @@ -94,7 +87,7 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): cursor.set_key(i) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") @@ -116,7 +109,7 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): cursor2.close() # Pin stable timestamp to 20. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) if not self.in_memory: self.session.checkpoint() @@ -137,17 +130,22 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): stat_cursor = self.session.open_cursor('statistics:', None, None) upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] - self.assertGreater(upd_aborted, 0) - self.assertGreater(keys_removed, 0) + + # After restart (not crash) the stats for the aborted updates will be 0, as the updates + # will be aborted during shutdown, and on startup there will be no updates to be aborted. + # This is similar case with keys removed. + if not self.in_memory and not self.crash: + self.assertEqual(upd_aborted, 0) + self.assertEqual(keys_removed, 0) + else: + self.assertGreater(upd_aborted, 0) + self.assertGreater(keys_removed, 0) + stat_cursor.close() def test_rollback_to_stable_with_history(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable19" ds = SimpleDataSet( @@ -155,8 +153,8 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = "aaaaa" * 100 valueb = "bbbbb" * 100 @@ -176,7 +174,7 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): cursor.set_key(i) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") @@ -198,7 +196,7 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): cursor2.close() # Pin stable timestamp to 40. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) if not self.in_memory: self.session.checkpoint() @@ -220,6 +218,15 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): stat_cursor = self.session.open_cursor('statistics:', None, None) upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] - self.assertGreater(upd_aborted, 0) + + # After restart (not crash) the stats for the aborted updates and history store removed will be 0, + # as the updates aborted and history store removed will occur during shutdown, and on startup there + # will be no updates to be removed. if not self.in_memory: - self.assertGreater(hs_removed, 0) + if self.crash: + self.assertGreater(hs_removed, 0) + else: + self.assertEqual(hs_removed, 0) + self.assertEqual(upd_aborted, 0) + else: + self.assertGreater(upd_aborted, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py index a2c301ec362..8a2f44c5a36 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py @@ -30,17 +30,22 @@ import time from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet +from wtscenario import make_scenarios from wiredtiger import stat from helper import simulate_crash_restart from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # Test that rollback to stable does not open any dhandles that don't have unstable updates. class test_rollback_to_stable20(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def conn_config(self): config = 'cache_size=50MB,statistics=(all)' return config @@ -51,12 +56,12 @@ class test_rollback_to_stable20(test_rollback_to_stable_base): create_params = 'key_format=i,value_format=S' uri = "table:rollback_to_stable20" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable timestamp to 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) valuea = "aaaaa" * 100 @@ -65,7 +70,7 @@ class test_rollback_to_stable20(test_rollback_to_stable_base): self.session.create(uri, create_params) self.large_updates(uri, valuea, ds, nrows, 0, 10) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py index 4e44b9c2d97..3a348d00dc0 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py @@ -37,9 +37,6 @@ from helper import simulate_crash_restart from wtdataset import SimpleDataSet from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable21.py # Test rollback to stable when an out of order prepared transaction is written to disk class test_rollback_to_stable21(test_rollback_to_stable_base): @@ -57,10 +54,6 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable21" ds = SimpleDataSet( @@ -68,8 +61,8 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' * 400 valueb = 'b' * 400 @@ -79,7 +72,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): for i in range(1, nrows + 1): cursor[i] = valuea - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) self.session.begin_transaction() for i in range(1, nrows + 1): @@ -87,7 +80,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): cursor.reset() cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) s = self.conn.open_session() s.begin_transaction('ignore_prepare = true') @@ -101,7 +94,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): evict_cursor.reset() s.rollback_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) s.checkpoint() # Rollback the prepared transaction @@ -123,10 +116,6 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): def test_rollback_to_stable_with_different_tombstone(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable21" ds = SimpleDataSet( @@ -134,8 +123,8 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' * 400 valueb = 'b' * 400 @@ -144,13 +133,13 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): self.session.begin_transaction() for i in range(1, nrows + 1): cursor[i] = valuea - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) self.session.begin_transaction() for i in range(1, nrows + 1): cursor.set_key(i) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(40)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) self.session.begin_transaction() for i in range(1, nrows + 1): @@ -158,10 +147,10 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): cursor.reset() cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) s = self.conn.open_session() - s.begin_transaction('ignore_prepare = true, read_timestamp = ' + timestamp_str(30)) + s.begin_transaction('ignore_prepare = true, read_timestamp = ' + self.timestamp_str(30)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = s.open_cursor(uri, None, "debug=(release_evict)") @@ -172,7 +161,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): evict_cursor.reset() s.rollback_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) s.checkpoint() # Rollback the prepared transaction @@ -197,10 +186,6 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): def test_rollback_to_stable_with_same_tombstone(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable21" ds = SimpleDataSet( @@ -208,8 +193,8 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' * 400 valueb = 'b' * 400 @@ -221,7 +206,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): cursor.set_key(i) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) self.session.begin_transaction() for i in range(1, nrows + 1): @@ -229,7 +214,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): cursor.reset() cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) s = self.conn.open_session() s.begin_transaction('ignore_prepare = true') @@ -242,7 +227,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): evict_cursor.reset() s.rollback_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) s.checkpoint() # Rollback the prepared transaction diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py index ac123fe9ccd..f6ef52cc388 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py @@ -29,16 +29,13 @@ from test_rollback_to_stable01 import test_rollback_to_stable_base from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable22 # Test history store operations conflicting with rollback to stable. We're trying to trigger a # history store eviction concurrently to a rollback to stable call. We'll do this by limiting # the cache size to 100MB and performing 100MB worth of inserts while periodically calling rollback # to stable. class test_rollback_to_stable22(test_rollback_to_stable_base): - conn_config = 'cache_size=100MB,statistics=(fast),statistics_log=(wait=1,json)' + conn_config = 'cache_size=100MB' session_config = 'isolation=snapshot' prepare = False @@ -82,5 +79,5 @@ class test_rollback_to_stable22(test_rollback_to_stable_base): if i % 100 == 0: # Put the timestamp backwards so we can rollback the updates we just did. stable_ts = (i - 1) * 10 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(stable_ts)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(stable_ts)) self.conn.rollback_to_stable() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable23.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable23.py new file mode 100644 index 00000000000..e4ff9e4dfb4 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable23.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from helper import simulate_crash_restart +from test_rollback_to_stable01 import test_rollback_to_stable_base +from wiredtiger import stat +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +def mod_val(value, char, location, nbytes=1): + return value[0:location] + char + value[location+nbytes:] + +# test_rollback_to_stable23.py +# Test to verify that search operation uses proper base update while returning modifies from +# the history store after the on-disk update is removed by the rollback to stable. +class test_rollback_to_stable23(test_rollback_to_stable_base): + session_config = 'isolation=snapshot' + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + prepare_values = [ + ('no_prepare', dict(prepare=False)), + ('prepare', dict(prepare=True)) + ] + + scenarios = make_scenarios(key_format_values, prepare_values) + + def conn_config(self): + config = 'cache_size=50MB,statistics=(all),statistics_log=(json,on_close,wait=1)' + return config + + def check_with_set_key(self, ds, check_value, uri, nrows, read_ts): + cursor = self.session.open_cursor(uri) + self.session.begin_transaction("read_timestamp = " + self.timestamp_str(read_ts)) + for i in range(1, nrows + 1): + cursor.set_key(ds.key(i)) + self.assertEquals(cursor.search(), 0) + self.assertEquals(cursor.get_value(), check_value) + cursor.close() + self.session.commit_transaction() + + def test_rollback_to_stable(self): + nrows = 1000 + + # Create a table without logging. + uri = "table:rollback_to_stable23" + ds = SimpleDataSet( + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + ds.populate() + + # Pin oldest and stable to timestamp 10. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) + + value_a = "aaaaa" * 100 + + value_modQ = mod_val(value_a, 'Q', 0) + value_modR = mod_val(value_modQ, 'R', 1) + value_modS = mod_val(value_modR, 'S', 2) + value_modT = mod_val(value_modS, 'T', 3) + + # Perform a combination of modifies and updates. + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + self.large_modifies(uri, 'Q', ds, 0, 1, nrows, self.prepare, 30) + self.large_modifies(uri, 'R', ds, 1, 1, nrows, self.prepare, 40) + self.large_modifies(uri, 'S', ds, 2, 1, nrows, self.prepare, 50) + self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 60) + + # Verify data is visible and correct. + self.check(value_a, uri, nrows, 20) + self.check(value_modQ, uri, nrows, 30) + self.check(value_modR, uri, nrows, 40) + self.check(value_modS, uri, nrows, 50) + self.check(value_modT, uri, nrows, 60) + + # Pin stable to timestamp 60 if prepare otherwise 50. + if self.prepare: + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) + else: + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) + + # Checkpoint the database. + self.session.checkpoint() + + # Simulate a server crash and restart. + simulate_crash_restart(self, ".", "RESTART") + + # Check that the correct data is seen at and after the stable timestamp. + self.check_with_set_key(ds, value_a, uri, nrows, 20) + self.check_with_set_key(ds, value_modQ, uri, nrows, 30) + self.check_with_set_key(ds, value_modR, uri, nrows, 40) + self.check_with_set_key(ds, value_modS, uri, nrows, 50) + + stat_cursor = self.session.open_cursor('statistics:', None, None) + hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] + hs_restore_updates = stat_cursor[stat.conn.txn_rts_hs_restore_updates][2] + upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] + stat_cursor.close() + + self.assertEqual(hs_restore_updates, nrows) + if self.prepare: + self.assertGreaterEqual(upd_aborted, 0) + else: + self.assertEqual(upd_aborted, 0) + self.assertGreaterEqual(hs_removed, nrows) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_search_near01.py b/src/third_party/wiredtiger/test/suite/test_search_near01.py index 80f50d2f97a..412cbd894f0 100755 --- a/src/third_party/wiredtiger/test/suite/test_search_near01.py +++ b/src/third_party/wiredtiger/test/suite/test_search_near01.py @@ -30,9 +30,6 @@ import time, wiredtiger, wttest, unittest from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_search_near01.py # Test various prefix search near scenarios. class test_search_near01(wttest.WiredTigerTestCase): diff --git a/src/third_party/wiredtiger/test/suite/test_stat09.py b/src/third_party/wiredtiger/test/suite/test_stat09.py index 9f775f7f949..5345449581e 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat09.py +++ b/src/third_party/wiredtiger/test/suite/test_stat09.py @@ -31,10 +31,6 @@ import wiredtiger, wttest # test_stat09.py # Check oldest active read timestamp statistic - -def timestamp_str(t): - return '%x' % t - class test_stat09(wttest.WiredTigerTestCase): tablename = 'test_stat09' uri = 'table:' + tablename @@ -88,7 +84,7 @@ class test_stat09(wttest.WiredTigerTestCase): for k in keys: self.session.begin_transaction() c[k] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # Create a cursor on statistics that we can use repeatedly allstat_cursor = self.session.open_cursor('statistics:', None, None) @@ -98,15 +94,15 @@ class test_stat09(wttest.WiredTigerTestCase): # Introduce multiple transactions with varying read_timestamp s1 = self.conn.open_session() - s1.begin_transaction('read_timestamp=' + timestamp_str(10)) + s1.begin_transaction('read_timestamp=' + self.timestamp_str(10)) s2 = self.conn.open_session() - s2.begin_transaction('read_timestamp=' + timestamp_str(20)) + s2.begin_transaction('read_timestamp=' + self.timestamp_str(20)) s3 = self.conn.open_session() - s3.begin_transaction('read_timestamp=' + timestamp_str(30)) + s3.begin_transaction('read_timestamp=' + self.timestamp_str(30)) s4 = self.conn.open_session() - s4.begin_transaction('read_timestamp=' + timestamp_str(40)) + s4.begin_transaction('read_timestamp=' + self.timestamp_str(40)) s5 = self.conn.open_session() - s5.begin_transaction('read_timestamp=' + timestamp_str(50)) + s5.begin_transaction('read_timestamp=' + self.timestamp_str(50)) # Check oldest reader self.check_stat_oldest_read(allstat_cursor, 10, commit_range) @@ -117,18 +113,18 @@ class test_stat09(wttest.WiredTigerTestCase): # Set and advance the oldest timestamp, it should be ignored for # determining the oldest active read. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) self.check_stat_oldest_read(allstat_cursor, 20, commit_range) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(30)) self.check_stat_oldest_read(allstat_cursor, 20, commit_range) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(150)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(150)) self.check_stat_oldest_read(allstat_cursor, 20, commit_range) # Move the commit timestamp and check again commit_range = 200 - s2.commit_transaction('commit_timestamp=' + timestamp_str(commit_range)) + s2.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_range)) self.check_stat_oldest_read(allstat_cursor, 30, commit_range) # Close all the readers and check the oldest reader, it should be back to 0 diff --git a/src/third_party/wiredtiger/test/suite/test_tiered03.py b/src/third_party/wiredtiger/test/suite/test_tiered03.py index 61e12c08e28..f605342fa5a 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered03.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered03.py @@ -56,7 +56,6 @@ class test_tiered03(wttest.WiredTigerTestCase): if not os.path.exists(self.bucket): os.mkdir(self.bucket) return \ - 'statistics=(all),' + \ 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ 'bucket_prefix=%s,' % self.bucket_prefix + \ diff --git a/src/third_party/wiredtiger/test/suite/test_tiered05.py b/src/third_party/wiredtiger/test/suite/test_tiered05.py index 2af464ba64b..1d199bad74b 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered05.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered05.py @@ -50,7 +50,6 @@ class test_tiered05(wttest.WiredTigerTestCase): def conn_config(self): os.mkdir(self.bucket) return \ - 'statistics=(fast),' + \ 'tiered_manager=(wait=%d),' % self.wait + \ 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp01.py b/src/third_party/wiredtiger/test/suite/test_timestamp01.py index 433347bcec4..f8c768eb2f8 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp01.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp01.py @@ -33,9 +33,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_timestamp01(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' @@ -43,28 +40,28 @@ class test_timestamp01(wttest.WiredTigerTestCase, suite_subprocess): # Cannot set a timestamp on a non-running transaction self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1 << 5000)), + 'commit_timestamp=' + self.timestamp_str(1 << 5000)), '/only permitted in a running/') # Zero is not permitted self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(0)), + 'commit_timestamp=' + self.timestamp_str(0)), '/zero not permitted/') # Too big is also not permitted self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(1 << 5000)), + 'commit_timestamp=' + self.timestamp_str(1 << 5000)), '/too long/') # Anything other than lower case hexadecimal characters is not permitted self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(-1)), + 'commit_timestamp=' + self.timestamp_str(-1)), '/Failed to parse commit timestamp/') self.session.begin_transaction() @@ -88,13 +85,13 @@ class test_timestamp01(wttest.WiredTigerTestCase, suite_subprocess): # One is okay, as is upper case hex and 2**64 - 1 self.session.begin_transaction() self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(1)) + 'commit_timestamp=' + self.timestamp_str(1)) self.session.begin_transaction() self.session.commit_transaction( 'commit_timestamp=0A78F') self.session.begin_transaction() self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(1 << 64 - 1)) + 'commit_timestamp=' + self.timestamp_str(1 << 64 - 1)) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp02.py b/src/third_party/wiredtiger/test/suite/test_timestamp02.py index b77f409a182..c0f45f82dc5 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp02.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp02.py @@ -35,9 +35,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp02(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp02' uri = 'table:' + tablename @@ -79,58 +76,58 @@ class test_timestamp02(wttest.WiredTigerTestCase, suite_subprocess): for k in keys: self.session.begin_transaction() c[k] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # Don't set a stable timestamp yet. Make sure we can read with # a timestamp before the stable timestamp has been set. # Now check that we see the expected state when reading at each # timestamp for i, t in enumerate(orig_keys): - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), dict((k, 1) for k in orig_keys[:i+1])) # Everything up to and including timestamp 100 has been committed. - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(100)) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(100)) # Bump the oldest timestamp, we're not going back... - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100)) # Update them and retry. random.shuffle(keys) for k in keys: self.session.begin_transaction() c[k] = 2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k + 100)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k + 100)) # Everything up to and including timestamp 200 has been committed. - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(200)) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(200)) # Test that we can manually move the commit timestamp back - self.conn.set_timestamp('commit_timestamp=' + timestamp_str(150)) - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(150)) - self.conn.set_timestamp('commit_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('commit_timestamp=' + self.timestamp_str(150)) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(150)) + self.conn.set_timestamp('commit_timestamp=' + self.timestamp_str(200)) # Now the stable timestamp before we read. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(200)) for i, t in enumerate(orig_keys): - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), dict((k, (2 if j <= i else 1)) for j, k in enumerate(orig_keys))) # Bump the oldest timestamp, we're not going back... - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200)) # Remove them and retry random.shuffle(keys) for k in keys: self.session.begin_transaction() del c[k] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k + 200)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k + 200)) # We have to continue to advance the stable timestamp before reading. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(300)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(300)) for i, t in enumerate(orig_keys): - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 200), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 200), dict((k, 2) for k in orig_keys[i+1:])) def test_read_your_writes(self): diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp03.py b/src/third_party/wiredtiger/test/suite/test_timestamp03.py index 862ed241b17..a18d3fd86b0 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp03.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp03.py @@ -36,9 +36,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): table_ts_log = 'ts03_ts_logged' table_ts_nolog = 'ts03_ts_nologged' @@ -46,11 +43,13 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): table_nots_nolog = 'ts03_nots_nologged' types = [ - ('file', dict(uri='file:', use_cg=False, use_index=False)), - ('lsm', dict(uri='lsm:', use_cg=False, use_index=False)), - ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), - ('table-index', dict(uri='table:', use_cg=False, use_index=True)), - ('table-simple', dict(uri='table:', use_cg=False, use_index=False)), + ('file-row', dict(uri='file:', key_format='i', use_cg=False, use_index=False)), + ('file-col', dict(uri='file:', key_format='r', use_cg=False, use_index=False)), + ('lsm', dict(uri='lsm:', key_format='i', use_cg=False, use_index=False)), + ('table-row', dict(uri='table:', key_format='i', use_cg=False, use_index=False)), + ('table-row-index', dict(uri='table:', key_format='i', use_cg=False, use_index=True)), + ('table-col', dict(uri='table:', key_format='r', use_cg=False, use_index=False)), + ('table-col-cg', dict(uri='table:', key_format='r', use_cg=True, use_index=False)), ] ckpt = [ @@ -166,13 +165,14 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # 3. Table is logged and does not use timestamps. # 4. Table is not logged and does not use timestamps. # - self.session.create(uri_ts_log, 'key_format=i,value_format=S') + format = 'key_format={},value_format=S'.format(self.key_format) + self.session.create(uri_ts_log, format) cur_ts_log = self.session.open_cursor(uri_ts_log) - self.session.create(uri_ts_nolog, 'key_format=i,value_format=S,log=(enabled=false)') + self.session.create(uri_ts_nolog, format + ',log=(enabled=false)') cur_ts_nolog = self.session.open_cursor(uri_ts_nolog) - self.session.create(uri_nots_log, 'key_format=i,value_format=S') + self.session.create(uri_nots_log, format) cur_nots_log = self.session.open_cursor(uri_nots_log) - self.session.create(uri_nots_nolog, 'key_format=i,value_format=S, log=(enabled=false)') + self.session.create(uri_nots_nolog, format + ',log=(enabled=false)') cur_nots_nolog = self.session.open_cursor(uri_nots_nolog) # Insert keys 1..100 each with timestamp=key, in some order @@ -187,7 +187,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = self.value cur_ts_nolog[k] = self.value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # Scenario: 1 # Check that we see all the inserted values as per transaction @@ -207,19 +207,19 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): for i, t in enumerate(orig_keys): # Tables using the timestamps should see the values as per the # given read_timestamp - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), self.table_ts_log, dict((k, self.value) for k in orig_keys[:i+1])) - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), self.table_ts_nolog, dict((k, self.value) for k in orig_keys[:i+1])) # Tables not using the timestamps should see all the values. - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), self.table_nots_log, dict((k, self.value) for k in orig_keys)) - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), self.table_nots_nolog, dict((k, self.value) for k in orig_keys)) # Bump the oldest_timestamp, we're not going back... - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(100)) - old_ts = timestamp_str(100) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(100)) + old_ts = self.timestamp_str(100) self.conn.set_timestamp('oldest_timestamp=' + old_ts) self.conn.set_timestamp('stable_timestamp=' + old_ts) @@ -248,7 +248,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = self.value2 cur_ts_nolog[k] = self.value2 - ts = timestamp_str(k + 100) + ts = self.timestamp_str(k + 100) self.session.commit_transaction('commit_timestamp=' + ts) count += 1 @@ -270,7 +270,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # This scenario is same as earlier one with read_timestamp earlier than # oldest_timestamp and using the option of rounding read_timestamp to # the oldest_timestamp - earlier_ts = timestamp_str(90) + earlier_ts = self.timestamp_str(90) self.check(self.session, 'read_timestamp=' + earlier_ts +',roundup_timestamps=(read=true)', self.table_ts_log, dict((k, self.value) for k in orig_keys)) @@ -294,15 +294,15 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): expected_dict[i+1] = self.value2 # Tables using the timestamps should see the updated values as per # the given read_timestamp - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), self.table_ts_log, expected_dict) - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), self.table_ts_nolog, expected_dict) # Tables not using the timestamps should see all the data values as # updated values (i.e. value2). - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), self.table_nots_log, dict((k, self.value2) for k in orig_keys)) - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), self.table_nots_nolog, dict((k, self.value2) for k in orig_keys)) # Take a checkpoint using the given configuration. Then verify @@ -359,7 +359,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # Update the stable_timestamp to the latest, but not the # oldest_timestamp and make sure we can see the data. Once the # stable_timestamp is moved we should see all keys with value2. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100+nkeys)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100+nkeys)) self.ckpt_backup(self.value2, nkeys, nkeys, nkeys, nkeys) # If we're not using the log we're done. @@ -378,7 +378,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = self.value3 cur_ts_nolog[k] = self.value3 - ts = timestamp_str(k + 200) + ts = self.timestamp_str(k + 200) self.session.commit_transaction('commit_timestamp=' + ts) count += 1 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp04.py b/src/third_party/wiredtiger/test/suite/test_timestamp04.py index 9aff0351e2d..e2aa7609fa7 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp04.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp04.py @@ -35,9 +35,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): table_ts_log = 'table:ts04_ts_logged' table_ts_nolog = 'table:ts04_ts_nologged' @@ -56,12 +53,13 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Minimum cache_size requirement of lsm is 31MB. types = [ - # The commented columnar tests needs to be enabled once rollback to stable for columnar is fixed in (WT-5548). - # ('col_fix', dict(empty=1, cacheSize='cache_size=20MB', extra_config=',key_format=r,value_format=8t')), - # ('col_var', dict(empty=0, cacheSize='cache_size=20MB', extra_config=',key_format=r')), + # FLCS does not yet work in a timestamp world. + #('col_fix', dict(empty=1, \ + # cacheSize='cache_size=20MB', extra_config=',key_format=r,value_format=8t')), ('lsm', dict(empty=0, cacheSize='cache_size=31MB', extra_config=',type=lsm')), ('row', dict(empty=0, cacheSize='cache_size=20MB', extra_config='',)), ('row-smallcache', dict(empty=0, cacheSize='cache_size=2MB', extra_config='',)), + ('var', dict(empty=0, cacheSize='cache_size=20MB', extra_config=',key_format=r')), ] scenarios = make_scenarios(conncfg, types) @@ -146,10 +144,10 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = 1 cur_ts_nolog[k] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # Setup an oldest timestamp to ensure state remains in cache. if k == 1: - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cur_ts_log.close() cur_ts_nolog.close() cur_nots_log.close() @@ -157,7 +155,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 1 # Check that we see all the inserted values(i.e 1) in all tables - latest_ts = timestamp_str(key_range) + latest_ts = self.timestamp_str(key_range) self.check(self.session, 'read_timestamp=' + latest_ts, self.table_nots_log, dict((k, 1) for k in keys[:])) self.check(self.session, 'read_timestamp=' + latest_ts, @@ -169,7 +167,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 2 # Roll back half timestamps. - stable_ts = timestamp_str(key_range // 2) + stable_ts = self.timestamp_str(key_range // 2) self.conn.set_timestamp('stable_timestamp=' + stable_ts) # We're about to test rollback-to-stable which requires a checkpoint to which we can roll back. @@ -231,7 +229,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = 2 cur_ts_nolog[k] = 2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k + key_range)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k + key_range)) cur_ts_log.close() cur_ts_nolog.close() cur_nots_log.close() @@ -239,7 +237,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 3 # Check that we see all values updated (i.e 2) in all tables. - latest_ts = timestamp_str(2 * key_range) + latest_ts = self.timestamp_str(2 * key_range) self.check(self.session, 'read_timestamp=' + latest_ts, self.table_nots_log, dict((k, 2) for k in keys[:])) self.check(self.session, 'read_timestamp=' + latest_ts, @@ -253,7 +251,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Advance the stable_timestamp by a quarter range and rollback. # Three-fourths of the later timestamps will be rolled back. rolled_range = key_range + key_range // 4 - stable_ts = timestamp_str(rolled_range) + stable_ts = self.timestamp_str(rolled_range) self.conn.set_timestamp('stable_timestamp=' + stable_ts) self.conn.rollback_to_stable() stat_cursor = self.session.open_cursor('statistics:', None, None) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp05.py b/src/third_party/wiredtiger/test/suite/test_timestamp05.py index e7be33de000..852823231c7 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp05.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp05.py @@ -35,13 +35,16 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:ts05' session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def test_create(self): s = self.session conn = self.conn @@ -51,8 +54,8 @@ class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): # Commit at 100 s.begin_transaction() - s.create(self.uri, 'key_format=i,value_format=S') - s.commit_transaction('commit_timestamp=' + timestamp_str(100)) + s.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + s.commit_transaction('commit_timestamp=' + self.timestamp_str(100)) # Make sure the tree is dirty c = s.open_cursor(self.uri) @@ -65,7 +68,7 @@ class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): s = self.session conn = self.conn - s.create(self.uri, 'key_format=i,value_format=S') + s.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) c = s.open_cursor(self.uri, None, 'bulk') # Insert keys 1..100 each with timestamp=key, in some order @@ -79,7 +82,7 @@ class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): # Commit at 100 s.begin_transaction() c.close() - s.commit_transaction('commit_timestamp=' + timestamp_str(100)) + s.commit_transaction('commit_timestamp=' + self.timestamp_str(100)) # Make sure the tree is dirty c = s.open_cursor(self.uri) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp06.py b/src/third_party/wiredtiger/test/suite/test_timestamp06.py index d958a3ba3ed..27e32309fdc 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp06.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp06.py @@ -36,15 +36,13 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): table_ts_log = 'table:ts06_ts_logged' table_ts_nolog = 'table:ts06_ts_nologged' types = [ - ('col_fix', dict(empty=1, extra_config=',key_format=r,value_format=8t')), + # FLCS does not yet work in a timestamp world. + #('col_fix', dict(empty=1, extra_config=',key_format=r,value_format=8t')), ('col_var', dict(empty=0, extra_config=',key_format=r')), ('lsm', dict(empty=0, extra_config=',type=lsm')), ('row', dict(empty=0, extra_config='',)), @@ -125,9 +123,9 @@ class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): # Open two timestamp tables: # 1. Table is logged and uses timestamps. # 2. Table is not logged and uses timestamps. - self.session.create(self.table_ts_log, 'key_format=i,value_format=i') + self.session.create(self.table_ts_log, 'key_format=i,value_format=i' + self.extra_config) cur_ts_log = self.session.open_cursor(self.table_ts_log) - self.session.create(self.table_ts_nolog, 'key_format=i,value_format=i,log=(enabled=false)') + self.session.create(self.table_ts_nolog, 'key_format=i,value_format=i,log=(enabled=false)' + self.extra_config) cur_ts_nolog = self.session.open_cursor(self.table_ts_nolog) # Insert keys 1..100 @@ -138,22 +136,22 @@ class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() # Make three updates with different timestamps. - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(1)) for k in keys: cur_ts_log[k] = 1 cur_ts_nolog[k] = 1 - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(101)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(101)) for k in keys: cur_ts_log[k] = 2 cur_ts_nolog[k] = 2 - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(201)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(201)) for k in keys: cur_ts_log[k] = 3 cur_ts_nolog[k] = 3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(301)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(301)) cur_ts_log.close() cur_ts_nolog.close() @@ -170,9 +168,9 @@ class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): # Check that we see the values till correctly from checkpointed data # files in case of multistep transactions. # Set oldest and stable timestamps - old_ts = timestamp_str(100) + old_ts = self.timestamp_str(100) # Set the stable timestamp such that last update is beyond it. - stable_ts = timestamp_str(200) + stable_ts = self.timestamp_str(200) self.conn.set_timestamp('oldest_timestamp=' + old_ts) self.conn.set_timestamp('stable_timestamp=' + stable_ts) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp07.py b/src/third_party/wiredtiger/test/suite/test_timestamp07.py index e84f9ea5f34..5e25afe1675 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp07.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp07.py @@ -36,14 +36,16 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'ts07_ts_nologged' tablename2 = 'ts07_nots_logged' tablename3 = 'ts07_ts_logged' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + types = [ ('file', dict(uri='file:', use_cg=False, use_index=False)), ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), @@ -61,7 +63,7 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): ('1000keys', dict(nkeys=1000)), ] - scenarios = make_scenarios(types, conncfg, nkeys) + scenarios = make_scenarios(key_format_values, types, conncfg, nkeys) # Binary values. value = u'\u0001\u0002abcd\u0007\u0004' @@ -183,11 +185,11 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): # 2. Table is logged and does not use timestamps. # 3. Table is logged and uses timestamps. # - self.session.create(uri, 'key_format=i,value_format=S,log=(enabled=false)') + self.session.create(uri, 'key_format={},value_format=S,log=(enabled=false)'.format(self.key_format)) c = self.session.open_cursor(uri) - self.session.create(uri2, 'key_format=i,value_format=S') + self.session.create(uri2, 'key_format={},value_format=S'.format(self.key_format)) c2 = self.session.open_cursor(uri2) - self.session.create(uri3, 'key_format=i,value_format=S') + self.session.create(uri3, 'key_format={},value_format=S'.format(self.key_format)) c3 = self.session.open_cursor(uri3) # print "tables created" @@ -201,23 +203,23 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[k] = self.value c3[k] = self.value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # print "value inserted in all tables, reading..." # Now check that we see the expected state when reading at each # timestamp. for k in orig_keys: - self.check(self.session, 'read_timestamp=' + timestamp_str(k), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(k), k, self.value) - self.check(self.session, 'read_timestamp=' + timestamp_str(k), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(k), k + 1, None) # print "all values read, updating timestamps" # Bump the oldest timestamp, we're not going back... - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(self.nkeys)) - self.oldts = self.stablets = timestamp_str(self.nkeys) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(self.nkeys)) + self.oldts = self.stablets = self.timestamp_str(self.nkeys) self.conn.set_timestamp('oldest_timestamp=' + self.oldts) self.conn.set_timestamp('stable_timestamp=' + self.stablets) # print "Oldest " + self.oldts @@ -237,7 +239,7 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[k] = self.value2 c3[k] = self.value2 - ts = timestamp_str(k + self.nkeys) + ts = self.timestamp_str(k + self.nkeys) self.session.commit_transaction('commit_timestamp=' + ts) # print "Commit key " + str(k) + " ts " + ts count += 1 @@ -252,7 +254,7 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): # Update the stable timestamp to the latest, but not the oldest # timestamp and make sure we can see the data. Once the stable # timestamp is moved we should see all keys with value2. - self.stablets = timestamp_str(self.nkeys*2) + self.stablets = self.timestamp_str(self.nkeys*2) self.conn.set_timestamp('stable_timestamp=' + self.stablets) # print "check_stable 2" self.check_stable(self.value2, self.nkeys, self.nkeys, self.nkeys) @@ -274,7 +276,7 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[k] = self.value3 c3[k] = self.value3 - ts = timestamp_str(k + self.nkeys*2) + ts = self.timestamp_str(k + self.nkeys*2) self.session.commit_transaction('commit_timestamp=' + ts) # print "Commit key " + str(k) + " ts " + ts count += 1 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp09.py b/src/third_party/wiredtiger/test/suite/test_timestamp09.py index 91c6435e7d2..2188b919c43 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp09.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp09.py @@ -33,9 +33,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp09' uri = 'table:' + tablename @@ -48,7 +45,7 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): # Begin by adding some data. self.session.begin_transaction() self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(1)) + 'commit_timestamp=' + self.timestamp_str(1)) c[1] = 1 # In a single transaction it is illegal to set a commit timestamp @@ -56,21 +53,21 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): # Check both timestamp_transaction and commit_transaction APIs. self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(3)) + 'commit_timestamp=' + self.timestamp_str(3)) c[3] = 3 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)), + 'commit_timestamp=' + self.timestamp_str(2)), '/older than the first commit timestamp/') self.session.rollback_transaction() self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(4)) + 'commit_timestamp=' + self.timestamp_str(4)) c[4] = 4 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(3)), + 'commit_timestamp=' + self.timestamp_str(3)), '/older than the first commit timestamp/') # Commit timestamp >= Oldest timestamp @@ -78,13 +75,13 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[3] = 3 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(3)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(3)) + 'commit_timestamp=' + self.timestamp_str(3)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(3)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)), + 'commit_timestamp=' + self.timestamp_str(2)), '/less than the oldest timestamp/') self.session.rollback_transaction() @@ -92,48 +89,48 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): c[2] = 2 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(2)), + 'commit_timestamp=' + self.timestamp_str(2)), '/less than the oldest timestamp/') self.session.begin_transaction() c[4] = 4 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(4)) + 'commit_timestamp=' + self.timestamp_str(4)) # Oldest timestamp <= Stable timestamp and both oldest and stable # timestamp should proceed forward. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.conn.set_timestamp('oldest_timestamp=' + - timestamp_str(3) + ',stable_timestamp=' + timestamp_str(1)), + self.timestamp_str(3) + ',stable_timestamp=' + self.timestamp_str(1)), '/oldest timestamp \(0, 3\) must not be later than stable timestamp \(0, 1\)/') # Oldest timestamp is 3 at the moment, trying to set it to an earlier # timestamp is a no-op. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), timestamp_str(3)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), self.timestamp_str(3)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(3) + - ',stable_timestamp=' + timestamp_str(3)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(3) + + ',stable_timestamp=' + self.timestamp_str(3)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) # Stable timestamp is 5 at the moment, trying to set it to an earlier # timestamp is a no-op. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(4)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(4)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), self.timestamp_str(5)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.conn.set_timestamp('oldest_timestamp=' + - timestamp_str(6)), + self.timestamp_str(6)), '/oldest timestamp \(0, 6\) must not be later than stable timestamp \(0, 5\)/') # Commit timestamp >= Stable timestamp. # Check both timestamp_transaction and commit_transaction APIs. # Oldest and stable timestamp are set to 5 at the moment. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(6)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(6)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(5)), + 'commit_timestamp=' + self.timestamp_str(5)), '/less than the stable timestamp/') self.session.rollback_transaction() @@ -141,7 +138,7 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): c[5] = 5 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(5)), + 'commit_timestamp=' + self.timestamp_str(5)), '/less than the stable timestamp/') # When explicitly set, commit timestamp for a transaction can be earlier @@ -149,48 +146,48 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[6] = 6 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(6)) + 'commit_timestamp=' + self.timestamp_str(6)) self.session.begin_transaction() c[8] = 8 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(8)) + 'commit_timestamp=' + self.timestamp_str(8)) self.session.begin_transaction() c[7] = 7 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(7)) + 'commit_timestamp=' + self.timestamp_str(7)) # Read timestamp >= Oldest timestamp - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(7) + - ',stable_timestamp=' + timestamp_str(7)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(7) + + ',stable_timestamp=' + self.timestamp_str(7)) with self.expectedStdoutPattern('less than the oldest timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, - lambda: self.session.begin_transaction('read_timestamp=' + timestamp_str(6))) + lambda: self.session.begin_transaction('read_timestamp=' + self.timestamp_str(6))) # c[8] is not visible at read_timestamp < 8 - self.session.begin_transaction('read_timestamp=' + timestamp_str(7)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(7)) self.assertEqual(c[6], 6) self.assertEqual(c[7], 7) c.set_key(8) self.assertEqual(c.search(), wiredtiger.WT_NOTFOUND) self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(8)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(8)) self.assertEqual(c[6], 6) self.assertEqual(c[7], 7) self.assertEqual(c[8], 8) self.assertTimestampsEqual( - self.conn.query_timestamp('get=oldest_reader'), timestamp_str(8)) + self.conn.query_timestamp('get=oldest_reader'), self.timestamp_str(8)) self.session.commit_transaction() # We can move the oldest timestamp backwards with "force" self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(5) + ',force') + 'oldest_timestamp=' + self.timestamp_str(5) + ',force') with self.expectedStdoutPattern('less than the oldest timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, - lambda: self.session.begin_transaction('read_timestamp=' + timestamp_str(4))) - self.session.begin_transaction('read_timestamp=' + timestamp_str(6)) + lambda: self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4))) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(6)) self.assertTimestampsEqual( - self.conn.query_timestamp('get=oldest_reader'), timestamp_str(6)) + self.conn.query_timestamp('get=oldest_reader'), self.timestamp_str(6)) self.session.commit_transaction() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp10.py b/src/third_party/wiredtiger/test/suite/test_timestamp10.py index 207370755ab..4af2004c162 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp10.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp10.py @@ -34,9 +34,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): conn_config = 'config_base=false,create,log=(enabled)' session_config = 'isolation=snapshot' @@ -90,16 +87,16 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): curs[i] = i self.pr("i: " + str(i)) self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(i)) + 'commit_timestamp=' + self.timestamp_str(i)) # Set the oldest and stable timestamp a bit earlier than the data # we inserted. Take a checkpoint to the stable timestamp. self.pr("stable ts: " + str(ts)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(ts) + - ',stable_timestamp=' + timestamp_str(ts)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(ts) + + ',stable_timestamp=' + self.timestamp_str(ts)) # This forces a different checkpoint timestamp for each table. self.session.checkpoint() q = self.conn.query_timestamp('get=last_checkpoint') - self.assertTimestampsEqual(q, timestamp_str(ts)) + self.assertTimestampsEqual(q, self.timestamp_str(ts)) return ts def close_and_recover(self, expected_rec_ts): @@ -124,7 +121,7 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): self.open_conn() q = self.conn.query_timestamp('get=recovery') self.pr("query recovery ts: " + q) - self.assertTimestampsEqual(q, timestamp_str(expected_rec_ts)) + self.assertTimestampsEqual(q, self.timestamp_str(expected_rec_ts)) def test_timestamp_recovery(self): # Add some data and checkpoint at a stable timestamp. diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp11.py b/src/third_party/wiredtiger/test/suite/test_timestamp11.py index de5d51ec45e..c4b4210df43 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp11.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp11.py @@ -33,9 +33,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' @@ -51,7 +48,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)) + 'commit_timestamp=' + self.timestamp_str(2)) c['key'] = 'value2' c['key2'] = 'value2' self.session.commit_transaction() @@ -64,7 +61,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(5)) + 'commit_timestamp=' + self.timestamp_str(5)) c['key'] = 'value5' self.session.commit_transaction() c.close() @@ -81,7 +78,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # remain at the non-timestamped value. Also the non-timestamped value # stays regardless of rollbacks or reading at a timestamp. # - stable_ts = timestamp_str(2) + stable_ts = self.timestamp_str(2) self.conn.set_timestamp('stable_timestamp=' + stable_ts) self.session.checkpoint() self.conn.rollback_to_stable() @@ -106,7 +103,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(5)) + 'commit_timestamp=' + self.timestamp_str(5)) c['key2'] = 'value5' self.session.commit_transaction() c.close() @@ -140,7 +137,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # one at that timestamp and inserted without a timestamp. For the second # we inserted at timestamp 5 after the non-timestamped insert. c = self.session.open_cursor(uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) self.assertEquals(c['key'], 'valueNOTS') self.assertEquals(c['key2'], 'value5') self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp12.py b/src/third_party/wiredtiger/test/suite/test_timestamp12.py index e21ba16158f..e54ed923474 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp12.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp12.py @@ -33,9 +33,6 @@ import shutil, os, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp12(wttest.WiredTigerTestCase): conn_config = 'config_base=false,create,log=(enabled)' session_config = 'isolation=snapshot' @@ -85,10 +82,10 @@ class test_timestamp12(wttest.WiredTigerTestCase): c_op[i] = 1 c_coll[i] = 1 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(i)) + 'commit_timestamp=' + self.timestamp_str(i)) # Set the oldest and stable timestamp to the end. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(nentries-1) + - ',stable_timestamp=' + timestamp_str(nentries-1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(nentries-1) + + ',stable_timestamp=' + self.timestamp_str(nentries-1)) # Add more data but don't advance the stable timestamp. for i in second_range: @@ -97,7 +94,7 @@ class test_timestamp12(wttest.WiredTigerTestCase): c_coll[i] = 1 self.pr("i: " + str(i)) self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(i)) + 'commit_timestamp=' + self.timestamp_str(i)) # Close and reopen the connection. We cannot use reopen_conn because # we want to test the specific close configuration string. diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp14.py b/src/third_party/wiredtiger/test/suite/test_timestamp14.py index c7a0738fa74..8d6e753bf4d 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp14.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp14.py @@ -35,9 +35,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp14' uri = 'table:' + tablename diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp16.py b/src/third_party/wiredtiger/test/suite/test_timestamp16.py index 5a378eae312..f74557ad2c6 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp16.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp16.py @@ -36,9 +36,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp16(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp16' uri = 'table:' + tablename diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp17.py b/src/third_party/wiredtiger/test/suite/test_timestamp17.py index b282ee49b0a..3c2453b0aa2 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp17.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp17.py @@ -38,9 +38,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp17(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp17' uri = 'table:' + tablename diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp18.py b/src/third_party/wiredtiger/test/suite/test_timestamp18.py index 746551b1dec..c7f20dbd9a4 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp18.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp18.py @@ -38,9 +38,6 @@ import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp18(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' @@ -53,7 +50,7 @@ class test_timestamp18(wttest.WiredTigerTestCase): def test_ts_writes_with_non_ts_write(self): uri = 'table:test_timestamp18' self.session.create(uri, 'key_format=S,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) value1 = 'a' * 500 @@ -65,17 +62,17 @@ class test_timestamp18(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Add a non-timestamped delete. # Let's do every second key to ensure that we get the truncation right and don't @@ -91,7 +88,7 @@ class test_timestamp18(wttest.WiredTigerTestCase): self.session.checkpoint() for ts in range(2, 4): - self.session.begin_transaction('read_timestamp=' + timestamp_str(ts)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts)) for i in range(1, 10000): # The non-timestamped delete should cover all the previous writes and make them effectively # invisible. diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp19.py b/src/third_party/wiredtiger/test/suite/test_timestamp19.py index fa83d53827a..9f5b71df843 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp19.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp19.py @@ -31,11 +31,8 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - class test_timestamp19(wttest.WiredTigerTestCase): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' def updates(self, uri, value, ds, nrows, commit_ts): @@ -44,7 +41,7 @@ class test_timestamp19(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def test_timestamp(self): @@ -62,8 +59,8 @@ class test_timestamp19(wttest.WiredTigerTestCase): value_z = 'z' * 1000 # Set the oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ', stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ', stable_timestamp=' + self.timestamp_str(10)) # Insert values with varying timestamps. self.updates(uri, value_x, ds, nrows, 20) @@ -74,8 +71,8 @@ class test_timestamp19(wttest.WiredTigerTestCase): self.session.checkpoint('use_timestamp=true') # Move the oldest and stable timestamps to 40. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(40) + - ', stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(40) + + ', stable_timestamp=' + self.timestamp_str(40)) # Update values. self.updates(uri, value_z, ds, nrows, 50) @@ -91,20 +88,20 @@ class test_timestamp19(wttest.WiredTigerTestCase): self.session = self.setUpSessionOpen(self.conn) # The oldest timestamp on recovery is 40. Trying to set it earlier is a no-op. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), timestamp_str(40)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), self.timestamp_str(40)) # Trying to set an earlier stable timestamp is an error. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)), + lambda: self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)), '/oldest timestamp \(0, 40\) must not be later than stable timestamp \(0, 10\)/') - self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), timestamp_str(40)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), self.timestamp_str(40)) # Move the oldest and stable timestamps to 70. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(70) + - ', stable_timestamp=' + timestamp_str(70)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), timestamp_str(70)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), timestamp_str(70)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(70) + + ', stable_timestamp=' + self.timestamp_str(70)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), self.timestamp_str(70)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), self.timestamp_str(70)) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp20.py b/src/third_party/wiredtiger/test/suite/test_timestamp20.py index c2ff132cd0b..c4edcbfe592 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp20.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp20.py @@ -28,9 +28,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_timestamp20.py # Exercise fixing up of out-of-order updates in the history store. class test_timestamp20(wttest.WiredTigerTestCase): @@ -40,7 +37,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): def test_timestamp20_standard(self): uri = 'table:test_timestamp20' self.session.create(uri, 'key_format=S,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) value1 = 'a' * 500 @@ -52,33 +49,33 @@ class test_timestamp20(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) old_reader_session = self.conn.open_session() old_reader_cursor = old_reader_session.open_cursor(uri) - old_reader_session.begin_transaction('read_timestamp=' + timestamp_str(20)) + old_reader_session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) # Now put two updates out of order. 5 will go to the history store and will trigger a # correction to the existing contents. for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value4 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(25)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(25)) self.session.begin_transaction() cursor[str(i)] = value5 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(40)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) - self.session.begin_transaction('read_timestamp=' + timestamp_str(30)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) for i in range(1, 10000): self.assertEqual(cursor[str(i)], value4) self.session.rollback_transaction() @@ -94,7 +91,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): def test_timestamp20_modify(self): uri = 'table:test_timestamp20' self.session.create(uri, 'key_format=S,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) value1 = 'a' * 500 @@ -105,20 +102,20 @@ class test_timestamp20(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Now apply a series of modifies. for i in range(1, 10000): self.session.begin_transaction() cursor.set_key(str(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) for i in range(1, 10000): self.session.begin_transaction() cursor.set_key(str(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) # Open an old reader at this point. # @@ -126,7 +123,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): # has been squashed into a full update. old_reader_session = self.conn.open_session() old_reader_cursor = old_reader_session.open_cursor(uri) - old_reader_session.begin_transaction('read_timestamp=' + timestamp_str(20)) + old_reader_session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) # Now apply the last modify. # This will be the end of the chain of modifies. @@ -134,21 +131,21 @@ class test_timestamp20(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(str(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(40)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) # Now put two updates out of order. 5 will go to the history store and will trigger a # correction to the existing contents. for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(25)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(25)) self.session.begin_transaction() cursor[str(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(50)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) # Open up a new transaction and read at 30. # We shouldn't be able to see past 5 due to txnid visibility. - self.session.begin_transaction('read_timestamp=' + timestamp_str(30)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) for i in range(1, 10000): self.assertEqual(cursor[str(i)], value2) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp21.py b/src/third_party/wiredtiger/test/suite/test_timestamp21.py index a3b9ca03ac1..6d553a36246 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp21.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp21.py @@ -28,9 +28,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_timestamp21.py # Test read timestamp configuration that allows read timestamp to be older than oldest. class test_timestamp21(wttest.WiredTigerTestCase): @@ -40,58 +37,58 @@ class test_timestamp21(wttest.WiredTigerTestCase): uri = 'table:test_timestamp21' self.session.create(uri, 'key_format=i,value_format=i') session2 = self.setUpSessionOpen(self.conn) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) cursor2 = session2.open_cursor(uri) # Insert first value at timestamp 10. self.session.begin_transaction() cursor[1] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Begin a read transaction at timestamp 5. - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) # Move the oldest timestamp beyond the currently open transactions read timestamp. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(8)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(8)) # Begin a transaction with a read timestamp of 6 and read_before_oldest specified. self.assertEqual(session2.begin_transaction( - 'read_timestamp=' + timestamp_str(6) + ',read_before_oldest=true'), 0) + 'read_timestamp=' + self.timestamp_str(6) + ',read_before_oldest=true'), 0) session2.rollback_transaction() # Begin a transaction with a read timestamp of 6 and no additional config. with self.expectedStdoutPattern('less than the oldest timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: session2.begin_transaction( - 'read_timestamp=' + timestamp_str(6))) + 'read_timestamp=' + self.timestamp_str(6))) # Begin a transaction with the config specified but no read timestamp. session2.begin_transaction('read_before_oldest=true') # Set a read timestamp behind the oldest timestamp. - self.assertEqual(session2.timestamp_transaction('read_timestamp=' + timestamp_str(5)), 0) + self.assertEqual(session2.timestamp_transaction('read_timestamp=' + self.timestamp_str(5)), 0) session2.rollback_transaction() # Begin a transaction with a read timestamp of 5 and read_before_oldest specified. self.assertEqual(session2.begin_transaction( - 'read_timestamp=' + timestamp_str(5) + ',read_before_oldest=true'), 0) + 'read_timestamp=' + self.timestamp_str(5) + ',read_before_oldest=true'), 0) session2.rollback_transaction() # Begin a transaction with a read timestamp of 4 and read_before_oldest specified. We get a # different std out message in this scenario. with self.expectedStdoutPattern('less than the pinned timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: session2.begin_transaction( - 'read_timestamp=' + timestamp_str(4) + ',read_before_oldest=true')) + 'read_timestamp=' + self.timestamp_str(4) + ',read_before_oldest=true')) # Begin a transaction with a read timestamp of 6 and read_before_oldest off, this will have # the same behaviour as not specifying it. with self.expectedStdoutPattern('less than the oldest timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: session2.begin_transaction( - 'read_timestamp=' + timestamp_str(6) + ',read_before_oldest=false')) + 'read_timestamp=' + self.timestamp_str(6) + ',read_before_oldest=false')) # Expect an error when we use roundup timestamps alongside allow read timestamp before # oldest. self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: session2.begin_transaction( - 'read_timestamp=' + timestamp_str(6) + + 'read_timestamp=' + self.timestamp_str(6) + ',read_before_oldest=true,roundup_timestamps=(read)'), '/cannot specify roundup_timestamps.read and read_before_oldest/') diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp22.py b/src/third_party/wiredtiger/test/suite/test_timestamp22.py index d03e18d8d72..e02dbf99e09 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp22.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp22.py @@ -32,9 +32,6 @@ import wiredtiger, wttest, re, suite_random from wtdataset import SimpleDataSet from contextlib import contextmanager -def timestamp_str(t): - return '%x' % t - class test_timestamp22(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' @@ -124,7 +121,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): this_commit_ts = -1 if self.do_illegal(): # setting durable timestamp must be after prepare call - config += ',durable_timestamp=' + timestamp_str(self.gen_ts(commit_ts)) + config += ',durable_timestamp=' + self.timestamp_str(self.gen_ts(commit_ts)) ok = False # ODDITY: if we set the durable timestamp (which is illegal at this point), and set a @@ -143,7 +140,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): else: # It's possible this will succeed, we'll check below. this_commit_ts = self.gen_ts(commit_ts) - config += ',commit_timestamp=' + timestamp_str(this_commit_ts) + config += ',commit_timestamp=' + self.timestamp_str(this_commit_ts) if this_commit_ts >= 0: if this_commit_ts < running_commit_ts: @@ -161,7 +158,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): session = self.session needs_rollback = False prepare_config = None - commit_config = 'commit_timestamp=' + timestamp_str(commit_ts) + commit_config = 'commit_timestamp=' + self.timestamp_str(commit_ts) tstxn1_config = '' tstxn2_config = '' @@ -173,11 +170,11 @@ class test_timestamp22(wttest.WiredTigerTestCase): # Occasionally put a durable timestamp on a commit without a prepare, # that will be an error. if do_prepare or not ok_commit: - commit_config += ',durable_timestamp=' + timestamp_str(durable_ts) + commit_config += ',durable_timestamp=' + self.timestamp_str(durable_ts) cursor = session.open_cursor(self.uri) prepare_ts = self.gen_ts(commit_ts) - prepare_config = 'prepare_timestamp=' + timestamp_str(prepare_ts) - begin_config = '' if read_ts < 0 else 'read_timestamp=' + timestamp_str(read_ts) + prepare_config = 'prepare_timestamp=' + self.timestamp_str(prepare_ts) + begin_config = '' if read_ts < 0 else 'read_timestamp=' + self.timestamp_str(read_ts) # We might do timestamp_transaction calls either before/after inserting # values, or both. @@ -302,7 +299,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): for ts_name in ['oldest', 'stable', 'commit', 'durable']: val = eval(ts_name) if val >= 0: - configs.append(ts_name + '_timestamp=' + timestamp_str(val)) + configs.append(ts_name + '_timestamp=' + self.timestamp_str(val)) return ','.join(configs) # Determine whether we expect the set_timestamp to succeed. @@ -357,8 +354,8 @@ class test_timestamp22(wttest.WiredTigerTestCase): self.pr('updating stable: ' + str(stable)) # Make sure the state of global timestamps is what we think. - expect_query_oldest = timestamp_str(self.oldest_ts) - expect_query_stable = timestamp_str(self.stable_ts) + expect_query_oldest = self.timestamp_str(self.oldest_ts) + expect_query_stable = self.timestamp_str(self.stable_ts) query_oldest = self.conn.query_timestamp('get=oldest') query_stable = self.conn.query_timestamp('get=stable') diff --git a/src/third_party/wiredtiger/test/suite/test_truncate05.py b/src/third_party/wiredtiger/test/suite/test_truncate05.py index eb9e1ec922f..73f9f5444ed 100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate05.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate05.py @@ -28,9 +28,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_truncate05.py # Test various fast truncate visibility scenarios class test_truncate05(wttest.WiredTigerTestCase): @@ -49,7 +46,7 @@ class test_truncate05(wttest.WiredTigerTestCase): for i in range(1, 1000): self.session.begin_transaction() cursor[i] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Reopen the connection to force all content to disk. self.reopen_conn() @@ -59,17 +56,17 @@ class test_truncate05(wttest.WiredTigerTestCase): # Insert a single update at a later timestamp. self.session.begin_transaction() cursor[500] = value2 - self.assertEqual(self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)), 0) + self.assertEqual(self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)), 0) # Insert a bunch of other content to fill the database and evict the committed update. for i in range(1000, 20000): self.session.begin_transaction() cursor[i] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Start a transaction with an earlier read timestamp than the commit timestamp of the # previous update. - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) # Truncate from key 1 to 1000. start = self.session.open_cursor(uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_txn03.py b/src/third_party/wiredtiger/test/suite/test_txn03.py index fd365240301..151b98ca8e2 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn03.py +++ b/src/third_party/wiredtiger/test/suite/test_txn03.py @@ -37,13 +37,18 @@ class test_txn03(wttest.WiredTigerTestCase): tablename = 'test_txn03' uri1 = 'table:' + tablename + "_1" uri2 = 'table:' + tablename + "_2" - key_str = "TEST_KEY1" - data_str1 = "VAL" - data_str2 = "TEST_VAL1" + key = "TEST_KEY1" + data1 = "VAL" + data2 = "TEST_VAL1" nentries = 1000 scenarios = make_scenarios([ - ('var', dict(create_params = "key_format=S,value_format=S")), + ('row', dict(create_params = "key_format=S,value_format=S", + key = "TEST_KEY1", data1 = "VAL", data2 = "TEST_VAL1")), + ('var', dict(create_params = "key_format=r,value_format=S", + key = 123, data1 = "VAL", data2 = "TEST_VAL1")), + ('fix', dict(create_params = "key_format=r,value_format=8t", + key = 123, data1 = 0x17, data2 = 0xaa)), ]) def test_ops(self): @@ -52,17 +57,17 @@ class test_txn03(wttest.WiredTigerTestCase): # Set up the table with entries for 1 and 10 # We use the overwrite config so insert can update as needed. c = self.session.open_cursor(self.uri1, None, 'overwrite') - c[self.key_str] = self.data_str1 + c[self.key] = self.data1 c.close() c = self.session.open_cursor(self.uri2, None, 'overwrite') - c[self.key_str] = self.data_str1 + c[self.key] = self.data1 c.close() # Update the first table - this update should be visible in the # new session. self.session.begin_transaction() c = self.session.open_cursor(self.uri1, None, 'overwrite') - c[self.key_str] = self.data_str2 + c[self.key] = self.data2 self.session.commit_transaction() c.close() @@ -75,16 +80,16 @@ class test_txn03(wttest.WiredTigerTestCase): # Make an update in the first session. self.session.begin_transaction() c = self.session.open_cursor(self.uri2, None, 'overwrite') - c[self.key_str] = self.data_str2 + c[self.key] = self.data2 self.session.commit_transaction() c.close() - t1c.set_key(self.key_str) + t1c.set_key(self.key) t1c.search() - t2c.set_key(self.key_str) + t2c.set_key(self.key) t2c.search() - self.assertEqual(t1c.get_value(), self.data_str2) - self.assertEqual(t2c.get_value(), self.data_str1) + self.assertEqual(t1c.get_value(), self.data2) + self.assertEqual(t2c.get_value(), self.data1) # Clean up t1c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_txn06.py b/src/third_party/wiredtiger/test/suite/test_txn06.py index 86b0913a17b..1bcdac1d4b0 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn06.py +++ b/src/third_party/wiredtiger/test/suite/test_txn06.py @@ -32,6 +32,7 @@ from suite_subprocess import suite_subprocess from wtdataset import SimpleDataSet import wiredtiger, wttest +from wtscenario import make_scenarios class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): conn_config = 'verbose=[transaction]' @@ -40,13 +41,23 @@ class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): source_uri = 'table:' + tablename + "_src" nrows = 100000 + format_values = [ + ('row', dict(key_format = 'S', value_format='S')), + ('var', dict(key_format = 'r', value_format='S')), + ('fix', dict(key_format = 'r', value_format='8t')), + ] + scenarios = make_scenarios(format_values) + def test_long_running(self): # Populate a table - SimpleDataSet(self, self.source_uri, self.nrows).populate() + ds = SimpleDataSet(self, self.source_uri, self.nrows, + key_format=self.key_format, value_format=self.value_format) + ds.populate() # Now scan the table and copy the rows into a new table. The cursor will keep the snapshot # in self.session pinned while the inserts cause new IDs to be allocated. - c_src = self.session.create(self.uri, "key_format=S,value_format=S") + format = "key_format={},value_format={}".format(self.key_format, self.value_format) + c_src = self.session.create(self.uri, format) c_src = self.session.open_cursor(self.source_uri) insert_session = self.conn.open_session() c = insert_session.open_cursor(self.uri) diff --git a/src/third_party/wiredtiger/test/suite/test_txn07.py b/src/third_party/wiredtiger/test/suite/test_txn07.py index 44624f82af9..b25b9edf6cc 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn07.py +++ b/src/third_party/wiredtiger/test/suite/test_txn07.py @@ -49,13 +49,9 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): ] types = [ - ('row', dict(tabletype='row', - create_params = 'key_format=i,value_format=S')), - # The commented columnar tests needs to be enabled once rollback to stable for columnar is fixed in (WT-5548). - # ('var', dict(tabletype='var', - # create_params = 'key_format=r,value_format=S')), - # ('fix', dict(tabletype='fix', - # create_params = 'key_format=r,value_format=8t')), + ('row', dict(tabletype='row', create_params = 'key_format=i,value_format=S')), + ('var', dict(tabletype='var', create_params = 'key_format=r,value_format=S')), + ('fix', dict(tabletype='fix', create_params = 'key_format=r,value_format=8t')), ] op1s = [ ('trunc-all', dict(op1=('all', 0))), diff --git a/src/third_party/wiredtiger/test/suite/test_txn08.py b/src/third_party/wiredtiger/test/suite/test_txn08.py index d8f4188a5d8..9f1032272cd 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn08.py +++ b/src/third_party/wiredtiger/test/suite/test_txn08.py @@ -34,20 +34,27 @@ import fnmatch, os, shutil, run, time from suite_subprocess import suite_subprocess from wiredtiger import stat import wttest +from wtscenario import make_scenarios class test_txn08(wttest.WiredTigerTestCase, suite_subprocess): logmax = "100K" tablename = 'test_txn08' uri = 'table:' + tablename + key_format_values = [ + ('col', dict(key_format='r')), + ('row', dict(key_format='i')) + ] + scenarios = make_scenarios(key_format_values) + # Turn on logging for this test. def conn_config(self): return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ 'transaction_sync="(method=dsync,enabled)"' def test_printlog_unicode(self): - # print "Creating %s with config '%s'" % (self.uri, self.create_params) - create_params = 'key_format=i,value_format=S' + create_params = 'key_format={},value_format=S'.format(self.key_format) + # print "Creating %s with config '%s'" % (self.uri, create_params) self.session.create(self.uri, create_params) c = self.session.open_cursor(self.uri, None) @@ -56,7 +63,7 @@ class test_txn08(wttest.WiredTigerTestCase, suite_subprocess): value = u'\u0001\u0002abcd\u0003\u0004' self.session.begin_transaction() - for k in range(5): + for k in range(1, 6): c[k] = value self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_txn13.py b/src/third_party/wiredtiger/test/suite/test_txn13.py index 72bfe35386d..b11a3cb41be 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn13.py +++ b/src/third_party/wiredtiger/test/suite/test_txn13.py @@ -43,14 +43,20 @@ class test_txn13(wttest.WiredTigerTestCase, suite_subprocess): # We use 8 ops here to get around the 10 operation check done by WiredTiger to determine if # a transaction is blocking or not. nops = 8 - create_params = 'key_format=i,value_format=S' + + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] # The 1gb, 2gb and 4gb scenario names refer to the valuesize * nops. - scenarios = make_scenarios([ + size_values = [ ('1gb', dict(expect_err=False, valuesize=134217728)), ('2gb', dict(expect_err=False, valuesize=268435456)), ('4gb', dict(expect_err=True, valuesize=536870912)) - ]) + ] + + scenarios = make_scenarios(key_format_values, size_values) # Turn on logging for this test. def conn_config(self): @@ -59,9 +65,11 @@ class test_txn13(wttest.WiredTigerTestCase, suite_subprocess): @wttest.longtest('txn tests with huge values') def test_large_values(self): - # print "Creating %s with config '%s'" % (self.uri, self.create_params) + create_params = 'key_format={},value_format=S'.format(self.key_format) + + # print "Creating %s with config '%s'" % (self.uri, create_params) # print "Running with %d" % (self.valuesize) - self.session.create(self.uri, self.create_params) + self.session.create(self.uri, create_params) c = self.session.open_cursor(self.uri, None) # We want to test very large values. Generate 'nops' records within @@ -70,7 +78,7 @@ class test_txn13(wttest.WiredTigerTestCase, suite_subprocess): gotException = False self.session.begin_transaction() - for k in range(self.nops): + for k in range(1, self.nops + 1): value = valuepfx + str(k) c[k] = value diff --git a/src/third_party/wiredtiger/test/suite/test_txn14.py b/src/third_party/wiredtiger/test/suite/test_txn14.py index 24c51023cb4..f3eaac6cd6b 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn14.py +++ b/src/third_party/wiredtiger/test/suite/test_txn14.py @@ -38,7 +38,6 @@ import wttest class test_txn14(wttest.WiredTigerTestCase, suite_subprocess): t1 = 'table:test_txn14_1' - create_params = 'key_format=i,value_format=i' entries = 10000 extra_entries = 5 conn_config = 'log=(archive=false,enabled,file_max=100K)' @@ -46,9 +45,12 @@ class test_txn14(wttest.WiredTigerTestCase, suite_subprocess): sync_list = [ ('write', dict(sync='off')), ('sync', dict(sync='on')), - ('bg', dict(sync='background')), ] - scenarios = make_scenarios(sync_list) + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(sync_list, key_format_values) def test_log_flush(self): # Here's the strategy: @@ -59,32 +61,27 @@ class test_txn14(wttest.WiredTigerTestCase, suite_subprocess): # - Make recovery run. # - Confirm flushed data is in the table. # - self.session.create(self.t1, self.create_params) + create_params = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.t1, create_params) c = self.session.open_cursor(self.t1, None, None) - for i in range(self.entries): + for i in range(1, self.entries + 1): c[i] = i + 1 cfgarg='sync=%s' % self.sync self.pr('cfgarg ' + cfgarg) self.session.log_flush(cfgarg) - for i in range(self.extra_entries): + for i in range(1, self.extra_entries + 1): c[i+self.entries] = i + self.entries + 1 c.close() self.session.log_flush(cfgarg) - if self.sync == 'background': - # If doing a background flush, wait 30 seconds. I have seen an - # individual log file's fsync take more than a second on some - # systems, and we've seen timeouts at 10 seconds on systems - # with slow I/O. So give it time to flush perhaps a few files. - self.session.transaction_sync('timeout_ms=30000') simulate_crash_restart(self, ".", "RESTART") c = self.session.open_cursor(self.t1, None, None) - i = 0 + i = 1 for key, value in c: self.assertEqual(i, key) self.assertEqual(i+1, value) i += 1 all = self.entries + self.extra_entries - self.assertEqual(i, all) + self.assertEqual(i, all + 1) c.close() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_txn15.py b/src/third_party/wiredtiger/test/suite/test_txn15.py index 319f9a9ec3a..a266bf37582 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn15.py +++ b/src/third_party/wiredtiger/test/suite/test_txn15.py @@ -38,7 +38,6 @@ import wttest class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:test_txn15_1' - create_params = 'key_format=i,value_format=i' entries = 100 # Turn on logging for this test. def conn_config(self): @@ -48,6 +47,10 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): 'transaction_sync=(enabled=%s),' % self.conn_enable + \ 'transaction_sync=(method=%s),' % self.conn_method + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] conn_sync_enabled = [ ('en_off', dict(conn_enable='false')), ('en_on', dict(conn_enable='true')), @@ -71,7 +74,7 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): ('c_none', dict(commit_sync=None)), ('c_off', dict(commit_sync='sync=off')), ] - scenarios = make_scenarios(conn_sync_enabled, conn_sync_method, + scenarios = make_scenarios(key_format_values, conn_sync_enabled, conn_sync_method, begin_sync, commit_sync) # Given the different configuration settings determine if this group @@ -111,7 +114,8 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): if self.begin_sync != None and self.commit_sync != None: return - self.session.create(self.uri, self.create_params) + create_params = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.uri, create_params) stat_cursor = self.session.open_cursor('statistics:', None, None) # @@ -126,7 +130,7 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(self.uri, None, None) self.session.begin_transaction(self.begin_sync) - for i in range(self.entries): + for i in range(1, self.entries + 1): c[i] = i + 1 self.session.commit_transaction(self.commit_sync) c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_txn17.py b/src/third_party/wiredtiger/test/suite/test_txn17.py index 84b7a5feaa3..f1f2c8a2b49 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn17.py +++ b/src/third_party/wiredtiger/test/suite/test_txn17.py @@ -35,16 +35,13 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_txn17(wttest.WiredTigerTestCase, suite_subprocess): def test_txn_api(self): # Test API functionality tagged as requires_transaction. # Cannot set a timestamp on a non-running transaction. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1 << 5000)), + 'commit_timestamp=' + self.timestamp_str(1 << 5000)), '/only permitted in a running/') # Cannot call commit on a non-running transaction. @@ -71,12 +68,5 @@ class test_txn17(wttest.WiredTigerTestCase, suite_subprocess): '/not permitted in a running transaction/') self.session.rollback_transaction() - # Cannot call transaction_sync while a transaction is running. - self.session.begin_transaction() - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.transaction_sync(), - '/not permitted in a running transaction/') - self.session.rollback_transaction() - if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_txn18.py b/src/third_party/wiredtiger/test/suite/test_txn18.py index c764993d682..e8bb10bdbcf 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn18.py +++ b/src/third_party/wiredtiger/test/suite/test_txn18.py @@ -33,15 +33,21 @@ import fnmatch, os, shutil, time from suite_subprocess import suite_subprocess import wiredtiger, wttest +from wtscenario import make_scenarios class test_txn18(wttest.WiredTigerTestCase, suite_subprocess): t1 = 'table:test_txn18' - create_params = 'key_format=i,value_format=i' conn_config = 'log=(archive=false,enabled,file_max=100K),' + \ 'transaction_sync=(method=dsync,enabled)' conn_recerror = conn_config + ',log=(recover=error)' conn_recon = conn_config + ',log=(recover=on)' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def simulate_crash(self, olddir, newdir): ''' Simulate a crash from olddir and restart in newdir. ''' # with the connection still open, copy files to new directory @@ -71,13 +77,14 @@ class test_txn18(wttest.WiredTigerTestCase, suite_subprocess): # # If we aren't tracking file IDs properly, it's possible that # we'd end up apply the log records for t2 to table t1. - self.session.create(self.t1, self.create_params) + create_params = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.t1, create_params) # # Since we're logging, we need to flush out the meta-data file # from the create. self.session.checkpoint() c = self.session.open_cursor(self.t1, None, None) - for i in range(10000): + for i in range(1, 10001): c[i] = i + 1 c.close() olddir = "." @@ -103,12 +110,12 @@ class test_txn18(wttest.WiredTigerTestCase, suite_subprocess): # Make sure the data we added originally is there self.session = self.setUpSessionOpen(self.conn) c = self.session.open_cursor(self.t1, None, None) - i = 0 + i = 1 for key, value in c: self.assertEqual(i, key) self.assertEqual(i+1, value) i += 1 - self.assertEqual(i, 10000) + self.assertEqual(i, 10001) c.close() self.close_conn() # Reopening with recover=error after a clean shutdown should succeed. diff --git a/src/third_party/wiredtiger/test/suite/test_txn19.py b/src/third_party/wiredtiger/test/suite/test_txn19.py index 18c0636d603..d975dc43a7c 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn19.py +++ b/src/third_party/wiredtiger/test/suite/test_txn19.py @@ -113,6 +113,11 @@ class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): nrecords = [('nrecords=10', dict(nrecords=10)), ('nrecords=11', dict(nrecords=11))] + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + # This function prunes out unnecessary or problematic test cases # from the list of scenarios. def includeFunc(name, dictarg): @@ -131,11 +136,10 @@ class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): return True scenarios = make_scenarios( - corruption_type, corruption_pos, nrecords, + key_format_values, corruption_type, corruption_pos, nrecords, include=includeFunc, prune=20, prunelong=1000) uri = 'table:test_txn19' - create_params = 'key_format=i,value_format=S' # Return the log file number that contains the given record # number. In this test, two records fit into each log file, and @@ -282,7 +286,8 @@ class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): # Then does a restart with recovery, then starts again with salvage, # and finally starts again with recovery (adding new records). - self.session.create(self.uri, self.create_params) + create_params = 'key_format=i,value_format=S'.format(self.key_format) + self.session.create(self.uri, create_params) self.inserts([x for x in range(0, self.nrecords)]) newdir = "RESTART" copy_for_crash_restart(self.home, newdir) @@ -385,6 +390,11 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): ('WiredTiger.wt', dict(filename='WiredTiger.wt')), ('WiredTigerHS.wt', dict(filename='WiredTigerHS.wt')), ] + # Configure the database type. + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] # In many cases, wiredtiger_open without any salvage options will # just work. We list those cases here. @@ -433,10 +443,9 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): "garbage-end:WiredTiger.basecfg", ] - scenarios = make_scenarios(corruption_scenarios, filename_scenarios) + scenarios = make_scenarios(key_format_values, corruption_scenarios, filename_scenarios) uri = 'table:test_txn19_meta_' ntables = 5 - create_params = 'key_format=i,value_format=S' nrecords = 1000 # records per table. suffixes = [ str(x) for x in range(0, ntables)] # [ '0', '1', ... ] @@ -507,11 +516,12 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): def test_corrupt_meta(self): newdir = "RESTART" newdir2 = "RESTART2" - expect = list(range(0, self.nrecords)) + expect = list(range(1, self.nrecords + 1)) salvage_config = self.base_config + ',salvage=true' + create_params = 'key_format={},value_format=S'.format(self.key_format) for suffix in self.suffixes: - self.session.create(self.uri + suffix, self.create_params) + self.session.create(self.uri + suffix, create_params) self.inserts(expect) # Simulate a crash by copying the contents of the directory diff --git a/src/third_party/wiredtiger/test/suite/test_txn20.py b/src/third_party/wiredtiger/test/suite/test_txn20.py index d51e180fe20..b6d0c8306f2 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn20.py +++ b/src/third_party/wiredtiger/test/suite/test_txn20.py @@ -36,18 +36,21 @@ from wtscenario import make_scenarios class test_txn20(wttest.WiredTigerTestCase): uri = 'table:test_txn' + key_format_values = [ + ('string-row', dict(key_format='S', key='key')), + ('column', dict(key_format='r', key=12)), + ] iso_types = [ ('isolation_read_uncommitted', dict(isolation='read-uncommitted')), ('isolation_read_committed', dict(isolation='read-committed')), ('isolation_snapshot', dict(isolation='snapshot')) ] - scenarios = make_scenarios(iso_types) - key = 'key' + scenarios = make_scenarios(key_format_values, iso_types) old_value = 'value: old' new_value = 'value: new' def test_isolation_level(self): - self.session.create(self.uri, 'key_format=S,value_format=S') + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(self.uri, None) cursor[self.key] = self.old_value diff --git a/src/third_party/wiredtiger/test/suite/test_txn22.py b/src/third_party/wiredtiger/test/suite/test_txn22.py index 310c6f8a829..42da29540e4 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn22.py +++ b/src/third_party/wiredtiger/test/suite/test_txn22.py @@ -52,6 +52,11 @@ class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): base_config = 'cache_size=1GB' conn_config = base_config + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + # File to be corrupted filename_scenarios = [ ('WiredTiger', dict(filename='WiredTiger')), @@ -75,9 +80,8 @@ class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): "removal:WiredTiger.wt", ] - scenarios = make_scenarios(filename_scenarios) + scenarios = make_scenarios(key_format_values, filename_scenarios) uri = 'table:test_txn22' - create_params = 'key_format=i,value_format=S' nrecords = 1000 # records per table. def valuegen(self, i): @@ -115,10 +119,11 @@ class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): def test_corrupt_meta(self): newdir = "RESTART" newdir2 = "RESTART2" - expect = list(range(0, self.nrecords)) + expect = list(range(1, self.nrecords + 1)) salvage_config = self.base_config + ',salvage=true' - self.session.create(self.uri, self.create_params) + create_params = 'key_format={},value_format=S'.format(self.key_format) + self.session.create(self.uri, create_params) self.inserts(expect) # Simulate a crash by copying the contents of the directory diff --git a/src/third_party/wiredtiger/test/suite/test_txn23.py b/src/third_party/wiredtiger/test/suite/test_txn23.py index 87c2c53a103..c0d420ca92b 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn23.py +++ b/src/third_party/wiredtiger/test/suite/test_txn23.py @@ -32,26 +32,30 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_txn23(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' conn_config = 'cache_size=5MB' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def large_updates(self, uri, value, ds, nrows, commit_ts): # Update a large number of records. cursor = self.session.open_cursor(uri) - for i in range(0, nrows): + for i in range(1, nrows + 1): self.session.begin_transaction() cursor[ds.key(i)] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, ds, nrows, read_ts): - for i in range(0, nrows): - self.session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + for i in range(1, nrows + 1): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = self.session.open_cursor(uri) self.assertEqual(cursor[ds.key(i)], check_value) cursor.close() @@ -63,18 +67,18 @@ class test_txn23(wttest.WiredTigerTestCase): # Create a table. uri_1 = "table:txn23_1" ds_1 = SimpleDataSet( - self, uri_1, 0, key_format="i", value_format="S") + self, uri_1, 0, key_format=self.key_format, value_format="S") ds_1.populate() # Create another table. uri_2 = "table:txn23_2" ds_2 = SimpleDataSet( - self, uri_2, 0, key_format="i", value_format="S") + self, uri_2, 0, key_format=self.key_format, value_format="S") ds_2.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_txn24.py b/src/third_party/wiredtiger/test/suite/test_txn24.py index d6196d6fe85..11bb8a02f4e 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn24.py +++ b/src/third_party/wiredtiger/test/suite/test_txn24.py @@ -33,11 +33,18 @@ import wiredtiger, wttest import time +from wtscenario import make_scenarios class test_txn24(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def conn_config(self): # We want to either eliminate or keep the application thread role in eviction to minimum. # This will ensure that the dedicated eviction threads are doing the heavy lifting. @@ -49,14 +56,14 @@ class test_txn24(wttest.WiredTigerTestCase): # Create and populate a table. uri = "table:test_txn24" - table_params = 'key_format=i,value_format=S' + table_params = 'key_format={},value_format=S'.format(self.key_format) default_val = 'ABCD' * 60 new_val = 'YYYY' * 60 n_rows = 480000 self.session.create(uri, table_params) cursor = self.session.open_cursor(uri, None) - for i in range(0, n_rows): + for i in range(1, n_rows + 1): cursor[i] = default_val cursor.close() @@ -66,7 +73,7 @@ class test_txn24(wttest.WiredTigerTestCase): # Start a transaction, make an update and keep it running. cursor = self.session.open_cursor(uri, None) self.session.begin_transaction('isolation=snapshot') - cursor[0] = new_val + cursor[1] = new_val # Start few sessions and transactions, make updates and try committing them. session2 = self.setUpSessionOpen(self.conn) @@ -94,7 +101,7 @@ class test_txn24(wttest.WiredTigerTestCase): session4 = self.setUpSessionOpen(self.conn) cursor4 = session4.open_cursor(uri) - start_row = 1 + start_row = 2 for i in range(0, 120000): cursor4[start_row] = new_val start_row += 1 diff --git a/src/third_party/wiredtiger/test/suite/test_txn25.py b/src/third_party/wiredtiger/test/suite/test_txn25.py index c8cebdeb05e..6f1b25d4eec 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn25.py +++ b/src/third_party/wiredtiger/test/suite/test_txn25.py @@ -31,17 +31,31 @@ # import wiredtiger, wttest +from wtscenario import make_scenarios class test_txn25(wttest.WiredTigerTestCase): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] + scenarios = make_scenarios(key_format_values) + + def getkey(self, i): + if self.usestrings: + return str(i) + else: + return i + def test_txn25(self): uri = 'file:test_txn25' - create_config = 'allocation_size=512,key_format=S,value_format=S' + create_config = 'allocation_size=512,key_format={},value_format=S'.format(self.key_format) self.session.create(uri, create_config) # Populate the file and ensure that we start seeing some high transaction IDs in the system. + nrows = 1000 value1 = 'aaaaa' * 100 value2 = 'bbbbb' * 100 value3 = 'ccccc' * 100 @@ -51,19 +65,19 @@ class test_txn25(wttest.WiredTigerTestCase): session2.begin_transaction() cursor = self.session.open_cursor(uri) - for i in range(1, 1000): + for i in range(1, nrows): self.session.begin_transaction() - cursor[str(i)] = value1 + cursor[self.getkey(i)] = value1 self.session.commit_transaction() - for i in range(1, 1000): + for i in range(1, nrows): self.session.begin_transaction() - cursor[str(i)] = value2 + cursor[self.getkey(i)] = value2 self.session.commit_transaction() - for i in range(1, 1000): + for i in range(1, nrows): self.session.begin_transaction() - cursor[str(i)] = value3 + cursor[self.getkey(i)] = value3 self.session.commit_transaction() session2.rollback_transaction() @@ -81,6 +95,6 @@ class test_txn25(wttest.WiredTigerTestCase): # so we have to wipe the cell's transaction IDs in order to see them. cursor = self.session.open_cursor(uri) self.session.begin_transaction() - for i in range(1, 1000): - self.assertEqual(cursor[str(i)], value3) + for i in range(1, nrows): + self.assertEqual(cursor[self.getkey(i)], value3) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_txn26.py b/src/third_party/wiredtiger/test/suite/test_txn26.py index 4618585b7f1..e5b0a719867 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn26.py +++ b/src/third_party/wiredtiger/test/suite/test_txn26.py @@ -31,37 +31,44 @@ # [END_TAGS] import wiredtiger, wttest +from wtscenario import make_scenarios # test_txn26.py -# Test that commit should fail if commit timestamp is smaller or equal to the active timestamp. Our handling of out of order timestamp relies on this to ensure repeated reads are working as expected. -def timestamp_str(t): - return '%x' % t +# Test that commit should fail if commit timestamp is smaller or equal to the active timestamp. +# Our handling of out of order timestamp relies on this to ensure repeated reads are working as +# expected. class test_txn26(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', key=str(0))), + ('column', dict(key_format='r', key=16)), + ] + scenarios = make_scenarios(key_format_values) + def test_commit_larger_than_active_timestamp(self): if not wiredtiger.diagnostic_build(): self.skipTest('requires a diagnostic build') uri = 'table:test_txn26' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) value = 'a' # Start a session with timestamp 10 session2 = self.conn.open_session(self.session_config) - session2.begin_transaction('read_timestamp=' + timestamp_str(10)) + session2.begin_transaction('read_timestamp=' + self.timestamp_str(10)) # Try to commit at timestamp 10 self.session.begin_transaction() - cursor[str(0)] = value + cursor[self.key] = value with self.expectedStderrPattern("must be greater than the latest active read timestamp"): try: - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) except wiredtiger.WiredTigerError as e: gotException = True self.pr('got expected exception: ' + str(e)) diff --git a/src/third_party/wiredtiger/test/suite/test_util01.py b/src/third_party/wiredtiger/test/suite/test_util01.py index f01eb310c23..20c6cea62d6 100755 --- a/src/third_party/wiredtiger/test/suite/test_util01.py +++ b/src/third_party/wiredtiger/test/suite/test_util01.py @@ -30,9 +30,6 @@ import string, os, sys, random from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_util01.py # Utilities: wt dump, as well as the dump cursor class test_util01(wttest.WiredTigerTestCase, suite_subprocess): @@ -164,7 +161,7 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): expectout.write(self.dumpstr(key, hexoutput)) expectout.write(self.dumpstr(value, hexoutput)) if commit_timestamp is not None: - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) def dump(self, usingapi, hexoutput, commit_timestamp, read_timestamp): params = self.table_config() diff --git a/src/third_party/wiredtiger/test/suite/test_util21.py b/src/third_party/wiredtiger/test/suite/test_util21.py index 2271ad8b312..6dc3e05e042 100644 --- a/src/third_party/wiredtiger/test/suite/test_util21.py +++ b/src/third_party/wiredtiger/test/suite/test_util21.py @@ -30,9 +30,6 @@ import wttest from suite_subprocess import suite_subprocess from helper import compare_files -def timestamp_str(t): - return '%x' % t - # test_util21.py # Ensure that wt dump can dump obsolete data in the history store. class test_util21(wttest.WiredTigerTestCase, suite_subprocess): @@ -45,7 +42,7 @@ class test_util21(wttest.WiredTigerTestCase, suite_subprocess): for i in range(1, 5): self.session.begin_transaction() cursor[str(i)] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def test_dump_obsolete_data(self): @@ -58,7 +55,7 @@ class test_util21(wttest.WiredTigerTestCase, suite_subprocess): value3 = 'c' * 100 value4 = 'd' * 100 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) self.add_data_with_timestamp(uri, value1, 2) self.add_data_with_timestamp(uri, value2, 3) @@ -68,14 +65,14 @@ class test_util21(wttest.WiredTigerTestCase, suite_subprocess): self.session.checkpoint() # Set stable timestamp, so we don't lose data when closing/opening connection when using wt dump. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Call dump on the values before the oldest timestamp is set self.runWt(['dump', 'file:WiredTigerHS.wt'], outfilename="before_oldest") # Set oldest timestamp, and checkpoint, the obsolete data should not removed as # the pages are clean. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(6)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(6)) self.session.checkpoint() self.runWt(['dump', 'file:WiredTigerHS.wt'], outfilename="after_oldest") diff --git a/src/third_party/wiredtiger/test/suite/wtbackup.py b/src/third_party/wiredtiger/test/suite/wtbackup.py index fe34d563da8..25de77ba7fa 100644 --- a/src/third_party/wiredtiger/test/suite/wtbackup.py +++ b/src/third_party/wiredtiger/test/suite/wtbackup.py @@ -202,16 +202,21 @@ class backup_base(wttest.WiredTigerTestCase, suite_subprocess): # # Perform a block range copy for a given offset and file. # - def range_copy(self, filename, offset, size, backup_incr_dir): + def range_copy(self, filename, offset, size, backup_incr_dir, consolidate): read_from = filename write_to = backup_incr_dir + '/' + filename rfp = open(read_from, "rb") rfp.seek(offset, 0) buf = rfp.read(size) # Perform between previous incremental directory, to check that - # the old file and the new file is different. + # the old file and the new file is different. We can only ensure + # the data are different if running in consolidate mode. It's + # possible that we change multiple blocks in a single write and + # some of the blocks are the same as before. If we are not running + # in consolidate mode, these blocks which are copied separately one + # by one will trigger this assert. old_to = self.home_tmp + '/' + filename - if os.path.exists(old_to): + if os.path.exists(old_to) and consolidate: self.pr('RANGE CHECK file ' + old_to + ' offset ' + str(offset) + ' len ' + str(size)) old_rfp = open(old_to, "rb") old_rfp.seek(offset, 0) @@ -238,7 +243,7 @@ class backup_base(wttest.WiredTigerTestCase, suite_subprocess): # # Note: we return the sizes of WT_BACKUP_RANGE type files for tests that check for consolidate config. # - def take_incr_backup_block(self, bkup_c, newfile, backup_incr_dir): + def take_incr_backup_block(self, bkup_c, newfile, backup_incr_dir, consolidate): config = 'incremental=(file=' + newfile + ')' self.pr('Open incremental cursor with ' + config) # For each file listed, open a duplicate backup cursor and copy the blocks. @@ -262,7 +267,7 @@ class backup_base(wttest.WiredTigerTestCase, suite_subprocess): else: # Copy the block range. self.pr('Range copy file ' + newfile + ' offset ' + str(offset) + ' len ' + str(size)) - self.range_copy(newfile, offset, size, backup_incr_dir) + self.range_copy(newfile, offset, size, backup_incr_dir, consolidate) lens.append(size) incr_c.close() return lens @@ -319,7 +324,7 @@ class backup_base(wttest.WiredTigerTestCase, suite_subprocess): # If that changes then this, and the use of the duplicate below can change. while bkup_c.next() == 0: newfile = bkup_c.get_key() - file_sizes += self.take_incr_backup_block(bkup_c, newfile, backup_incr_dir) + file_sizes += self.take_incr_backup_block(bkup_c, newfile, backup_incr_dir, consolidate) file_names.append(newfile) # Copy into temp directory for tests that require further iterations of incremental backups. self.copy_file(newfile, self.home_tmp) diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 1d22162f535..01c4f315f9c 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -653,6 +653,9 @@ class WiredTigerTestCase(unittest.TestCase): """ self.assertEqual(int(ts1, 16), int(ts2, 16)) + def timestamp_str(self, t): + return '%x' % t + def exceptionToStderr(self, expr): """ Used by assertRaisesHavingMessage to convert an expression |
